Merge pull request 'An archive may be an upstream release pinned by URL and sha256, mirrored by the build seat (hq issue 345, ADR 0316)' (#255) from fix/345-builder-archive-pins into main

cmd/mesh-builder/Dockerfile

@@ -16,7 +16,8 @@ RUN CGO_ENABLED=0 go build -trimpath -ldflags '-s -w' -o /mesh-builder ./cmd/mes
 FROM ${ALPINE_BASE}
 # git to clone what it is asked to build, and the docker client to build and push it. The daemon
 # is the machine's, reached through its socket — a build machine shares the runtime it was given
-# rather than running one inside itself.
-RUN apk add --no-cache git docker-cli
+# rather than running one inside itself. xz to read an upstream release an archive is pinned to
+# (novox/hq ADR 0316): Go's standard library reads gzip and bzip2 and not xz.
+RUN apk add --no-cache git docker-cli xz
 COPY --from=build /mesh-builder /usr/local/bin/mesh-builder
 ENTRYPOINT ["/usr/local/bin/mesh-builder"]

internal/builder/builder.go

@@ -971,8 +971,19 @@ func one(ctx context.Context, run Runner, publish Publisher,
 		return catalogue.Built{Name: a.Name, Kind: a.Kind, Reference: reference}, nil
 
 	case catalogue.ArtifactArchive:
-		src.ownHas(cleanEntry(a.From) + "/**")
-		body, err := pack(filepath.Join(tree, a.From))
+		var body []byte
+		var err error
+		if a.URL != "" {
+			// An upstream file pinned by URL and sha256 (novox/hq ADR 0316): its build source is the pin, which
+			// is in module.json, always part of it.
+			body, err = upstreamArchive(ctx, workspace, a, say)
+			if err != nil {
+				return catalogue.Built{}, fmt.Errorf("%s: %s: %w", module, a.Name, err)
+			}
+		} else {
+			src.ownHas(cleanEntry(a.From) + "/**")
+			body, err = pack(filepath.Join(tree, a.From))
+		}
 		if err != nil {
 			return catalogue.Built{}, fmt.Errorf("%s: packing %s failed: %w", module, a.Name, err)
 		}

internal/builder/upstream_file.go

@@ -0,0 +1,326 @@
+package builder
+
+import (
+	"archive/tar"
+	"bufio"
+	"bytes"
+	"compress/bzip2"
+	"compress/gzip"
+	"context"
+	"crypto/sha256"
+	"encoding/hex"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"os"
+	"os/exec"
+	"path"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"github.com/novox/mesh-controller/internal/catalogue"
+)
+
+// An archive whose source is somebody else's published file, pinned by URL and sha256 (novox/hq ADR 0316
+// §2, §6, to-be 56 §7): a mail client's release, an add-on's xpi.
+//
+// **Fetched by the build seat, checked against the pin before anything is read from it, and packed as any
+// archive is packed.** What a machine receives is the mesh's own archive from the artifact store, by digest,
+// exactly as for a directory of the repository; no machine ever asks the vendor. A file whose bytes are not
+// the pin's fails the build and names both digests, because the address serving something else is the one
+// thing the pin exists to catch.
+//
+// **Kept in the build seat's workspace under its sha256**, so it is fetched once: a vendor that serves only
+// its newest version at one address (Owl) would otherwise break every later build of the same pin.
+
+// maxUpstream is the largest upstream file fetched, and the largest a release may unpack to: the node-engine
+// unpacks no archive larger than this either (mesh-host internal/apply/archive.go, maxArchive).
+const maxUpstream = 512 << 20
+
+// maxEntries is the most entries an upstream archive may hold: millions of empty ones would fill the build
+// seat's inodes and the node's. Betterbird's release holds 64.
+const maxEntries = 100_000
+
+// noDowngrade refuses a redirect off https: the digest still guards the bytes, and a fetch in the clear says
+// to whoever is on the path what the build seat fetches.
+func noDowngrade(req *http.Request, via []*http.Request) error {
+	if req.URL.Scheme != "https" {
+		return fmt.Errorf("%s redirected to %s, which is not https", via[0].URL, req.URL)
+	}
+	if len(via) >= 10 {
+		return errors.New("more than 10 redirects")
+	}
+	return nil
+}
+
+// upstreamClient fetches upstream files: a variable so a test serves its own.
+var upstreamClient = &http.Client{Timeout: 15 * time.Minute}
+
+// upstreamDir is where the build seat keeps the upstream files it fetched, by sha256.
+func upstreamDir(workspace string) string { return filepath.Join(workspace, "upstream") }
+
+// upstreamArchive is the packed archive of an artifact pinned by URL: the file fetched (or kept from an
+// earlier build) and checked, then laid out as the artifact says and packed.
+func upstreamArchive(ctx context.Context, workspace string, a catalogue.Artifact,
+	say func(step, format string, args ...any)) ([]byte, error) {
+	file, err := upstreamFile(ctx, workspace, a, say)
+	if err != nil {
+		return nil, err
+	}
+	laid := filepath.Join(workspace, "upstream-"+a.Name)
+	if err := os.RemoveAll(laid); err != nil {
+		return nil, err
+	}
+	defer os.RemoveAll(laid)
+	if err := os.MkdirAll(laid, 0o755); err != nil {
+		return nil, err
+	}
+	if a.File != "" {
+		body, err := os.ReadFile(file)
+		if err != nil {
+			return nil, err
+		}
+		if err := os.WriteFile(filepath.Join(laid, a.File), body, 0o644); err != nil {
+			return nil, err
+		}
+	} else if err := unpackUpstream(ctx, file, laid, a.Within); err != nil {
+		return nil, fmt.Errorf("%s (%s): %w", a.URL, a.Version, err)
+	}
+	return pack(laid)
+}
+
+// upstreamFile is the path of the pinned file in the workspace: kept from an earlier build when its bytes
+// are still the pin's, else fetched now and kept only when they are.
+func upstreamFile(ctx context.Context, workspace string, a catalogue.Artifact,
+	say func(step, format string, args ...any)) (string, error) {
+	want := strings.ToLower(a.SHA256)
+	dir := upstreamDir(workspace)
+	kept := filepath.Join(dir, want)
+	if got, err := sumOf(kept); err == nil {
+		if got == want {
+			say("upstream", "%s %s: kept from an earlier build (sha256:%s)", a.Name, a.Version, short(want))
+			return kept, nil
+		}
+		// Changed on disk since it was kept: fetched again rather than trusted.
+		os.Remove(kept)
+	}
+	if err := os.MkdirAll(dir, 0o755); err != nil {
+		return "", err
+	}
+	say("upstream", "fetching %s (%s)", a.URL, a.Version)
+	request, err := http.NewRequestWithContext(ctx, http.MethodGet, a.URL, nil)
+	if err != nil {
+		return "", err
+	}
+	client := *upstreamClient
+	client.CheckRedirect = noDowngrade
+	response, err := client.Do(request)
+	if err != nil {
+		return "", fmt.Errorf("cannot fetch %s: %w", a.URL, err)
+	}
+	defer response.Body.Close()
+	if response.StatusCode != http.StatusOK {
+		return "", fmt.Errorf("%s answered %s", a.URL, response.Status)
+	}
+	part, err := os.CreateTemp(dir, want+".*.part")
+	if err != nil {
+		return "", err
+	}
+	defer os.Remove(part.Name())
+	hash := sha256.New()
+	n, err := io.Copy(io.MultiWriter(part, hash), io.LimitReader(response.Body, maxUpstream+1))
+	if cerr := part.Close(); err == nil {
+		err = cerr
+	}
+	if err != nil {
+		return "", fmt.Errorf("fetching %s: %w", a.URL, err)
+	}
+	if n > maxUpstream {
+		return "", fmt.Errorf("%s is larger than %d bytes, more than a machine unpacks; nothing was kept", a.URL, maxUpstream)
+	}
+	got := hex.EncodeToString(hash.Sum(nil))
+	if got != want {
+		return "", fmt.Errorf("%s was pinned as sha256:%s and what it served is sha256:%s (%d bytes); nothing "+
+			"was kept. The address serves another file than the one reviewed: compare it, then move the pin "+
+			"in a reviewed change (novox/hq ADR 0316)", a.URL, want, got, n)
+	}
+	if err := os.Rename(part.Name(), kept); err != nil {
+		return "", err
+	}
+	say("upstream", "%s %s: %d bytes, sha256:%s as pinned", a.Name, a.Version, n, short(want))
+	return kept, nil
+}
+
+// sumOf is the sha256 of a file, in hexadecimal.
+func sumOf(file string) (string, error) {
+	f, err := os.Open(file)
+	if err != nil {
+		return "", err
+	}
+	defer f.Close()
+	hash := sha256.New()
+	if _, err := io.Copy(hash, f); err != nil {
+		return "", err
+	}
+	return hex.EncodeToString(hash.Sum(nil)), nil
+}
+
+// unpackUpstream unpacks a tar archive, compressed with gzip, bzip2 or xz (told by its first bytes), into a
+// directory: of its entries only those within `within` (all, when empty), with that prefix taken off.
+//
+// **Only files and directories, and nothing outside.** A link, a device or an entry naming a path outside
+// the archive, or outside `within`, refuses the build by name: the node-engine unpacks only files and
+// directories, and an archive quietly unpacked without what it held would be a release that is not the one
+// pinned.
+func unpackUpstream(ctx context.Context, file, into, within string) error {
+	f, err := os.Open(file)
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+	buffered := bufio.NewReader(f)
+	magic, _ := buffered.Peek(6)
+	var stream io.Reader
+	var wait func() error
+	switch {
+	case bytes.HasPrefix(magic, []byte{0x1f, 0x8b}):
+		zipped, err := gzip.NewReader(buffered)
+		if err != nil {
+			return err
+		}
+		defer zipped.Close()
+		stream = zipped
+	case bytes.HasPrefix(magic, []byte("BZh")):
+		stream = bzip2.NewReader(buffered)
+	case bytes.HasPrefix(magic, []byte{0xfd, '7', 'z', 'X', 'Z', 0x00}):
+		// Go's standard library reads no xz; the build seat's image carries xz (cmd/mesh-builder/Dockerfile).
+		if _, err := exec.LookPath("xz"); err != nil {
+			return errors.New("it is compressed with xz, and this build seat has no xz to read it")
+		}
+		cmd := exec.CommandContext(ctx, "xz", "-dc")
+		cmd.Stdin = buffered
+		var stderr bytes.Buffer
+		cmd.Stderr = &stderr
+		out, err := cmd.StdoutPipe()
+		if err != nil {
+			return err
+		}
+		if err := cmd.Start(); err != nil {
+			return err
+		}
+		stream = out
+		wait = func() error {
+			if err := cmd.Wait(); err != nil {
+				return fmt.Errorf("xz could not read it: %v %s", err, strings.TrimSpace(stderr.String()))
+			}
+			return nil
+		}
+		defer func() {
+			if wait != nil {
+				_ = cmd.Process.Kill()
+				_ = cmd.Wait()
+			}
+		}()
+	default:
+		return errors.New("it is not a tar archive compressed with gzip, bzip2 or xz; a single file is kept " +
+			"whole by naming it in `file`")
+	}
+	if err := untar(stream, into, within); err != nil {
+		return err
+	}
+	if wait != nil {
+		// Whatever follows the tar's end is read before xz is waited for: xz blocks on a full pipe otherwise.
+		if _, err := io.Copy(io.Discard, stream); err != nil {
+			return err
+		}
+		w := wait
+		wait = nil
+		return w()
+	}
+	return nil
+}
+
+// untar writes a tar stream's files and directories within `within` into a directory.
+func untar(stream io.Reader, into, within string) error {
+	reader := tar.NewReader(stream)
+	prefix := ""
+	if within != "" {
+		prefix = path.Clean(within) + "/"
+	}
+	var total int64
+	files, entries := 0, 0
+	for {
+		header, err := reader.Next()
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			return err
+		}
+		if entries++; entries > maxEntries {
+			return fmt.Errorf("it holds more than %d entries", maxEntries)
+		}
+		if header.Typeflag == tar.TypeXGlobalHeader {
+			// A pax global header (git archive writes one) names no file.
+			continue
+		}
+		name := path.Clean(strings.TrimPrefix(header.Name, "./"))
+		if path.IsAbs(name) || name == ".." || strings.HasPrefix(name, "../") {
+			return fmt.Errorf("%s names a path outside the archive", header.Name)
+		}
+		if name == "." || name+"/" == prefix ||
+			(header.Typeflag == tar.TypeDir && strings.HasPrefix(prefix, name+"/")) {
+			// The archive's root, the directory taken, or one of its parents.
+			continue
+		}
+		if prefix != "" {
+			if !strings.HasPrefix(name, prefix) {
+				return fmt.Errorf("%s is outside %s, the directory the artifact is taken from", header.Name, within)
+			}
+			name = strings.TrimPrefix(name, prefix)
+		}
+		target := filepath.Join(into, filepath.FromSlash(name))
+		switch header.Typeflag {
+		case tar.TypeDir:
+			if err := os.MkdirAll(target, 0o755); err != nil {
+				return err
+			}
+		case tar.TypeReg:
+			total += header.Size
+			if total > maxUpstream {
+				return fmt.Errorf("it unpacks to more than %d bytes, more than a machine unpacks", maxUpstream)
+			}
+			if err := os.MkdirAll(filepath.Dir(target), 0o755); err != nil {
+				return err
+			}
+			mode := os.FileMode(0o644)
+			if header.Mode&0o111 != 0 {
+				mode = 0o755
+			}
+			out, err := os.OpenFile(target, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, mode)
+			if err != nil {
+				return err
+			}
+			if _, err := io.Copy(out, io.LimitReader(reader, header.Size)); err != nil {
+				out.Close()
+				return err
+			}
+			if err := out.Close(); err != nil {
+				return err
+			}
+			files++
+		default:
+			return fmt.Errorf("%s is a %c, and an archive carries only files and directories", header.Name,
+				header.Typeflag)
+		}
+	}
+	if files == 0 {
+		if within != "" {
+			return fmt.Errorf("it holds no file within %s", within)
+		}
+		return errors.New("it holds no file")
+	}
+	return nil
+}

internal/builder/upstream_file_test.go

@@ -0,0 +1,385 @@
+package builder
+
+import (
+	"archive/tar"
+	"bytes"
+	"compress/gzip"
+	"context"
+	"crypto/sha256"
+	"encoding/hex"
+	"fmt"
+	"io"
+	"net"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"sort"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/novox/mesh-controller/internal/catalogue"
+)
+
+// An archive pinned by URL and sha256 (novox/hq ADR 0316 §2, §6; How it is checked: "an upstream artifact
+// whose download does not match its sha256 fails the build and names both digests").
+
+type tarEntry struct {
+	name, body string
+	mode       int64
+	kind       byte
+}
+
+func tarOf(t *testing.T, entries []tarEntry) []byte {
+	t.Helper()
+	var raw bytes.Buffer
+	w := tar.NewWriter(&raw)
+	for _, e := range entries {
+		kind := e.kind
+		if kind == 0 {
+			kind = tar.TypeReg
+		}
+		h := &tar.Header{Name: e.name, Mode: e.mode, Typeflag: kind, Size: int64(len(e.body))}
+		if kind == tar.TypeSymlink {
+			h.Linkname, h.Size = "/usr/share/hunspell", 0
+		}
+		if kind == tar.TypeDir {
+			h.Size = 0
+		}
+		if err := w.WriteHeader(h); err != nil {
+			t.Fatal(err)
+		}
+		if kind == tar.TypeReg {
+			if _, err := w.Write([]byte(e.body)); err != nil {
+				t.Fatal(err)
+			}
+		}
+	}
+	if err := w.Close(); err != nil {
+		t.Fatal(err)
+	}
+	return raw.Bytes()
+}
+
+func gzipped(t *testing.T, body []byte) []byte {
+	t.Helper()
+	var out bytes.Buffer
+	z := gzip.NewWriter(&out)
+	if _, err := z.Write(body); err != nil {
+		t.Fatal(err)
+	}
+	if err := z.Close(); err != nil {
+		t.Fatal(err)
+	}
+	return out.Bytes()
+}
+
+func sumHex(b []byte) string {
+	s := sha256.Sum256(b)
+	return hex.EncodeToString(s[:])
+}
+
+// vendor serves files by path over https and counts what it was asked.
+type vendor struct {
+	files map[string][]byte
+	asked int
+}
+
+func serveVendor(t *testing.T, v *vendor) string {
+	t.Helper()
+	server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		v.asked++
+		body, ok := v.files[r.URL.Path]
+		if !ok {
+			http.NotFound(w, r)
+			return
+		}
+		w.Write(body)
+	}))
+	t.Cleanup(server.Close)
+	before := upstreamClient
+	// Under a public name, as a manifest must say it: example.com, which the test server's certificate
+	// names, dialled at the server.
+	transport := server.Client().Transport.(*http.Transport).Clone()
+	addr := server.Listener.Addr().String()
+	transport.DialContext = func(ctx context.Context, network, _ string) (net.Conn, error) {
+		return (&net.Dialer{}).DialContext(ctx, network, addr)
+	}
+	upstreamClient = &http.Client{Transport: transport}
+	t.Cleanup(func() { upstreamClient = before })
+	return "https://example.com"
+}
+
+// entriesOf lists a packed archive: each file with its mode and content.
+func entriesOf(t *testing.T, packed []byte) map[string]string {
+	t.Helper()
+	z, err := gzip.NewReader(bytes.NewReader(packed))
+	if err != nil {
+		t.Fatal(err)
+	}
+	r := tar.NewReader(z)
+	out := map[string]string{}
+	for {
+		h, err := r.Next()
+		if err == io.EOF {
+			return out
+		}
+		if err != nil {
+			t.Fatal(err)
+		}
+		body, _ := io.ReadAll(r)
+		out[h.Name] = string(body) + " " + os.FileMode(h.Mode).String()
+	}
+}
+
+func release(t *testing.T) []byte {
+	return gzipped(t, tarOf(t, []tarEntry{
+		{name: "betterbird/", kind: tar.TypeDir, mode: 0o755},
+		{name: "betterbird/application.ini", body: "[App]\nVersion=153.4.0\n", mode: 0o644},
+		{name: "betterbird/betterbird", body: "#!/bin/sh\n", mode: 0o755},
+		{name: "betterbird/defaults/pref/channel-prefs.js", body: "pref(\"app.update.channel\", \"default\");\n", mode: 0o644},
+	}))
+}
+
+func pinnedManifest(base, sum string) string {
+	return `{"module":"betterbird","version":"1",
+  "build":{"artifacts":[
+    {"name":"archive","kind":"archive","url":"` + base + `/release.tar.gz","sha256":"` + sum + `","version":"153.4.0esr-bb10","within":"betterbird"}]},
+  "resources":[{"id":"installation","type":"archive","path":"/opt/betterbird","artifact":"archive"}]}`
+}
+
+func TestAnArchivePinnedByURLIsFetchedCheckedAndPackedWithoutItsOwnDirectory(t *testing.T) {
+	body := release(t)
+	v := &vendor{files: map[string][]byte{"/release.tar.gz": body}}
+	base := serveVendor(t, v)
+	r, workspace := aRepository(t, pinnedManifest(base, sumHex(body)), nil)
+	got, err := Build(context.Background(), r.run, r, "https://forge.invalid/catalogue.git", "", "", workspace, nil,
+		Npmrc{}, GitCredential{}, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	digest, _ := got.Manifest.Resources[0]["digest"].(string)
+	if !strings.HasPrefix(digest, "sha256:") || r.archives["betterbird/archive"] != digest {
+		t.Fatalf("the release was not published and pinned as the module's archive: %v, published %v",
+			got.Manifest.Resources[0], r.archives)
+	}
+
+	packed, err := upstreamArchive(context.Background(), workspace, catalogue.Artifact{Name: "archive", Kind: "archive",
+		URL: base + "/release.tar.gz", SHA256: sumHex(body), Version: "153.4.0esr-bb10", Within: "betterbird"},
+		func(string, string, ...any) {})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if "sha256:"+sumHex(packed) != digest {
+		t.Fatalf("a second packing of one pin gave another digest: %s and %s", digest, sumHex(packed))
+	}
+	entries := entriesOf(t, packed)
+	want := map[string]string{
+		"application.ini":                "[App]\nVersion=153.4.0\n -rw-r--r--",
+		"betterbird":                     "#!/bin/sh\n -rwxr-xr-x",
+		"defaults/pref/channel-prefs.js": "pref(\"app.update.channel\", \"default\");\n -rw-r--r--",
+	}
+	if len(entries) != len(want) {
+		t.Fatalf("the archive holds %v, want %v", entries, want)
+	}
+	for name, w := range want {
+		if entries[name] != w {
+			t.Errorf("%s is %q in the archive, want %q", name, entries[name], w)
+		}
+	}
+}
+
+func TestAnUpstreamFileIsFetchedOnceAndKeptByItsDigest(t *testing.T) {
+	// The vendor of an add-on serves only its newest version at one address: a pin that was fetched once
+	// must still build after the address moved on.
+	body := release(t)
+	v := &vendor{files: map[string][]byte{"/release.tar.gz": body}}
+	base := serveVendor(t, v)
+	r, workspace := aRepository(t, pinnedManifest(base, sumHex(body)), nil)
+	first, err := Build(context.Background(), r.run, r, "https://forge.invalid/catalogue.git", "", "", workspace, nil,
+		Npmrc{}, GitCredential{}, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	v.files["/release.tar.gz"] = []byte("the vendor's next release")
+	second, err := Build(context.Background(), r.run, r, "https://forge.invalid/catalogue.git", "", "", workspace, nil,
+		Npmrc{}, GitCredential{}, nil)
+	if err != nil {
+		t.Fatalf("a pin fetched before failed to build once the address served something else: %v", err)
+	}
+	if v.asked != 1 {
+		t.Fatalf("the vendor was asked %d times for one pin, want once", v.asked)
+	}
+	if first.Manifest.Resources[0]["digest"] != second.Manifest.Resources[0]["digest"] {
+		t.Fatal("two builds of one pin gave two digests")
+	}
+}
+
+func TestAnUpstreamFileWhoseDownloadDoesNotMatchItsPinFailsAndNamesBothDigests(t *testing.T) {
+	pinned := release(t)
+	served := gzipped(t, tarOf(t, []tarEntry{{name: "betterbird/betterbird", body: "not what was reviewed", mode: 0o755}}))
+	v := &vendor{files: map[string][]byte{"/release.tar.gz": served}}
+	base := serveVendor(t, v)
+	r, workspace := aRepository(t, pinnedManifest(base, sumHex(pinned)), nil)
+	_, err := Build(context.Background(), r.run, r, "https://forge.invalid/catalogue.git", "", "", workspace, nil,
+		Npmrc{}, GitCredential{}, nil)
+	if err == nil {
+		t.Fatal("a download that is not the pinned file was built")
+	}
+	for _, digest := range []string{sumHex(pinned), sumHex(served)} {
+		if !strings.Contains(err.Error(), "sha256:"+digest) {
+			t.Errorf("the refusal does not name sha256:%s: %v", digest, err)
+		}
+	}
+	if len(r.archives) != 0 {
+		t.Fatalf("something was published: %v", r.archives)
+	}
+	kept, _ := filepath.Glob(filepath.Join(upstreamDir(workspace), "*"))
+	if len(kept) != 0 {
+		t.Fatalf("the refused download was kept: %v", kept)
+	}
+}
+
+func TestAnUpstreamFileThatIsNotAnArchiveIsKeptWholeUnderItsName(t *testing.T) {
+	xpi := []byte("PK\x03\x04 an add-on")
+	v := &vendor{files: map[string][]byte{"/owl.xpi": xpi}}
+	base := serveVendor(t, v)
+	packed, err := upstreamArchive(context.Background(), t.TempDir(), catalogue.Artifact{Name: "owl", Kind: "archive",
+		URL: base + "/owl.xpi", SHA256: sumHex(xpi), Version: "1.5.6.1", File: "owl.xpi"}, func(string, string, ...any) {})
+	if err != nil {
+		t.Fatal(err)
+	}
+	entries := entriesOf(t, packed)
+	if len(entries) != 1 || entries["owl.xpi"] != string(xpi)+" -rw-r--r--" {
+		t.Fatalf("the add-on was not kept whole as owl.xpi: %v", entries)
+	}
+}
+
+func TestAnUpstreamArchiveHoldingALinkOrAnEntryOutsideItsDirectoryIsRefused(t *testing.T) {
+	cases := map[string][]tarEntry{
+		// The community package links the dictionaries; a release carrying a link is not one a machine unpacks.
+		"is a 2": {{name: "betterbird/betterbird", body: "x", mode: 0o755},
+			{name: "betterbird/dictionaries", kind: tar.TypeSymlink}},
+		"outside betterbird": {{name: "betterbird/betterbird", body: "x", mode: 0o755},
+			{name: "README", body: "x", mode: 0o644}},
+		"outside the archive": {{name: "../etc/passwd", body: "x", mode: 0o644}},
+		"no file within":      {{name: "betterbird/", kind: tar.TypeDir, mode: 0o755}},
+	}
+	for want, entries := range cases {
+		body := gzipped(t, tarOf(t, entries))
+		v := &vendor{files: map[string][]byte{"/r.tar.gz": body}}
+		base := serveVendor(t, v)
+		_, err := upstreamArchive(context.Background(), t.TempDir(), catalogue.Artifact{Name: "archive", Kind: "archive",
+			URL: base + "/r.tar.gz", SHA256: sumHex(body), Version: "1", Within: "betterbird"}, func(string, string, ...any) {})
+		if err == nil || !strings.Contains(err.Error(), want) {
+			t.Errorf("an archive whose entry is %q was not refused for it: %v", want, err)
+		}
+	}
+}
+
+func TestAnUpstreamArchiveCompressedWithXzIsRead(t *testing.T) {
+	// Betterbird publishes its Linux release as .tar.xz; the build seat's image carries xz for it.
+	if _, err := exec.LookPath("xz"); err != nil {
+		t.Skip("NOT TESTED here: no xz on this machine (the build seat's image carries it)")
+	}
+	plain := tarOf(t, []tarEntry{{name: "betterbird/betterbird", body: "#!/bin/sh\n", mode: 0o755}})
+	cmd := exec.Command("xz", "-zc")
+	cmd.Stdin = bytes.NewReader(plain)
+	compressed, err := cmd.Output()
+	if err != nil {
+		t.Fatal(err)
+	}
+	v := &vendor{files: map[string][]byte{"/r.tar.xz": compressed}}
+	base := serveVendor(t, v)
+	packed, err := upstreamArchive(context.Background(), t.TempDir(), catalogue.Artifact{Name: "archive", Kind: "archive",
+		URL: base + "/r.tar.xz", SHA256: sumHex(compressed), Version: "1", Within: "betterbird"}, func(string, string, ...any) {})
+	if err != nil {
+		t.Fatal(err)
+	}
+	var names []string
+	for n := range entriesOf(t, packed) {
+		names = append(names, n)
+	}
+	sort.Strings(names)
+	if strings.Join(names, ",") != "betterbird" {
+		t.Fatalf("the xz release unpacked to %v", names)
+	}
+}
+
+func TestAnUpstreamTarWithAGlobalHeaderNestedWithinAndTrailingDataIsRead(t *testing.T) {
+	// git archive writes a pax global header; a release may unpack under a/b with a/ listed first; xz may be
+	// followed by bytes after the tar's end, which must be read before xz is waited for.
+	var raw bytes.Buffer
+	w := tar.NewWriter(&raw)
+	w.WriteHeader(&tar.Header{Typeflag: tar.TypeXGlobalHeader, Name: "pax_global_header", PAXRecords: map[string]string{"comment": "abc"}})
+	w.WriteHeader(&tar.Header{Name: "a/", Typeflag: tar.TypeDir, Mode: 0o755})
+	w.WriteHeader(&tar.Header{Name: "a/b/", Typeflag: tar.TypeDir, Mode: 0o755})
+	w.WriteHeader(&tar.Header{Name: "a/b/f", Typeflag: tar.TypeReg, Mode: 0o644, Size: 1})
+	w.Write([]byte("x"))
+	w.Close()
+	body := gzipped(t, raw.Bytes())
+	v := &vendor{files: map[string][]byte{"/r.tar.gz": body}}
+	base := serveVendor(t, v)
+	packed, err := upstreamArchive(context.Background(), t.TempDir(), catalogue.Artifact{Name: "a", Kind: "archive",
+		URL: base + "/r.tar.gz", SHA256: sumHex(body), Version: "1", Within: "a/b"}, func(string, string, ...any) {})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if e := entriesOf(t, packed); len(e) != 1 || e["f"] == "" {
+		t.Fatalf("%v", e)
+	}
+	if _, err := exec.LookPath("xz"); err != nil {
+		t.Skip("NOT TESTED here: no xz")
+	}
+	// A megabyte after the tar's end: more than a pipe holds.
+	trailing := append(append([]byte{}, raw.Bytes()...), bytes.Repeat([]byte{0}, 1<<20)...)
+	cmd := exec.Command("xz", "-zc")
+	cmd.Stdin = bytes.NewReader(trailing)
+	compressed, err := cmd.Output()
+	if err != nil {
+		t.Fatal(err)
+	}
+	v.files["/r.tar.xz"] = compressed
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+	if _, err := upstreamArchive(ctx, t.TempDir(), catalogue.Artifact{Name: "a", Kind: "archive", URL: base + "/r.tar.xz",
+		SHA256: sumHex(compressed), Version: "1", Within: "a/b"}, func(string, string, ...any) {}); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestAnUpstreamArchiveWithTooManyEntriesIsRefused(t *testing.T) {
+	var raw bytes.Buffer
+	w := tar.NewWriter(&raw)
+	for i := 0; i <= maxEntries; i++ {
+		w.WriteHeader(&tar.Header{Name: fmt.Sprintf("d%d/", i), Typeflag: tar.TypeDir, Mode: 0o755})
+	}
+	w.Close()
+	body := gzipped(t, raw.Bytes())
+	v := &vendor{files: map[string][]byte{"/r.tar.gz": body}}
+	base := serveVendor(t, v)
+	_, err := upstreamArchive(context.Background(), t.TempDir(), catalogue.Artifact{Name: "a", Kind: "archive",
+		URL: base + "/r.tar.gz", SHA256: sumHex(body), Version: "1"}, func(string, string, ...any) {})
+	if err == nil || !strings.Contains(err.Error(), "entries") {
+		t.Fatalf("%v", err)
+	}
+}
+
+func TestARedirectOffHttpsIsNotFollowed(t *testing.T) {
+	plain := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Write([]byte("x")) }))
+	t.Cleanup(plain.Close)
+	secure := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		http.Redirect(w, r, plain.URL+"/r", http.StatusFound)
+	}))
+	t.Cleanup(secure.Close)
+	before := upstreamClient
+	upstreamClient = secure.Client()
+	t.Cleanup(func() { upstreamClient = before })
+	_, err := upstreamFile(context.Background(), t.TempDir(), catalogue.Artifact{Name: "a", URL: secure.URL + "/r",
+		SHA256: sumHex([]byte("x")), Version: "1", File: "a"}, func(string, string, ...any) {})
+	if err == nil || !strings.Contains(err.Error(), "not https") {
+		t.Fatalf("%v", err)
+	}
+}

internal/catalogue/build.go

@@ -2,7 +2,10 @@ package catalogue
 
 import (
 	"fmt"
+	"net"
+	"net/url"
 	"path"
+	"regexp"
 	"sort"
 	"strings"
 )
@@ -312,7 +315,7 @@ func (b *Build) problems(module string) []string {
 					module, a.Name, a.System, spokenSystems()))
 			}
 		} else {
-			if a.From == "" {
+			if a.From == "" && a.URL == "" {
 				problems = append(problems, fmt.Sprintf(
 					"%s: %q says nothing about what it is built from", module, a.Name))
 			}
@@ -350,6 +353,85 @@ func (b *Build) problems(module string) []string {
 			problems = append(problems, fmt.Sprintf(
 				"%s: %q mirrors %q, which names no tag or digest", module, a.Name, a.From))
 		}
+		problems = append(problems, upstreamFileProblems(module, a)...)
+	}
+	return problems
+}
+
+// reservedSuffixes are names no public vendor publishes under: an installation's own and documentation's.
+var reservedSuffixes = []string{".internal", ".local", ".lan", ".home", ".corp", ".localhost", ".home.arpa",
+	".test", ".invalid", ".example", ".intranet", ".private"}
+
+// notAPublicHost says why a host is not a public vendor's name, or "".
+func notAPublicHost(host string) string {
+	h := strings.ToLower(strings.TrimSuffix(host, "."))
+	if net.ParseIP(h) != nil {
+		return "an address rather than a name"
+	}
+	if !strings.Contains(h, ".") {
+		return "a name with no domain"
+	}
+	for _, s := range reservedSuffixes {
+		if strings.HasSuffix(h, s) {
+			return "a name under " + s + ", which is nobody's public domain"
+		}
+	}
+	return ""
+}
+
+// sha256Hex is a pinned sha256 as a manifest writes it: 64 lower-case hexadecimal characters.
+var sha256Hex = regexp.MustCompile(`^[0-9a-f]{64}$`)
+
+// upstreamFileProblems is what is wrong with an archive pinned by URL (novox/hq ADR 0316 §2, §6): it is an
+// archive, fetched over https, pinned by a sha256 and a version, and it names no directory of the repository
+// besides; a file name or a directory within is a plain path that stays where it is put.
+func upstreamFileProblems(module string, a Artifact) []string {
+	pinned := a.URL != "" || a.SHA256 != "" || a.Version != "" || a.File != "" || a.Within != ""
+	if !pinned {
+		return nil
+	}
+	var problems []string
+	say := func(format string, args ...any) {
+		problems = append(problems, fmt.Sprintf("%s: %q ", module, a.Name)+fmt.Sprintf(format, args...))
+	}
+	if a.Kind != ArtifactArchive {
+		say("is a %q and names an upstream file (url, sha256, version, file, within): only an archive is "+
+			"mirrored from one (novox/hq ADR 0316)", a.Kind)
+		return problems
+	}
+	if a.URL == "" {
+		say("names a sha256, a version, a file or a directory within, and no url to fetch it from")
+		return problems
+	}
+	if a.From != "" {
+		say("is built from %q and fetched from %s: an archive is one or the other", a.From, a.URL)
+	}
+	if u, err := url.Parse(a.URL); err != nil || u.Scheme != "https" || u.Host == "" {
+		say("is fetched from %q, and an upstream file is fetched over https from a named host", a.URL)
+	} else if why := notAPublicHost(u.Hostname()); why != "" {
+		say("is fetched from %s, %s: an upstream file is somebody else's release, published under a public name", a.URL, why)
+	}
+	if !sha256Hex.MatchString(a.SHA256) {
+		say("is fetched from %s without a pinned sha256 (64 lower-case hexadecimal characters): what that "+
+			"address serves changes under the mesh, and only the digest says it is what was reviewed", a.URL)
+	}
+	if strings.TrimSpace(a.Version) == "" {
+		say("pins %s and says no version, so a person reading the pin cannot tell which release it is", a.URL)
+	}
+	if a.File != "" && a.Within != "" {
+		say("names a file (%q) and a directory within (%q): a file is kept whole, a directory is taken out "+
+			"of an archive", a.File, a.Within)
+	}
+	for field, p := range map[string]string{"file": a.File, "within": a.Within} {
+		if p == "" {
+			continue
+		}
+		clean := path.Clean(p)
+		if clean != p || strings.HasPrefix(p, "/") || clean == "." || clean == ".." || strings.HasPrefix(clean, "../") ||
+			(field == "file" && strings.Contains(p, "/")) {
+			say("names %s %q, which is not a plain relative %s", field, p, map[string]string{
+				"file": "file name", "within": "directory"}[field])
+		}
 	}
 	return problems
 }

internal/catalogue/installation.go

@@ -71,6 +71,9 @@ var hostname = regexp.MustCompile(
 		`(?:be|nl|de|fr|uk|eu|com|net|org|io|dev|app|cloud|site|online|me|co|ch|at|lu|` +
 		`internal|example|tld|test|invalid)`)
 
+// upstreamVersion is where an artifact pinned by URL says which release it is (Artifact.Version).
+var upstreamVersion = regexp.MustCompile(`^build\.artifacts\[[0-9]+\]\.version$`)
+
 // address is a dotted quad.
 var address = regexp.MustCompile(`(?:[0-9]{1,3}\.){3}[0-9]{1,3}`)
 
@@ -120,7 +123,10 @@ func InstallationProblems(m Manifest) []string {
 			problems = append(problems, fmt.Sprintf("%s names %s at %s", m.Module, name, at))
 		}
 		for _, ip := range addressesIn(value) {
-			if meant[ip] {
+			// An upstream pin's version may have four dotted numbers (Owl 1.5.6.1), which is no address; a name
+			// in it is still judged above. Its URL is judged whole: the vendor's host goes through the artifact's
+			// names-on-purpose like any name a definition means (novox/hq ADR 0316).
+			if meant[ip] || upstreamVersion.MatchString(at) {
 				continue
 			}
 			problems = append(problems, fmt.Sprintf("%s names the public address %s at %s", m.Module, ip, at))

internal/catalogue/manifest.go

@@ -976,6 +976,35 @@ type Artifact struct {
 	// success, and was invisible to the launcher (novox/hq 04-ISSUES/142).
 	Binary string `json:"binary,omitempty"`
 
+	// URL is where an archive's upstream file is fetched from, for an archive that is a release
+	// somebody else published rather than a directory of this repository (novox/hq ADR 0316 §2, §6).
+	// With it come SHA256 and Version, and From stays empty.
+	//
+	// **Pinned by the digest of what the vendor published, mirrored by the build seat.** A release of
+	// about 80 MB copied into a repository is kept for ever in its history at every upgrade, and an
+	// add-on whose licence forbids copying it to third parties may not be in a repository at all. So the
+	// manifest names the file, the build seat fetches it, refuses it when its sha256 is not the pin's and
+	// names both digests, and packs it as any archive is packed: what reaches a machine is the mesh's own
+	// archive in the artifact store, fetched by digest like every other, and a machine never asks the
+	// vendor. Upgrading is changing the pin, reviewed like any other change.
+	URL string `json:"url,omitempty"`
+	// SHA256 is the sha256 of the file at URL, as 64 lower-case hexadecimal characters.
+	SHA256 string `json:"sha256,omitempty"`
+	// Version is what the vendor calls the release pinned, said where a person reads the pin; the build
+	// does not read it.
+	Version string `json:"version,omitempty"`
+	// File is the name an upstream file that is not an archive is kept under: the archive built from it
+	// holds that one file (an add-on's xpi). Empty means the file is a tar archive (gzip, bzip2 or xz)
+	// whose files are the artifact.
+	File string `json:"file,omitempty"`
+	// Within is the directory inside an upstream tar archive whose content is the artifact, for a
+	// release that unpacks into a directory of its own name (`betterbird/`). An entry outside it refuses
+	// the build. Empty means the archive's root.
+	Within string `json:"within,omitempty"`
+	// NamesOnPurpose are the names this artifact means to name, each with why: the vendor's host an upstream
+	// file is fetched from, which the installation check judges like any name (InstallationProblems).
+	NamesOnPurpose map[string]string `json:"names-on-purpose,omitempty"`
+
 	// Entrypoints are the compiled files a tool host should load from this module, relative to the
 	// bundle's root.
 	//

internal/catalogue/upstream_file_test.go

@@ -0,0 +1,101 @@
+package catalogue
+
+import (
+	"strings"
+	"testing"
+)
+
+// An archive pinned by URL and sha256 (novox/hq ADR 0316 §2, §6): the vendor's file, mirrored by the build
+// seat, never a copy in the repository.
+
+const pinnedSum = "38dd24890a66b4cf9a45bc24e15192e00c26b929e1bb8df7974c16a4539129e2"
+
+func upstreamManifest(artifact string) string {
+	return `{"module":"a","version":"1","build":{"artifacts":[` + artifact + `]}}`
+}
+
+func TestAnArchivePinnedByURLAndSha256IsAccepted(t *testing.T) {
+	for _, a := range []string{
+		`{"name":"archive","kind":"archive","url":"https://vendor.example.org/release-1.tar.xz","sha256":"` + pinnedSum +
+			`","version":"1.0","within":"release"}`,
+		`{"name":"addon","kind":"archive","url":"https://vendor.example.org/addon.xpi","sha256":"` + pinnedSum +
+			`","version":"1.5.6.1","file":"addon.xpi"}`,
+	} {
+		if _, err := ParseManifest([]byte(upstreamManifest(a))); err != nil {
+			t.Errorf("%s was refused: %v", a, err)
+		}
+	}
+}
+
+func TestAnUpstreamFileThatIsNotPinnedIsRefused(t *testing.T) {
+	cases := map[string]string{
+		// No digest: what the address serves changes under the mesh.
+		`{"name":"x","kind":"archive","url":"https://vendor.invalid/r.tar.xz","version":"1"}`:                "without a pinned sha256",
+		`{"name":"x","kind":"archive","url":"https://vendor.invalid/r.tar.xz","sha256":"ABC","version":"1"}`: "without a pinned sha256",
+		// No version: a person reading the pin cannot tell which release.
+		`{"name":"x","kind":"archive","url":"https://vendor.invalid/r.tar.xz","sha256":"` + pinnedSum + `"}`: "says no version",
+		// Not https.
+		`{"name":"x","kind":"archive","url":"http://vendor.invalid/r.tar.xz","sha256":"` + pinnedSum + `","version":"1"}`: "over https",
+		// Both a directory of the repository and a URL.
+		`{"name":"x","kind":"archive","from":"files","url":"https://vendor.invalid/r.tar.xz","sha256":"` + pinnedSum +
+			`","version":"1"}`: "one or the other",
+		// A pin without a URL.
+		`{"name":"x","kind":"archive","from":"files","sha256":"` + pinnedSum + `"}`: "no url",
+		// Only an archive is mirrored from a file.
+		`{"name":"x","kind":"image","from":"Dockerfile","url":"https://vendor.invalid/r.tar.xz","sha256":"` + pinnedSum +
+			`","version":"1"}`: "only an archive",
+		// A file name that is a path, a directory within that leaves the archive, and both at once.
+		`{"name":"x","kind":"archive","url":"https://vendor.invalid/a.xpi","sha256":"` + pinnedSum +
+			`","version":"1","file":"../a.xpi"}`: "not a plain relative file name",
+		`{"name":"x","kind":"archive","url":"https://vendor.invalid/r.tar.xz","sha256":"` + pinnedSum +
+			`","version":"1","within":"../up"}`: "not a plain relative directory",
+		`{"name":"x","kind":"archive","url":"https://vendor.invalid/r.tar.xz","sha256":"` + pinnedSum +
+			`","version":"1","within":"r","file":"r.tar.xz"}`: "a file is kept whole",
+	}
+	for artifact, want := range cases {
+		_, err := ParseManifest([]byte(upstreamManifest(artifact)))
+		if err == nil {
+			t.Errorf("%s was accepted", artifact)
+			continue
+		}
+		if !strings.Contains(err.Error(), want) {
+			t.Errorf("%s was refused for another reason than %q: %v", artifact, want, err)
+		}
+	}
+}
+
+func TestAnUpstreamFilesVendorIsNamedOnPurposeAndAnInstallationsIsRefused(t *testing.T) {
+	// The vendor's host is judged like any name, and passes through names-on-purpose; a version of four dotted
+	// numbers is not an address, and a name in it is still judged.
+	m, err := ParseManifest([]byte(upstreamManifest(`{"name":"owl","kind":"archive","url":"https://vendor.be/owl/owl.xpi","sha256":"` +
+		pinnedSum + `","version":"1.5.6.1","file":"owl.xpi","names-on-purpose":{"vendor.be":"Owl's vendor publishes it there"}}`)))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if problems := InstallationProblems(m); len(problems) != 0 {
+		t.Fatalf("the declared vendor and the version were taken for an installation: %v", problems)
+	}
+	m.Build.Artifacts[0].NamesOnPurpose = nil
+	if problems := InstallationProblems(m); len(problems) != 1 || !strings.Contains(problems[0], "vendor.be") {
+		t.Fatalf("an undeclared vendor host is judged: %v", problems)
+	}
+	m.Build.Artifacts[0].Version = "git.novox.be 10.0.0.1"
+	if problems := InstallationProblems(m); len(problems) != 2 {
+		t.Fatalf("a name in the version is judged: %v", problems)
+	}
+	m.Build.Artifacts[0].URL, m.Build.Artifacts[0].Version = "https://mail.novox.be/owl.xpi", "1"
+	if problems := InstallationProblems(m); len(problems) != 1 || !strings.Contains(problems[0], "mail.novox.be") {
+		t.Fatalf("an installation's host in the url is judged: %v", problems)
+	}
+}
+
+func TestAnUpstreamFileFromAnAddressOrAReservedNameIsRefused(t *testing.T) {
+	for _, u := range []string{"https://git.novox.internal/r.tar.xz", "https://10.1.2.3/r.tar.xz",
+		"https://203.0.113.9/r.tar.xz", "https://[2001:db8::1]/r.tar.xz", "https://nas.lan/r.tar.xz", "https://vendor/r.tar.xz"} {
+		_, err := ParseManifest([]byte(upstreamManifest(`{"name":"x","kind":"archive","url":"` + u + `","sha256":"` +
+			pinnedSum + `","version":"1"}`)))
+		if err == nil || !strings.Contains(err.Error(), "published under a public name") {
+			t.Errorf("%s: %v", u, err)
+		}
+	}
+}