Judge an upstream pin's host like any name, and harden the upstream unpack (review 35, hq issue 345)

The url was skipped by the installation check, so an installation's host or
an address passed; it is judged whole now, the vendor's host declared in the
artifact's names-on-purpose, and an address or a reserved name as the host
is refused. Only the version's dotted-quad reading is skipped. The unpack
caps its entries, drains xz before waiting for it, reads a pax global
header and a nested within's parents, and a redirect off https is not
followed.

internal/catalogue/build.go

@@ -2,6 +2,7 @@ package catalogue
 
 import (
 	"fmt"
+	"net"
 	"net/url"
 	"path"
 	"regexp"
@@ -357,6 +358,27 @@ func (b *Build) problems(module string) []string {
 	return problems
 }
 
+// reservedSuffixes are names no public vendor publishes under: an installation's own and documentation's.
+var reservedSuffixes = []string{".internal", ".local", ".lan", ".home", ".corp", ".localhost", ".home.arpa",
+	".test", ".invalid", ".example", ".intranet", ".private"}
+
+// notAPublicHost says why a host is not a public vendor's name, or "".
+func notAPublicHost(host string) string {
+	h := strings.ToLower(strings.TrimSuffix(host, "."))
+	if net.ParseIP(h) != nil {
+		return "an address rather than a name"
+	}
+	if !strings.Contains(h, ".") {
+		return "a name with no domain"
+	}
+	for _, s := range reservedSuffixes {
+		if strings.HasSuffix(h, s) {
+			return "a name under " + s + ", which is nobody's public domain"
+		}
+	}
+	return ""
+}
+
 // sha256Hex is a pinned sha256 as a manifest writes it: 64 lower-case hexadecimal characters.
 var sha256Hex = regexp.MustCompile(`^[0-9a-f]{64}$`)
 
@@ -386,6 +408,8 @@ func upstreamFileProblems(module string, a Artifact) []string {
 	}
 	if u, err := url.Parse(a.URL); err != nil || u.Scheme != "https" || u.Host == "" {
 		say("is fetched from %q, and an upstream file is fetched over https from a named host", a.URL)
+	} else if why := notAPublicHost(u.Hostname()); why != "" {
+		say("is fetched from %s, %s: an upstream file is somebody else's release, published under a public name", a.URL, why)
 	}
 	if !sha256Hex.MatchString(a.SHA256) {
 		say("is fetched from %s without a pinned sha256 (64 lower-case hexadecimal characters): what that "+

internal/catalogue/installation.go

@@ -71,9 +71,8 @@ var hostname = regexp.MustCompile(
 		`(?:be|nl|de|fr|uk|eu|com|net|org|io|dev|app|cloud|site|online|me|co|ch|at|lu|` +
 		`internal|example|tld|test|invalid)`)
 
-// upstreamPin is where an artifact pinned by URL says where it is published and which release it is: the
-// vendor's address and the vendor's version, which name no installation (Artifact.URL).
-var upstreamPin = regexp.MustCompile(`^build\.artifacts\[[0-9]+\]\.(url|version)$`)
+// upstreamVersion is where an artifact pinned by URL says which release it is (Artifact.Version).
+var upstreamVersion = regexp.MustCompile(`^build\.artifacts\[[0-9]+\]\.version$`)
 
 // address is a dotted quad.
 var address = regexp.MustCompile(`(?:[0-9]{1,3}\.){3}[0-9]{1,3}`)
@@ -117,11 +116,6 @@ func InstallationProblems(m Manifest) []string {
 			"%s is named after %s, and a module is named for what it is, not for where it runs", m.Module, name))
 	}
 	walk(tree, "", nil, func(at string, value string, meant map[string]bool, isImage bool) {
-		// An archive pinned by URL names the vendor that publishes it, and the version it pins, which may
-		// have four dotted numbers (novox/hq ADR 0316): the world's, as a public registry is an image's.
-		if upstreamPin.MatchString(at) {
-			return
-		}
 		for _, name := range namesIn(value) {
 			if (isImage && worldsRegistries[strings.ToLower(name)]) || meant[name] {
 				continue
@@ -129,7 +123,10 @@ func InstallationProblems(m Manifest) []string {
 			problems = append(problems, fmt.Sprintf("%s names %s at %s", m.Module, name, at))
 		}
 		for _, ip := range addressesIn(value) {
-			if meant[ip] {
+			// An upstream pin's version may have four dotted numbers (Owl 1.5.6.1), which is no address; a name
+			// in it is still judged above. Its URL is judged whole: the vendor's host goes through the artifact's
+			// names-on-purpose like any name a definition means (novox/hq ADR 0316).
+			if meant[ip] || upstreamVersion.MatchString(at) {
 				continue
 			}
 			problems = append(problems, fmt.Sprintf("%s names the public address %s at %s", m.Module, ip, at))

internal/catalogue/manifest.go

@@ -1001,6 +1001,9 @@ type Artifact struct {
 	// release that unpacks into a directory of its own name (`betterbird/`). An entry outside it refuses
 	// the build. Empty means the archive's root.
 	Within string `json:"within,omitempty"`
+	// NamesOnPurpose are the names this artifact means to name, each with why: the vendor's host an upstream
+	// file is fetched from, which the installation check judges like any name (InstallationProblems).
+	NamesOnPurpose map[string]string `json:"names-on-purpose,omitempty"`
 
 	// Entrypoints are the compiled files a tool host should load from this module, relative to the
 	// bundle's root.

internal/catalogue/upstream_file_test.go

@@ -16,9 +16,9 @@ func upstreamManifest(artifact string) string {
 
 func TestAnArchivePinnedByURLAndSha256IsAccepted(t *testing.T) {
 	for _, a := range []string{
-		`{"name":"archive","kind":"archive","url":"https://vendor.invalid/release-1.tar.xz","sha256":"` + pinnedSum +
+		`{"name":"archive","kind":"archive","url":"https://vendor.example.org/release-1.tar.xz","sha256":"` + pinnedSum +
 			`","version":"1.0","within":"release"}`,
-		`{"name":"addon","kind":"archive","url":"https://vendor.invalid/addon.xpi","sha256":"` + pinnedSum +
+		`{"name":"addon","kind":"archive","url":"https://vendor.example.org/addon.xpi","sha256":"` + pinnedSum +
 			`","version":"1.5.6.1","file":"addon.xpi"}`,
 	} {
 		if _, err := ParseManifest([]byte(upstreamManifest(a))); err != nil {
@@ -64,19 +64,38 @@ func TestAnUpstreamFileThatIsNotPinnedIsRefused(t *testing.T) {
 	}
 }
 
-func TestAnUpstreamFilesVendorAndVersionNameNoInstallation(t *testing.T) {
-	// The vendor's address is the world's, as a public registry is an image's; a version of four dotted numbers
-	// is not an address. Anything else still is judged.
+func TestAnUpstreamFilesVendorIsNamedOnPurposeAndAnInstallationsIsRefused(t *testing.T) {
+	// The vendor's host is judged like any name, and passes through names-on-purpose; a version of four dotted
+	// numbers is not an address, and a name in it is still judged.
 	m, err := ParseManifest([]byte(upstreamManifest(`{"name":"owl","kind":"archive","url":"https://vendor.be/owl/owl.xpi","sha256":"` +
-		pinnedSum + `","version":"1.5.6.1","file":"owl.xpi"}`)))
+		pinnedSum + `","version":"1.5.6.1","file":"owl.xpi","names-on-purpose":{"vendor.be":"Owl's vendor publishes it there"}}`)))
 	if err != nil {
 		t.Fatal(err)
 	}
 	if problems := InstallationProblems(m); len(problems) != 0 {
-		t.Fatalf("the vendor and the version were taken for an installation: %v", problems)
+		t.Fatalf("the declared vendor and the version were taken for an installation: %v", problems)
 	}
-	m.Build.Artifacts[0].File = "mail.novox.be"
+	m.Build.Artifacts[0].NamesOnPurpose = nil
+	if problems := InstallationProblems(m); len(problems) != 1 || !strings.Contains(problems[0], "vendor.be") {
+		t.Fatalf("an undeclared vendor host is judged: %v", problems)
+	}
+	m.Build.Artifacts[0].Version = "git.novox.be 10.0.0.1"
+	if problems := InstallationProblems(m); len(problems) != 2 {
+		t.Fatalf("a name in the version is judged: %v", problems)
+	}
+	m.Build.Artifacts[0].URL, m.Build.Artifacts[0].Version = "https://mail.novox.be/owl.xpi", "1"
 	if problems := InstallationProblems(m); len(problems) != 1 || !strings.Contains(problems[0], "mail.novox.be") {
-		t.Fatalf("a name elsewhere in the artifact is still judged: %v", problems)
+		t.Fatalf("an installation's host in the url is judged: %v", problems)
+	}
+}
+
+func TestAnUpstreamFileFromAnAddressOrAReservedNameIsRefused(t *testing.T) {
+	for _, u := range []string{"https://git.novox.internal/r.tar.xz", "https://10.1.2.3/r.tar.xz",
+		"https://203.0.113.9/r.tar.xz", "https://[2001:db8::1]/r.tar.xz", "https://nas.lan/r.tar.xz", "https://vendor/r.tar.xz"} {
+		_, err := ParseManifest([]byte(upstreamManifest(`{"name":"x","kind":"archive","url":"` + u + `","sha256":"` +
+			pinnedSum + `","version":"1"}`)))
+		if err == nil || !strings.Contains(err.Error(), "published under a public name") {
+			t.Errorf("%s: %v", u, err)
+		}
 	}
 }