Let an archive be an upstream release pinned by URL and sha256, mirrored by the build seat (hq issue 345, ADR 0316)
A mail client's release is about 80 MB per version, too large to keep in a repository at every upgrade, and an add-on whose licence forbids copying it to third parties may not be in a repository at all. An archive artifact may now name url, sha256 and version (and file or within) instead of from: the build seat fetches it once, refuses it when its digest is not the pin's and names both, keeps it in its workspace under the digest (a vendor that serves only its newest version cannot break a later build of the same pin), and packs it like any archive, so a node gets the mesh's own archive by digest and never asks the vendor. The build seat's image carries xz, which Go's standard library does not read. The installation check takes the vendor's address and version of such a pin for the world's, as a registry is an image's.
This commit is contained in:
jochen
@@ -971,8 +971,19 @@ func one(ctx context.Context, run Runner, publish Publisher,
|
||||
return catalogue.Built{Name: a.Name, Kind: a.Kind, Reference: reference}, nil
|
||||
|
||||
case catalogue.ArtifactArchive:
|
||||
src.ownHas(cleanEntry(a.From) + "/**")
|
||||
body, err := pack(filepath.Join(tree, a.From))
|
||||
var body []byte
|
||||
var err error
|
||||
if a.URL != "" {
|
||||
// An upstream file pinned by URL and sha256 (novox/hq ADR 0316): its build source is the pin, which
|
||||
// is in module.json, always part of it.
|
||||
body, err = upstreamArchive(ctx, workspace, a, say)
|
||||
if err != nil {
|
||||
return catalogue.Built{}, fmt.Errorf("%s: %s: %w", module, a.Name, err)
|
||||
}
|
||||
} else {
|
||||
src.ownHas(cleanEntry(a.From) + "/**")
|
||||
body, err = pack(filepath.Join(tree, a.From))
|
||||
}
|
||||
if err != nil {
|
||||
return catalogue.Built{}, fmt.Errorf("%s: packing %s failed: %w", module, a.Name, err)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,295 @@
|
||||
package builder
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"bufio"
|
||||
"bytes"
|
||||
"compress/bzip2"
|
||||
"compress/gzip"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
)
|
||||
|
||||
// An archive whose source is somebody else's published file, pinned by URL and sha256 (novox/hq ADR 0316
|
||||
// §2, §6, to-be 56 §7): a mail client's release, an add-on's xpi.
|
||||
//
|
||||
// **Fetched by the build seat, checked against the pin before anything is read from it, and packed as any
|
||||
// archive is packed.** What a machine receives is the mesh's own archive from the artifact store, by digest,
|
||||
// exactly as for a directory of the repository; no machine ever asks the vendor. A file whose bytes are not
|
||||
// the pin's fails the build and names both digests, because the address serving something else is the one
|
||||
// thing the pin exists to catch.
|
||||
//
|
||||
// **Kept in the build seat's workspace under its sha256**, so it is fetched once: a vendor that serves only
|
||||
// its newest version at one address (Owl) would otherwise break every later build of the same pin.
|
||||
|
||||
// maxUpstream is the largest upstream file fetched, and the largest a release may unpack to: the node-engine
|
||||
// unpacks no archive larger than this either (mesh-host internal/apply/archive.go, maxArchive).
|
||||
const maxUpstream = 512 << 20
|
||||
|
||||
// upstreamClient fetches upstream files: a variable so a test serves its own.
|
||||
var upstreamClient = &http.Client{Timeout: 15 * time.Minute}
|
||||
|
||||
// upstreamDir is where the build seat keeps the upstream files it fetched, by sha256.
|
||||
func upstreamDir(workspace string) string { return filepath.Join(workspace, "upstream") }
|
||||
|
||||
// upstreamArchive is the packed archive of an artifact pinned by URL: the file fetched (or kept from an
|
||||
// earlier build) and checked, then laid out as the artifact says and packed.
|
||||
func upstreamArchive(ctx context.Context, workspace string, a catalogue.Artifact,
|
||||
say func(step, format string, args ...any)) ([]byte, error) {
|
||||
file, err := upstreamFile(ctx, workspace, a, say)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
laid := filepath.Join(workspace, "upstream-"+a.Name)
|
||||
if err := os.RemoveAll(laid); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer os.RemoveAll(laid)
|
||||
if err := os.MkdirAll(laid, 0o755); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if a.File != "" {
|
||||
body, err := os.ReadFile(file)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(laid, a.File), body, 0o644); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
} else if err := unpackUpstream(ctx, file, laid, a.Within); err != nil {
|
||||
return nil, fmt.Errorf("%s (%s): %w", a.URL, a.Version, err)
|
||||
}
|
||||
return pack(laid)
|
||||
}
|
||||
|
||||
// upstreamFile is the path of the pinned file in the workspace: kept from an earlier build when its bytes
|
||||
// are still the pin's, else fetched now and kept only when they are.
|
||||
func upstreamFile(ctx context.Context, workspace string, a catalogue.Artifact,
|
||||
say func(step, format string, args ...any)) (string, error) {
|
||||
want := strings.ToLower(a.SHA256)
|
||||
dir := upstreamDir(workspace)
|
||||
kept := filepath.Join(dir, want)
|
||||
if got, err := sumOf(kept); err == nil {
|
||||
if got == want {
|
||||
say("upstream", "%s %s: kept from an earlier build (sha256:%s)", a.Name, a.Version, short(want))
|
||||
return kept, nil
|
||||
}
|
||||
// Changed on disk since it was kept: fetched again rather than trusted.
|
||||
os.Remove(kept)
|
||||
}
|
||||
if err := os.MkdirAll(dir, 0o755); err != nil {
|
||||
return "", err
|
||||
}
|
||||
say("upstream", "fetching %s (%s)", a.URL, a.Version)
|
||||
request, err := http.NewRequestWithContext(ctx, http.MethodGet, a.URL, nil)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
response, err := upstreamClient.Do(request)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("cannot fetch %s: %w", a.URL, err)
|
||||
}
|
||||
defer response.Body.Close()
|
||||
if response.StatusCode != http.StatusOK {
|
||||
return "", fmt.Errorf("%s answered %s", a.URL, response.Status)
|
||||
}
|
||||
part, err := os.CreateTemp(dir, want+".*.part")
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer os.Remove(part.Name())
|
||||
hash := sha256.New()
|
||||
n, err := io.Copy(io.MultiWriter(part, hash), io.LimitReader(response.Body, maxUpstream+1))
|
||||
if cerr := part.Close(); err == nil {
|
||||
err = cerr
|
||||
}
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("fetching %s: %w", a.URL, err)
|
||||
}
|
||||
if n > maxUpstream {
|
||||
return "", fmt.Errorf("%s is larger than %d bytes, more than a machine unpacks; nothing was kept", a.URL, maxUpstream)
|
||||
}
|
||||
got := hex.EncodeToString(hash.Sum(nil))
|
||||
if got != want {
|
||||
return "", fmt.Errorf("%s was pinned as sha256:%s and what it served is sha256:%s (%d bytes); nothing "+
|
||||
"was kept. The address serves another file than the one reviewed: compare it, then move the pin "+
|
||||
"in a reviewed change (novox/hq ADR 0316)", a.URL, want, got, n)
|
||||
}
|
||||
if err := os.Rename(part.Name(), kept); err != nil {
|
||||
return "", err
|
||||
}
|
||||
say("upstream", "%s %s: %d bytes, sha256:%s as pinned", a.Name, a.Version, n, short(want))
|
||||
return kept, nil
|
||||
}
|
||||
|
||||
// sumOf is the sha256 of a file, in hexadecimal.
|
||||
func sumOf(file string) (string, error) {
|
||||
f, err := os.Open(file)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer f.Close()
|
||||
hash := sha256.New()
|
||||
if _, err := io.Copy(hash, f); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return hex.EncodeToString(hash.Sum(nil)), nil
|
||||
}
|
||||
|
||||
// unpackUpstream unpacks a tar archive, compressed with gzip, bzip2 or xz (told by its first bytes), into a
|
||||
// directory: of its entries only those within `within` (all, when empty), with that prefix taken off.
|
||||
//
|
||||
// **Only files and directories, and nothing outside.** A link, a device or an entry naming a path outside
|
||||
// the archive, or outside `within`, refuses the build by name: the node-engine unpacks only files and
|
||||
// directories, and an archive quietly unpacked without what it held would be a release that is not the one
|
||||
// pinned.
|
||||
func unpackUpstream(ctx context.Context, file, into, within string) error {
|
||||
f, err := os.Open(file)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer f.Close()
|
||||
buffered := bufio.NewReader(f)
|
||||
magic, _ := buffered.Peek(6)
|
||||
var stream io.Reader
|
||||
var wait func() error
|
||||
switch {
|
||||
case bytes.HasPrefix(magic, []byte{0x1f, 0x8b}):
|
||||
zipped, err := gzip.NewReader(buffered)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer zipped.Close()
|
||||
stream = zipped
|
||||
case bytes.HasPrefix(magic, []byte("BZh")):
|
||||
stream = bzip2.NewReader(buffered)
|
||||
case bytes.HasPrefix(magic, []byte{0xfd, '7', 'z', 'X', 'Z', 0x00}):
|
||||
// Go's standard library reads no xz; the build seat's image carries xz (cmd/mesh-builder/Dockerfile).
|
||||
if _, err := exec.LookPath("xz"); err != nil {
|
||||
return errors.New("it is compressed with xz, and this build seat has no xz to read it")
|
||||
}
|
||||
cmd := exec.CommandContext(ctx, "xz", "-dc")
|
||||
cmd.Stdin = buffered
|
||||
var stderr bytes.Buffer
|
||||
cmd.Stderr = &stderr
|
||||
out, err := cmd.StdoutPipe()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := cmd.Start(); err != nil {
|
||||
return err
|
||||
}
|
||||
stream = out
|
||||
wait = func() error {
|
||||
if err := cmd.Wait(); err != nil {
|
||||
return fmt.Errorf("xz could not read it: %v %s", err, strings.TrimSpace(stderr.String()))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
defer func() {
|
||||
if wait != nil {
|
||||
_ = cmd.Process.Kill()
|
||||
_ = cmd.Wait()
|
||||
}
|
||||
}()
|
||||
default:
|
||||
return errors.New("it is not a tar archive compressed with gzip, bzip2 or xz; a single file is kept " +
|
||||
"whole by naming it in `file`")
|
||||
}
|
||||
if err := untar(stream, into, within); err != nil {
|
||||
return err
|
||||
}
|
||||
if wait != nil {
|
||||
w := wait
|
||||
wait = nil
|
||||
return w()
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// untar writes a tar stream's files and directories within `within` into a directory.
|
||||
func untar(stream io.Reader, into, within string) error {
|
||||
reader := tar.NewReader(stream)
|
||||
prefix := ""
|
||||
if within != "" {
|
||||
prefix = path.Clean(within) + "/"
|
||||
}
|
||||
var total int64
|
||||
files := 0
|
||||
for {
|
||||
header, err := reader.Next()
|
||||
if err == io.EOF {
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
name := path.Clean(strings.TrimPrefix(header.Name, "./"))
|
||||
if path.IsAbs(name) || name == ".." || strings.HasPrefix(name, "../") {
|
||||
return fmt.Errorf("%s names a path outside the archive", header.Name)
|
||||
}
|
||||
if name == "." || name+"/" == prefix {
|
||||
continue
|
||||
}
|
||||
if prefix != "" {
|
||||
if !strings.HasPrefix(name, prefix) {
|
||||
return fmt.Errorf("%s is outside %s, the directory the artifact is taken from", header.Name, within)
|
||||
}
|
||||
name = strings.TrimPrefix(name, prefix)
|
||||
}
|
||||
target := filepath.Join(into, filepath.FromSlash(name))
|
||||
switch header.Typeflag {
|
||||
case tar.TypeDir:
|
||||
if err := os.MkdirAll(target, 0o755); err != nil {
|
||||
return err
|
||||
}
|
||||
case tar.TypeReg:
|
||||
total += header.Size
|
||||
if total > maxUpstream {
|
||||
return fmt.Errorf("it unpacks to more than %d bytes, more than a machine unpacks", maxUpstream)
|
||||
}
|
||||
if err := os.MkdirAll(filepath.Dir(target), 0o755); err != nil {
|
||||
return err
|
||||
}
|
||||
mode := os.FileMode(0o644)
|
||||
if header.Mode&0o111 != 0 {
|
||||
mode = 0o755
|
||||
}
|
||||
out, err := os.OpenFile(target, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, mode)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := io.Copy(out, io.LimitReader(reader, header.Size)); err != nil {
|
||||
out.Close()
|
||||
return err
|
||||
}
|
||||
if err := out.Close(); err != nil {
|
||||
return err
|
||||
}
|
||||
files++
|
||||
default:
|
||||
return fmt.Errorf("%s is a %c, and an archive carries only files and directories", header.Name,
|
||||
header.Typeflag)
|
||||
}
|
||||
}
|
||||
if files == 0 {
|
||||
if within != "" {
|
||||
return fmt.Errorf("it holds no file within %s", within)
|
||||
}
|
||||
return errors.New("it holds no file")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,299 @@
|
||||
package builder
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
)
|
||||
|
||||
// An archive pinned by URL and sha256 (novox/hq ADR 0316 §2, §6; How it is checked: "an upstream artifact
|
||||
// whose download does not match its sha256 fails the build and names both digests").
|
||||
|
||||
type tarEntry struct {
|
||||
name, body string
|
||||
mode int64
|
||||
kind byte
|
||||
}
|
||||
|
||||
func tarOf(t *testing.T, entries []tarEntry) []byte {
|
||||
t.Helper()
|
||||
var raw bytes.Buffer
|
||||
w := tar.NewWriter(&raw)
|
||||
for _, e := range entries {
|
||||
kind := e.kind
|
||||
if kind == 0 {
|
||||
kind = tar.TypeReg
|
||||
}
|
||||
h := &tar.Header{Name: e.name, Mode: e.mode, Typeflag: kind, Size: int64(len(e.body))}
|
||||
if kind == tar.TypeSymlink {
|
||||
h.Linkname, h.Size = "/usr/share/hunspell", 0
|
||||
}
|
||||
if kind == tar.TypeDir {
|
||||
h.Size = 0
|
||||
}
|
||||
if err := w.WriteHeader(h); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if kind == tar.TypeReg {
|
||||
if _, err := w.Write([]byte(e.body)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
if err := w.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return raw.Bytes()
|
||||
}
|
||||
|
||||
func gzipped(t *testing.T, body []byte) []byte {
|
||||
t.Helper()
|
||||
var out bytes.Buffer
|
||||
z := gzip.NewWriter(&out)
|
||||
if _, err := z.Write(body); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := z.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return out.Bytes()
|
||||
}
|
||||
|
||||
func sumHex(b []byte) string {
|
||||
s := sha256.Sum256(b)
|
||||
return hex.EncodeToString(s[:])
|
||||
}
|
||||
|
||||
// vendor serves files by path over https and counts what it was asked.
|
||||
type vendor struct {
|
||||
files map[string][]byte
|
||||
asked int
|
||||
}
|
||||
|
||||
func serveVendor(t *testing.T, v *vendor) string {
|
||||
t.Helper()
|
||||
server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
v.asked++
|
||||
body, ok := v.files[r.URL.Path]
|
||||
if !ok {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
w.Write(body)
|
||||
}))
|
||||
t.Cleanup(server.Close)
|
||||
before := upstreamClient
|
||||
upstreamClient = server.Client()
|
||||
t.Cleanup(func() { upstreamClient = before })
|
||||
return server.URL
|
||||
}
|
||||
|
||||
// entriesOf lists a packed archive: each file with its mode and content.
|
||||
func entriesOf(t *testing.T, packed []byte) map[string]string {
|
||||
t.Helper()
|
||||
z, err := gzip.NewReader(bytes.NewReader(packed))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
r := tar.NewReader(z)
|
||||
out := map[string]string{}
|
||||
for {
|
||||
h, err := r.Next()
|
||||
if err == io.EOF {
|
||||
return out
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
body, _ := io.ReadAll(r)
|
||||
out[h.Name] = string(body) + " " + os.FileMode(h.Mode).String()
|
||||
}
|
||||
}
|
||||
|
||||
func release(t *testing.T) []byte {
|
||||
return gzipped(t, tarOf(t, []tarEntry{
|
||||
{name: "betterbird/", kind: tar.TypeDir, mode: 0o755},
|
||||
{name: "betterbird/application.ini", body: "[App]\nVersion=153.4.0\n", mode: 0o644},
|
||||
{name: "betterbird/betterbird", body: "#!/bin/sh\n", mode: 0o755},
|
||||
{name: "betterbird/defaults/pref/channel-prefs.js", body: "pref(\"app.update.channel\", \"default\");\n", mode: 0o644},
|
||||
}))
|
||||
}
|
||||
|
||||
func pinnedManifest(base, sum string) string {
|
||||
return `{"module":"betterbird","version":"1",
|
||||
"build":{"artifacts":[
|
||||
{"name":"archive","kind":"archive","url":"` + base + `/release.tar.gz","sha256":"` + sum + `","version":"153.4.0esr-bb10","within":"betterbird"}]},
|
||||
"resources":[{"id":"installation","type":"archive","path":"/opt/betterbird","artifact":"archive"}]}`
|
||||
}
|
||||
|
||||
func TestAnArchivePinnedByURLIsFetchedCheckedAndPackedWithoutItsOwnDirectory(t *testing.T) {
|
||||
body := release(t)
|
||||
v := &vendor{files: map[string][]byte{"/release.tar.gz": body}}
|
||||
base := serveVendor(t, v)
|
||||
r, workspace := aRepository(t, pinnedManifest(base, sumHex(body)), nil)
|
||||
got, err := Build(context.Background(), r.run, r, "https://forge.invalid/catalogue.git", "", "", workspace, nil,
|
||||
Npmrc{}, GitCredential{}, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
digest, _ := got.Manifest.Resources[0]["digest"].(string)
|
||||
if !strings.HasPrefix(digest, "sha256:") || r.archives["betterbird/archive"] != digest {
|
||||
t.Fatalf("the release was not published and pinned as the module's archive: %v, published %v",
|
||||
got.Manifest.Resources[0], r.archives)
|
||||
}
|
||||
|
||||
packed, err := upstreamArchive(context.Background(), workspace, catalogue.Artifact{Name: "archive", Kind: "archive",
|
||||
URL: base + "/release.tar.gz", SHA256: sumHex(body), Version: "153.4.0esr-bb10", Within: "betterbird"},
|
||||
func(string, string, ...any) {})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if "sha256:"+sumHex(packed) != digest {
|
||||
t.Fatalf("a second packing of one pin gave another digest: %s and %s", digest, sumHex(packed))
|
||||
}
|
||||
entries := entriesOf(t, packed)
|
||||
want := map[string]string{
|
||||
"application.ini": "[App]\nVersion=153.4.0\n -rw-r--r--",
|
||||
"betterbird": "#!/bin/sh\n -rwxr-xr-x",
|
||||
"defaults/pref/channel-prefs.js": "pref(\"app.update.channel\", \"default\");\n -rw-r--r--",
|
||||
}
|
||||
if len(entries) != len(want) {
|
||||
t.Fatalf("the archive holds %v, want %v", entries, want)
|
||||
}
|
||||
for name, w := range want {
|
||||
if entries[name] != w {
|
||||
t.Errorf("%s is %q in the archive, want %q", name, entries[name], w)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnUpstreamFileIsFetchedOnceAndKeptByItsDigest(t *testing.T) {
|
||||
// The vendor of an add-on serves only its newest version at one address: a pin that was fetched once
|
||||
// must still build after the address moved on.
|
||||
body := release(t)
|
||||
v := &vendor{files: map[string][]byte{"/release.tar.gz": body}}
|
||||
base := serveVendor(t, v)
|
||||
r, workspace := aRepository(t, pinnedManifest(base, sumHex(body)), nil)
|
||||
first, err := Build(context.Background(), r.run, r, "https://forge.invalid/catalogue.git", "", "", workspace, nil,
|
||||
Npmrc{}, GitCredential{}, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
v.files["/release.tar.gz"] = []byte("the vendor's next release")
|
||||
second, err := Build(context.Background(), r.run, r, "https://forge.invalid/catalogue.git", "", "", workspace, nil,
|
||||
Npmrc{}, GitCredential{}, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("a pin fetched before failed to build once the address served something else: %v", err)
|
||||
}
|
||||
if v.asked != 1 {
|
||||
t.Fatalf("the vendor was asked %d times for one pin, want once", v.asked)
|
||||
}
|
||||
if first.Manifest.Resources[0]["digest"] != second.Manifest.Resources[0]["digest"] {
|
||||
t.Fatal("two builds of one pin gave two digests")
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnUpstreamFileWhoseDownloadDoesNotMatchItsPinFailsAndNamesBothDigests(t *testing.T) {
|
||||
pinned := release(t)
|
||||
served := gzipped(t, tarOf(t, []tarEntry{{name: "betterbird/betterbird", body: "not what was reviewed", mode: 0o755}}))
|
||||
v := &vendor{files: map[string][]byte{"/release.tar.gz": served}}
|
||||
base := serveVendor(t, v)
|
||||
r, workspace := aRepository(t, pinnedManifest(base, sumHex(pinned)), nil)
|
||||
_, err := Build(context.Background(), r.run, r, "https://forge.invalid/catalogue.git", "", "", workspace, nil,
|
||||
Npmrc{}, GitCredential{}, nil)
|
||||
if err == nil {
|
||||
t.Fatal("a download that is not the pinned file was built")
|
||||
}
|
||||
for _, digest := range []string{sumHex(pinned), sumHex(served)} {
|
||||
if !strings.Contains(err.Error(), "sha256:"+digest) {
|
||||
t.Errorf("the refusal does not name sha256:%s: %v", digest, err)
|
||||
}
|
||||
}
|
||||
if len(r.archives) != 0 {
|
||||
t.Fatalf("something was published: %v", r.archives)
|
||||
}
|
||||
kept, _ := filepath.Glob(filepath.Join(upstreamDir(workspace), "*"))
|
||||
if len(kept) != 0 {
|
||||
t.Fatalf("the refused download was kept: %v", kept)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnUpstreamFileThatIsNotAnArchiveIsKeptWholeUnderItsName(t *testing.T) {
|
||||
xpi := []byte("PK\x03\x04 an add-on")
|
||||
v := &vendor{files: map[string][]byte{"/owl.xpi": xpi}}
|
||||
base := serveVendor(t, v)
|
||||
packed, err := upstreamArchive(context.Background(), t.TempDir(), catalogue.Artifact{Name: "owl", Kind: "archive",
|
||||
URL: base + "/owl.xpi", SHA256: sumHex(xpi), Version: "1.5.6.1", File: "owl.xpi"}, func(string, string, ...any) {})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
entries := entriesOf(t, packed)
|
||||
if len(entries) != 1 || entries["owl.xpi"] != string(xpi)+" -rw-r--r--" {
|
||||
t.Fatalf("the add-on was not kept whole as owl.xpi: %v", entries)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnUpstreamArchiveHoldingALinkOrAnEntryOutsideItsDirectoryIsRefused(t *testing.T) {
|
||||
cases := map[string][]tarEntry{
|
||||
// The community package links the dictionaries; a release carrying a link is not one a machine unpacks.
|
||||
"is a 2": {{name: "betterbird/betterbird", body: "x", mode: 0o755},
|
||||
{name: "betterbird/dictionaries", kind: tar.TypeSymlink}},
|
||||
"outside betterbird": {{name: "betterbird/betterbird", body: "x", mode: 0o755},
|
||||
{name: "README", body: "x", mode: 0o644}},
|
||||
"outside the archive": {{name: "../etc/passwd", body: "x", mode: 0o644}},
|
||||
"no file within": {{name: "betterbird/", kind: tar.TypeDir, mode: 0o755}},
|
||||
}
|
||||
for want, entries := range cases {
|
||||
body := gzipped(t, tarOf(t, entries))
|
||||
v := &vendor{files: map[string][]byte{"/r.tar.gz": body}}
|
||||
base := serveVendor(t, v)
|
||||
_, err := upstreamArchive(context.Background(), t.TempDir(), catalogue.Artifact{Name: "archive", Kind: "archive",
|
||||
URL: base + "/r.tar.gz", SHA256: sumHex(body), Version: "1", Within: "betterbird"}, func(string, string, ...any) {})
|
||||
if err == nil || !strings.Contains(err.Error(), want) {
|
||||
t.Errorf("an archive whose entry is %q was not refused for it: %v", want, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnUpstreamArchiveCompressedWithXzIsRead(t *testing.T) {
|
||||
// Betterbird publishes its Linux release as .tar.xz; the build seat's image carries xz for it.
|
||||
if _, err := exec.LookPath("xz"); err != nil {
|
||||
t.Skip("NOT TESTED here: no xz on this machine (the build seat's image carries it)")
|
||||
}
|
||||
plain := tarOf(t, []tarEntry{{name: "betterbird/betterbird", body: "#!/bin/sh\n", mode: 0o755}})
|
||||
cmd := exec.Command("xz", "-zc")
|
||||
cmd.Stdin = bytes.NewReader(plain)
|
||||
compressed, err := cmd.Output()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
v := &vendor{files: map[string][]byte{"/r.tar.xz": compressed}}
|
||||
base := serveVendor(t, v)
|
||||
packed, err := upstreamArchive(context.Background(), t.TempDir(), catalogue.Artifact{Name: "archive", Kind: "archive",
|
||||
URL: base + "/r.tar.xz", SHA256: sumHex(compressed), Version: "1", Within: "betterbird"}, func(string, string, ...any) {})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var names []string
|
||||
for n := range entriesOf(t, packed) {
|
||||
names = append(names, n)
|
||||
}
|
||||
sort.Strings(names)
|
||||
if strings.Join(names, ",") != "betterbird" {
|
||||
t.Fatalf("the xz release unpacked to %v", names)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user