mesh-build-go: read a pinned release compressed with xz as the build agent does, so betterbird's archive gets a done key (issue 1590)
The xz reader is github.com/therootcompany/xz, declared in go.mod as build-agent declares it, with build-agent's framing walk copied (xz_framing.go and its tests): too many blocks or streams are refused before anything is decoded, a dictionary over 64 MiB is refused. The same release tar under gzip and under xz packs to the build agent's archive.
This commit is contained in:
@@ -107,5 +107,5 @@ release. An artifact that is neither a Go bundle nor an archive fails the step n
|
||||
The same tool packs an archive artifact of any module's tree, whatever its language (ADR 0478 §1, issue 1589): the
|
||||
directory named by `from`, or a file pinned by `url` and `sha256`, packed with the same `Pack` as the build agent's `pack`
|
||||
(`testdata/captured/README` says how its digest was captured). A pinned file is fetched, checked against its sha256 (a
|
||||
mismatch fails the step naming both digests) and laid out as the artifact says (`file`, or unpacked from `within`; gzip and
|
||||
bzip2, not yet xz); it runs no command and needs no Go.
|
||||
mismatch fails the step naming both digests) and laid out as the artifact says (`file`, or unpacked from `within`; gzip, bzip2
|
||||
and xz, read in Go as the build agent reads it, issue 1590); it runs no command and needs no Go.
|
||||
|
||||
@@ -24,6 +24,7 @@ import (
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"github.com/therootcompany/xz"
|
||||
)
|
||||
|
||||
// maxUpstream is the largest pinned file fetched and the largest it may unpack to, as the build agent's and the
|
||||
@@ -190,10 +191,13 @@ func sumOf(file string) (string, error) {
|
||||
return hex.EncodeToString(hash.Sum(nil)), nil
|
||||
}
|
||||
|
||||
// unpackUpstream unpacks a tar archive compressed with gzip or bzip2 into a directory: of its entries only those within
|
||||
// `within` (all, when empty), with that prefix taken off. Only files and directories, and nothing outside: the
|
||||
// node-engine unpacks nothing else. An xz archive is refused by name; the build agent reads it, this module does not
|
||||
// yet (it is a later issue, not a quiet difference in the archive).
|
||||
// xzDictMax is the largest xz dictionary read (the build agent's): xz's own presets use at most 64 MiB (-9); a stream
|
||||
// asking for more is refused rather than given the memory.
|
||||
const xzDictMax = 64 << 20
|
||||
|
||||
// unpackUpstream unpacks a tar archive compressed with gzip, bzip2 or xz into a directory: of its entries only those
|
||||
// within `within` (all, when empty), with that prefix taken off. Only files and directories, and nothing outside: the
|
||||
// node-engine unpacks nothing else.
|
||||
func unpackUpstream(file, into, within string) error {
|
||||
f, err := os.Open(file)
|
||||
if err != nil {
|
||||
@@ -214,9 +218,22 @@ func unpackUpstream(file, into, within string) error {
|
||||
case bytes.HasPrefix(magic, []byte("BZh")):
|
||||
stream = bzip2.NewReader(buffered)
|
||||
case bytes.HasPrefix(magic, []byte{0xfd, '7', 'z', 'X', 'Z', 0x00}):
|
||||
return errors.New("it is an xz archive, which this build module does not read yet; the build agent does")
|
||||
// As the build agent reads it (mesh-controller internal/builder/upstream_file.go; issue 1590): in Go
|
||||
// (github.com/therootcompany/xz, a port of xz-embedded), never by a tool the node may not carry. xzDictMax is
|
||||
// a real maximum: a block whose LZMA2 dictionary is larger is refused before its dictionary is allocated. The
|
||||
// reader keeps a filter decoder for every block of a stream, so the file's framing is walked first and a file
|
||||
// of more blocks than xzBlocksMax is refused before anything is decoded (xz_framing.go). What the stream
|
||||
// unpacks to is bounded by untar (maxUpstream, maxEntries).
|
||||
if err := xzWithinLimits(file); err != nil {
|
||||
return err
|
||||
}
|
||||
zipped, err := xz.NewReader(buffered, xzDictMax)
|
||||
if err != nil {
|
||||
return fmt.Errorf("it is not an xz stream this build module reads (a dictionary of %d MiB at most): %v", xzDictMax>>20, err)
|
||||
}
|
||||
stream = zipped
|
||||
default:
|
||||
return errors.New("it is not a tar archive compressed with gzip or bzip2; a single file is kept whole by naming it in `file`")
|
||||
return errors.New("it is not a tar archive compressed with gzip, bzip2 or xz; a single file is kept whole by naming it in `file`")
|
||||
}
|
||||
return untar(stream, into, within)
|
||||
}
|
||||
|
||||
Binary file not shown.
@@ -0,0 +1,293 @@
|
||||
package main
|
||||
|
||||
// Copied from mesh-controller origin/main internal/builder/xz_framing.go (issue 1590): the build agent's walk of an xz
|
||||
// file's framing before its reader decodes it. "The build seat" below is the build module now.
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"encoding/binary"
|
||||
"errors"
|
||||
"fmt"
|
||||
"hash/crc32"
|
||||
"io"
|
||||
"os"
|
||||
)
|
||||
|
||||
// xzBlocksMax and xzStreamsMax are the most blocks and the most concatenated streams an xz file may hold for
|
||||
// the build seat to read it.
|
||||
//
|
||||
// **Why a block count is limited at all.** The xz reader (github.com/therootcompany/xz v1.0.1) keeps one
|
||||
// filter decoder for every block that names a delta or BCJ filter, and frees them only when a stream ends:
|
||||
// its block counters are reset per stream, not per block. A stream of many tiny blocks therefore grows the
|
||||
// heap with its length: an `xz --block-size=1 --delta` file of 6 MB kept about 67 MiB live, which comes to
|
||||
// about 5.6 GiB at the 512 MiB download cap (review 68 on mesh-controller pull request 264, hq issue 345).
|
||||
//
|
||||
// **Why 1,024.** At about 340 bytes of decoder a block, 1,024 blocks keep at most about 0.35 MiB live. A
|
||||
// real release is far below it: Betterbird's archive is 1 stream with 1 block, and multi-threaded xz cuts
|
||||
// blocks of three times the dictionary (24 MiB at the default preset), so even 512 MiB unpacked in blocks
|
||||
// of 512 KiB stays within it. Streams are limited to 16 because a release is one stream; the decoders are
|
||||
// reused from one stream to the next, so the stream limit only bounds the work of this walk.
|
||||
const (
|
||||
xzBlocksMax = 1024
|
||||
xzStreamsMax = 16
|
||||
)
|
||||
|
||||
// xzWithinLimits walks an xz file's framing, without decompressing anything, and refuses one holding more
|
||||
// blocks or streams than the build seat reads.
|
||||
//
|
||||
// **The framing, not the index.** Each stream's index at its end lists its blocks, but the reader compares
|
||||
// the index with the blocks only after it has decoded them all, so a file whose index lies would still be
|
||||
// decoded block by block. This walk follows the same framing the reader follows: stream header, each block
|
||||
// header, the block's LZMA2 chunks (each chunk header carries its own compressed size, so a chunk is skipped
|
||||
// without being decoded), its padding and check, then the index and the footer. It counts the blocks the
|
||||
// reader would decode, and it also refuses an index whose count of blocks is not the count walked.
|
||||
func xzWithinLimits(file string) error {
|
||||
f, err := os.Open(file)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer f.Close()
|
||||
_, _, err = xzFraming(bufio.NewReader(f))
|
||||
if errors.Is(err, errXzTooManyBlocks) || errors.Is(err, errXzTooManyStreams) {
|
||||
return err
|
||||
}
|
||||
if err != nil {
|
||||
return fmt.Errorf("it is not an xz file this build seat reads: %v", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
var (
|
||||
errXzTooManyBlocks = fmt.Errorf("it holds more than %d xz blocks, the most this build seat reads: "+
|
||||
"its xz reader keeps a filter decoder for every block (hq issue 345)", xzBlocksMax)
|
||||
errXzTooManyStreams = fmt.Errorf("it holds more than %d concatenated xz streams, the most this build "+
|
||||
"seat reads", xzStreamsMax)
|
||||
)
|
||||
|
||||
// xzFraming counts the streams and blocks of an xz file, checking its framing as it goes. It stops with
|
||||
// errXzTooManyBlocks or errXzTooManyStreams at the first block or stream over the limit.
|
||||
func xzFraming(r *bufio.Reader) (streams, blocks int, err error) {
|
||||
full := func(n int) ([]byte, error) {
|
||||
b := make([]byte, n)
|
||||
if _, err := io.ReadFull(r, b); err != nil {
|
||||
if err == io.EOF {
|
||||
err = io.ErrUnexpectedEOF
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
return b, nil
|
||||
}
|
||||
skip := func(n int) error {
|
||||
if got, err := r.Discard(n); err != nil || got != n {
|
||||
return io.ErrUnexpectedEOF
|
||||
}
|
||||
return nil
|
||||
}
|
||||
for {
|
||||
header, err := full(12)
|
||||
if err != nil {
|
||||
return streams, blocks, fmt.Errorf("stream %d: header: %v", streams+1, err)
|
||||
}
|
||||
if !bytes.Equal(header[:6], []byte{0xfd, '7', 'z', 'X', 'Z', 0x00}) {
|
||||
return streams, blocks, fmt.Errorf("stream %d: no xz stream header", streams+1)
|
||||
}
|
||||
if header[6] != 0 || header[7]&0xf0 != 0 ||
|
||||
crc32.ChecksumIEEE(header[6:8]) != binary.LittleEndian.Uint32(header[8:12]) {
|
||||
return streams, blocks, fmt.Errorf("stream %d: a corrupt stream header", streams+1)
|
||||
}
|
||||
if streams++; streams > xzStreamsMax {
|
||||
return streams, blocks, errXzTooManyStreams
|
||||
}
|
||||
checkSize := 0
|
||||
if check := int(header[7]); check != 0 {
|
||||
checkSize = 4 << ((check - 1) / 3)
|
||||
}
|
||||
inStream := 0
|
||||
for {
|
||||
first, err := r.ReadByte()
|
||||
if err != nil {
|
||||
return streams, blocks, fmt.Errorf("stream %d: no index: %v", streams, io.ErrUnexpectedEOF)
|
||||
}
|
||||
if first == 0 {
|
||||
break // the index indicator
|
||||
}
|
||||
rest, err := full(int(first)*4 + 3)
|
||||
if err != nil {
|
||||
return streams, blocks, fmt.Errorf("stream %d, block %d: header: %v", streams, inStream+1, err)
|
||||
}
|
||||
if err := xzBlockHeader(append([]byte{first}, rest...)); err != nil {
|
||||
return streams, blocks, fmt.Errorf("stream %d, block %d: %v", streams, inStream+1, err)
|
||||
}
|
||||
inStream++
|
||||
if blocks++; blocks > xzBlocksMax {
|
||||
return streams, blocks, errXzTooManyBlocks
|
||||
}
|
||||
// The block's LZMA2 chunks, up to the end marker.
|
||||
size := 0
|
||||
for {
|
||||
control, err := r.ReadByte()
|
||||
if err != nil {
|
||||
return streams, blocks, fmt.Errorf("stream %d, block %d: %v", streams, inStream, io.ErrUnexpectedEOF)
|
||||
}
|
||||
size++
|
||||
if control == 0 {
|
||||
break
|
||||
}
|
||||
switch {
|
||||
case control == 1 || control == 2: // stored: a 16-bit size less one, then the bytes
|
||||
b, err := full(2)
|
||||
if err != nil {
|
||||
return streams, blocks, err
|
||||
}
|
||||
n := int(binary.BigEndian.Uint16(b)) + 1
|
||||
if err := skip(n); err != nil {
|
||||
return streams, blocks, err
|
||||
}
|
||||
size += 2 + n
|
||||
case control >= 0x80: // LZMA: unpacked and packed sizes, the properties byte on a reset
|
||||
b, err := full(4)
|
||||
if err != nil {
|
||||
return streams, blocks, err
|
||||
}
|
||||
n := int(binary.BigEndian.Uint16(b[2:])) + 1
|
||||
size += 4
|
||||
if control >= 0xc0 {
|
||||
if err := skip(1); err != nil {
|
||||
return streams, blocks, err
|
||||
}
|
||||
size++
|
||||
}
|
||||
if err := skip(n); err != nil {
|
||||
return streams, blocks, err
|
||||
}
|
||||
size += n
|
||||
default:
|
||||
return streams, blocks, fmt.Errorf("stream %d, block %d: an LZMA2 chunk of kind %#x",
|
||||
streams, inStream, control)
|
||||
}
|
||||
}
|
||||
if pad := (4 - size%4) % 4; pad > 0 {
|
||||
b, err := full(pad)
|
||||
if err != nil || !bytes.Equal(b, make([]byte, pad)) {
|
||||
return streams, blocks, fmt.Errorf("stream %d, block %d: corrupt block padding", streams, inStream)
|
||||
}
|
||||
}
|
||||
if err := skip(checkSize); err != nil {
|
||||
return streams, blocks, err
|
||||
}
|
||||
}
|
||||
// The index: its count of records, two numbers each, padding, CRC32.
|
||||
used := 1
|
||||
count, n, err := xzVLI(r)
|
||||
if err != nil {
|
||||
return streams, blocks, fmt.Errorf("stream %d: index: %v", streams, err)
|
||||
}
|
||||
used += n
|
||||
if count != uint64(inStream) {
|
||||
return streams, blocks, fmt.Errorf("stream %d: its index lists %d blocks, but it holds %d",
|
||||
streams, count, inStream)
|
||||
}
|
||||
for i := 0; i < 2*inStream; i++ {
|
||||
_, n, err := xzVLI(r)
|
||||
if err != nil {
|
||||
return streams, blocks, fmt.Errorf("stream %d: index: %v", streams, err)
|
||||
}
|
||||
used += n
|
||||
}
|
||||
if err := skip((4-used%4)%4 + 4); err != nil {
|
||||
return streams, blocks, fmt.Errorf("stream %d: index: %v", streams, err)
|
||||
}
|
||||
footer, err := full(12)
|
||||
if err != nil || footer[10] != 'Y' || footer[11] != 'Z' {
|
||||
return streams, blocks, fmt.Errorf("stream %d: no stream footer", streams)
|
||||
}
|
||||
// Stream padding: zero bytes, a multiple of four, then the end or another stream.
|
||||
zeros := 0
|
||||
for {
|
||||
b, err := r.ReadByte()
|
||||
if err == io.EOF {
|
||||
if zeros%4 != 0 {
|
||||
return streams, blocks, errors.New("stream padding not a multiple of four bytes")
|
||||
}
|
||||
return streams, blocks, nil
|
||||
}
|
||||
if err != nil {
|
||||
return streams, blocks, err
|
||||
}
|
||||
if b != 0 {
|
||||
_ = r.UnreadByte()
|
||||
break
|
||||
}
|
||||
zeros++
|
||||
}
|
||||
if zeros%4 != 0 {
|
||||
return streams, blocks, errors.New("stream padding not a multiple of four bytes")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// xzBlockHeader checks one block header: its CRC32, its flags, and that its last filter is LZMA2, the only
|
||||
// one whose chunks this walk can skip (and the only last filter the reader decodes).
|
||||
func xzBlockHeader(h []byte) error {
|
||||
size := len(h)
|
||||
if crc32.ChecksumIEEE(h[:size-4]) != binary.LittleEndian.Uint32(h[size-4:]) {
|
||||
return errors.New("a corrupt block header")
|
||||
}
|
||||
flags := h[1]
|
||||
if flags&0x3c != 0 {
|
||||
return errors.New("a block header with reserved flags")
|
||||
}
|
||||
body := bytes.NewReader(h[2 : size-4])
|
||||
if flags&0x40 != 0 {
|
||||
if _, _, err := xzVLI(body); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if flags&0x80 != 0 {
|
||||
if _, _, err := xzVLI(body); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
var last uint64
|
||||
for i := 0; i <= int(flags&3); i++ {
|
||||
id, _, err := xzVLI(body)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
props, _, err := xzVLI(body)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if props > uint64(body.Len()) {
|
||||
return errors.New("a corrupt block header")
|
||||
}
|
||||
if _, err := body.Seek(int64(props), io.SeekCurrent); err != nil {
|
||||
return err
|
||||
}
|
||||
last = id
|
||||
}
|
||||
if last != 0x21 {
|
||||
return fmt.Errorf("a block whose last filter is %#x, not LZMA2", last)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// xzVLI reads one of xz's variable-length integers (at most 9 bytes, 7 bits each), and how many bytes it took.
|
||||
func xzVLI(r io.ByteReader) (uint64, int, error) {
|
||||
var v uint64
|
||||
for i := 0; i < 9; i++ {
|
||||
b, err := r.ReadByte()
|
||||
if err != nil {
|
||||
return 0, i, io.ErrUnexpectedEOF
|
||||
}
|
||||
v |= uint64(b&0x7f) << (7 * i)
|
||||
if b&0x80 == 0 {
|
||||
if i > 0 && b == 0 {
|
||||
return 0, i + 1, errors.New("a number not in its shortest form")
|
||||
}
|
||||
return v, i + 1, nil
|
||||
}
|
||||
}
|
||||
return 0, 9, errors.New("a number longer than nine bytes")
|
||||
}
|
||||
@@ -0,0 +1,180 @@
|
||||
package main
|
||||
|
||||
// Copied from mesh-controller origin/main internal/builder/xz_framing_test.go (issue 1590), with xz_framing.go: the
|
||||
// build agent's walk of an xz file's framing, which this module now uses as the build agent does.
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"bufio"
|
||||
"bytes"
|
||||
"encoding/binary"
|
||||
"errors"
|
||||
"hash/crc32"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// xzStream is one xz stream (no integrity check) of data cut into blocks of blockSize bytes, each block's
|
||||
// LZMA2 data stored uncompressed, each block a delta filter before LZMA2 when delta is set: the shape of
|
||||
// `xz --block-size=N --filters=delta,lzma2`, built without any xz tool. listed is how many blocks its index
|
||||
// lists (-1: as many as it holds), so a stream whose index lies can be built too.
|
||||
func xzStream(data []byte, blockSize int, delta bool, listed int) []byte {
|
||||
var out bytes.Buffer
|
||||
flags := []byte{0, 0}
|
||||
out.Write([]byte{0xfd, '7', 'z', 'X', 'Z', 0x00})
|
||||
out.Write(flags)
|
||||
out.Write(binary.LittleEndian.AppendUint32(nil, crc32.ChecksumIEEE(flags)))
|
||||
type record struct{ unpadded, uncompressed uint64 }
|
||||
var records []record
|
||||
for at := 0; at < len(data); at += blockSize {
|
||||
block := data[at:min(at+blockSize, len(data))]
|
||||
if delta {
|
||||
// Delta-encoded with a distance of one; the decoder's history starts empty at each block.
|
||||
encoded := make([]byte, len(block))
|
||||
for i := range block {
|
||||
encoded[i] = block[i]
|
||||
if i > 0 {
|
||||
encoded[i] -= block[i-1]
|
||||
}
|
||||
}
|
||||
block = encoded
|
||||
}
|
||||
header := []byte{2, 0} // 12 bytes
|
||||
if delta {
|
||||
header[1] = 1 // two filters
|
||||
header = append(header, 0x03, 0x01, 0x00)
|
||||
}
|
||||
header = append(header, 0x21, 0x01, 0x00) // LZMA2, a 4 KiB dictionary
|
||||
for len(header) < 8 {
|
||||
header = append(header, 0)
|
||||
}
|
||||
header = binary.LittleEndian.AppendUint32(header, crc32.ChecksumIEEE(header))
|
||||
out.Write(header)
|
||||
size := 0
|
||||
for i, chunk := 0, block; len(chunk) > 0; i++ {
|
||||
n := min(len(chunk), 1<<16)
|
||||
control := byte(2)
|
||||
if i == 0 {
|
||||
control = 1 // the first chunk of a block resets the dictionary
|
||||
}
|
||||
out.WriteByte(control)
|
||||
out.Write(binary.BigEndian.AppendUint16(nil, uint16(n-1)))
|
||||
out.Write(chunk[:n])
|
||||
size += 3 + n
|
||||
chunk = chunk[n:]
|
||||
}
|
||||
out.WriteByte(0)
|
||||
size++
|
||||
out.Write(make([]byte, (4-size%4)%4))
|
||||
records = append(records, record{uint64(len(header) + size), uint64(len(block))})
|
||||
}
|
||||
if listed >= 0 {
|
||||
for len(records) < listed {
|
||||
records = append(records, records[0])
|
||||
}
|
||||
records = records[:listed]
|
||||
}
|
||||
vli := func(b []byte, v uint64) []byte {
|
||||
for v >= 0x80 {
|
||||
b = append(b, byte(v)|0x80)
|
||||
v >>= 7
|
||||
}
|
||||
return append(b, byte(v))
|
||||
}
|
||||
index := vli([]byte{0}, uint64(len(records)))
|
||||
for _, r := range records {
|
||||
index = vli(vli(index, r.unpadded), r.uncompressed)
|
||||
}
|
||||
for len(index)%4 != 0 {
|
||||
index = append(index, 0)
|
||||
}
|
||||
index = binary.LittleEndian.AppendUint32(index, crc32.ChecksumIEEE(index))
|
||||
out.Write(index)
|
||||
footer := binary.LittleEndian.AppendUint32(nil, uint32(len(index)/4-1))
|
||||
footer = append(footer, flags...)
|
||||
out.Write(binary.LittleEndian.AppendUint32(nil, crc32.ChecksumIEEE(footer)))
|
||||
out.Write(footer)
|
||||
out.WriteString("YZ")
|
||||
return out.Bytes()
|
||||
}
|
||||
|
||||
// tarHolding is a tar holding betterbird/data with body.
|
||||
func tarHolding(t *testing.T, body []byte) []byte {
|
||||
t.Helper()
|
||||
var out bytes.Buffer
|
||||
w := tar.NewWriter(&out)
|
||||
if err := w.WriteHeader(&tar.Header{Name: "betterbird/data", Mode: 0o644, Size: int64(len(body)),
|
||||
Typeflag: tar.TypeReg}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
w.Write(body)
|
||||
if err := w.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return out.Bytes()
|
||||
}
|
||||
|
||||
func writeTemp(t *testing.T, body []byte) string {
|
||||
t.Helper()
|
||||
file := filepath.Join(t.TempDir(), "r.tar.xz")
|
||||
if err := os.WriteFile(file, body, 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return file
|
||||
}
|
||||
|
||||
func TestAnXzFileOfManyDeltaBlocksIsRefusedBeforeItIsDecoded(t *testing.T) {
|
||||
// Review 68 on pull request 264: `xz --block-size=1 --delta` of a 200 kB tar kept about 67 MiB live in
|
||||
// the reader, one delta decoder per block. Its index here lists one block, as a file crafted to pass a
|
||||
// check of the index alone would; the walk counts the blocks the reader would decode.
|
||||
body := make([]byte, 200_000)
|
||||
for i := range body {
|
||||
body[i] = byte(i * 7)
|
||||
}
|
||||
crafted := xzStream(tarHolding(t, body), 1, true, 1)
|
||||
path := writeTemp(t, crafted)
|
||||
var before, after runtime.MemStats
|
||||
runtime.GC()
|
||||
runtime.ReadMemStats(&before)
|
||||
err := unpackUpstream(path, t.TempDir(), "betterbird")
|
||||
runtime.ReadMemStats(&after)
|
||||
if err == nil || !strings.Contains(err.Error(), "more than 1024 xz blocks") {
|
||||
t.Fatalf("a file of %d bytes in blocks of one byte was not refused for its blocks: %v", len(crafted), err)
|
||||
}
|
||||
if grew := after.TotalAlloc - before.TotalAlloc; grew > 4<<20 {
|
||||
t.Fatalf("%d bytes were allocated for a refused file", grew)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheXzBlockLimitIsExact(t *testing.T) {
|
||||
data := make([]byte, xzBlocksMax+1)
|
||||
streams, blocks, err := xzFraming(bufio.NewReader(bytes.NewReader(xzStream(data[:xzBlocksMax], 1, true, -1))))
|
||||
if err != nil || streams != 1 || blocks != xzBlocksMax {
|
||||
t.Fatalf("%d blocks: %d streams, %d blocks, %v", xzBlocksMax, streams, blocks, err)
|
||||
}
|
||||
if _, _, err := xzFraming(bufio.NewReader(bytes.NewReader(xzStream(data, 1, true, -1)))); !errors.Is(err, errXzTooManyBlocks) {
|
||||
t.Fatalf("%d blocks: %v", xzBlocksMax+1, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnXzFileOfTooManyStreamsIsRefused(t *testing.T) {
|
||||
var file []byte
|
||||
for i := 0; i <= xzStreamsMax; i++ {
|
||||
file = append(file, xzStream([]byte("x"), 1, false, -1)...)
|
||||
}
|
||||
err := unpackUpstream(writeTemp(t, file), t.TempDir(), "")
|
||||
if err == nil || !strings.Contains(err.Error(), "more than 16 concatenated xz streams") {
|
||||
t.Fatalf("%v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnXzIndexThatDoesNotListItsBlocksIsRefused(t *testing.T) {
|
||||
file := xzStream(tarHolding(t, []byte("x")), 512, false, 1)
|
||||
err := unpackUpstream(writeTemp(t, file), t.TempDir(), "betterbird")
|
||||
if err == nil || !strings.Contains(err.Error(), "its index lists 1 blocks, but it holds 4") {
|
||||
t.Fatalf("%v", err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,52 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// releaseXzSHA256 is the sha256 of testdata/upstream-release.tar.xz: the tar stream of upstream-release.tar.gz,
|
||||
// compressed with `xz -9e` (one stream, one block).
|
||||
const releaseXzSHA256 = "7e59472615d7d654d6f72e8275c83e36203da9db1b91372dcd38a3ff7984f997"
|
||||
|
||||
// pinnedModuleServing is pinnedModule for any fixture file, served at /<its name>.
|
||||
func pinnedModuleServing(t *testing.T, fixture, sha, extra string) (*Module, *keptStates) {
|
||||
t.Helper()
|
||||
body, err := os.ReadFile(filepath.Join("testdata", fixture))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.URL.Path != "/"+fixture {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
w.Write(body)
|
||||
}))
|
||||
t.Cleanup(srv.Close)
|
||||
m, states, _ := archiveModule(t, pinned("https://example.com/"+fixture, sha, extra))
|
||||
client := srv.Client()
|
||||
client.Transport.(*http.Transport).DialContext = func(ctx context.Context, network, _ string) (net.Conn, error) {
|
||||
return (&net.Dialer{}).DialContext(ctx, network, srv.Listener.Addr().String())
|
||||
}
|
||||
m.HTTP = client
|
||||
return m, states
|
||||
}
|
||||
|
||||
// Issue 1590: a pinned release compressed with xz (as betterbird's is) is unpacked as the build agent unpacks it, and
|
||||
// gives the build agent's archive: the same tar under gzip and under xz packs to one stream.
|
||||
func TestAPinnedXzArchiveIsUnpackedAsTheBuildAgentDoes(t *testing.T) {
|
||||
m, states := pinnedModuleServing(t, "upstream-release.tar.xz", releaseXzSHA256, `,"within":"release/app"`)
|
||||
res, st := runBuild(t, m, states, "files")
|
||||
if st.State != delivery.StepPassed {
|
||||
t.Fatalf("the step ended %s: %q", st.State, st.Reason)
|
||||
}
|
||||
sameAsBuildAgent(t, res, buildAgentWithinStream, buildAgentWithinDigest)
|
||||
}
|
||||
@@ -6,6 +6,7 @@ require (
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022160024-9cb4d59093a7
|
||||
github.com/nats-io/nats-server/v2 v2.11.17
|
||||
github.com/nats-io/nats.go v1.54.0
|
||||
github.com/therootcompany/xz v1.0.1
|
||||
)
|
||||
|
||||
require (
|
||||
|
||||
@@ -18,6 +18,8 @@ github.com/nats-io/nkeys v0.4.16 h1:rd5oAuLOb8mnAycB0xleuEBNS1pVVnN0fv/FF34Eypg=
|
||||
github.com/nats-io/nkeys v0.4.16/go.mod h1:llLgWoI0o4z/Q57q2R1kHfmocyhGV6VG/U18Glg1Afs=
|
||||
github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw=
|
||||
github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c=
|
||||
github.com/therootcompany/xz v1.0.1 h1:CmOtsn1CbtmyYiusbfmhmkpAAETj0wBIH6kCYaX+xzw=
|
||||
github.com/therootcompany/xz v1.0.1/go.mod h1:3K3UH1yCKgBneZYhuQUvJ9HPD19UEXEI0BWbMn8qNMY=
|
||||
golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M=
|
||||
golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA=
|
||||
golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
|
||||
Reference in New Issue
Block a user