mesh-controller/internal/builder/xz_framing.go

package builder

import (
	"bufio"
	"bytes"
	"encoding/binary"
	"errors"
	"fmt"
	"hash/crc32"
	"io"
	"os"
)

// xzBlocksMax and xzStreamsMax are the most blocks and the most concatenated streams an xz file may hold for
// the build seat to read it.
//
// **Why a block count is limited at all.** The xz reader (github.com/therootcompany/xz v1.0.1) keeps one
// filter decoder for every block that names a delta or BCJ filter, and frees them only when a stream ends:
// its block counters are reset per stream, not per block. A stream of many tiny blocks therefore grows the
// heap with its length: an `xz --block-size=1 --delta` file of 6 MB kept about 67 MiB live, which comes to
// about 5.6 GiB at the 512 MiB download cap (review 68 on mesh-controller pull request 264, hq issue 345).
//
// **Why 1,024.** At about 340 bytes of decoder a block, 1,024 blocks keep at most about 0.35 MiB live. A
// real release is far below it: Betterbird's archive is 1 stream with 1 block, and multi-threaded xz cuts
// blocks of three times the dictionary (24 MiB at the default preset), so even 512 MiB unpacked in blocks
// of 512 KiB stays within it. Streams are limited to 16 because a release is one stream; the decoders are
// reused from one stream to the next, so the stream limit only bounds the work of this walk.
const (
	xzBlocksMax  = 1024
	xzStreamsMax = 16
)

// xzWithinLimits walks an xz file's framing, without decompressing anything, and refuses one holding more
// blocks or streams than the build seat reads.
//
// **The framing, not the index.** Each stream's index at its end lists its blocks, but the reader compares
// the index with the blocks only after it has decoded them all, so a file whose index lies would still be
// decoded block by block. This walk follows the same framing the reader follows: stream header, each block
// header, the block's LZMA2 chunks (each chunk header carries its own compressed size, so a chunk is skipped
// without being decoded), its padding and check, then the index and the footer. It counts the blocks the
// reader would decode, and it also refuses an index whose count of blocks is not the count walked.
func xzWithinLimits(file string) error {
	f, err := os.Open(file)
	if err != nil {
		return err
	}
	defer f.Close()
	_, _, err = xzFraming(bufio.NewReader(f))
	if errors.Is(err, errXzTooManyBlocks) || errors.Is(err, errXzTooManyStreams) {
		return err
	}
	if err != nil {
		return fmt.Errorf("it is not an xz file this build seat reads: %v", err)
	}
	return nil
}

var (
	errXzTooManyBlocks = fmt.Errorf("it holds more than %d xz blocks, the most this build seat reads: "+
		"its xz reader keeps a filter decoder for every block (hq issue 345)", xzBlocksMax)
	errXzTooManyStreams = fmt.Errorf("it holds more than %d concatenated xz streams, the most this build "+
		"seat reads", xzStreamsMax)
)

// xzFraming counts the streams and blocks of an xz file, checking its framing as it goes. It stops with
// errXzTooManyBlocks or errXzTooManyStreams at the first block or stream over the limit.
func xzFraming(r *bufio.Reader) (streams, blocks int, err error) {
	full := func(n int) ([]byte, error) {
		b := make([]byte, n)
		if _, err := io.ReadFull(r, b); err != nil {
			if err == io.EOF {
				err = io.ErrUnexpectedEOF
			}
			return nil, err
		}
		return b, nil
	}
	skip := func(n int) error {
		if got, err := r.Discard(n); err != nil || got != n {
			return io.ErrUnexpectedEOF
		}
		return nil
	}
	for {
		header, err := full(12)
		if err != nil {
			return streams, blocks, fmt.Errorf("stream %d: header: %v", streams+1, err)
		}
		if !bytes.Equal(header[:6], []byte{0xfd, '7', 'z', 'X', 'Z', 0x00}) {
			return streams, blocks, fmt.Errorf("stream %d: no xz stream header", streams+1)
		}
		if header[6] != 0 || header[7]&0xf0 != 0 ||
			crc32.ChecksumIEEE(header[6:8]) != binary.LittleEndian.Uint32(header[8:12]) {
			return streams, blocks, fmt.Errorf("stream %d: a corrupt stream header", streams+1)
		}
		if streams++; streams > xzStreamsMax {
			return streams, blocks, errXzTooManyStreams
		}
		checkSize := 0
		if check := int(header[7]); check != 0 {
			checkSize = 4 << ((check - 1) / 3)
		}
		inStream := 0
		for {
			first, err := r.ReadByte()
			if err != nil {
				return streams, blocks, fmt.Errorf("stream %d: no index: %v", streams, io.ErrUnexpectedEOF)
			}
			if first == 0 {
				break // the index indicator
			}
			rest, err := full(int(first)*4 + 3)
			if err != nil {
				return streams, blocks, fmt.Errorf("stream %d, block %d: header: %v", streams, inStream+1, err)
			}
			if err := xzBlockHeader(append([]byte{first}, rest...)); err != nil {
				return streams, blocks, fmt.Errorf("stream %d, block %d: %v", streams, inStream+1, err)
			}
			inStream++
			if blocks++; blocks > xzBlocksMax {
				return streams, blocks, errXzTooManyBlocks
			}
			// The block's LZMA2 chunks, up to the end marker.
			size := 0
			for {
				control, err := r.ReadByte()
				if err != nil {
					return streams, blocks, fmt.Errorf("stream %d, block %d: %v", streams, inStream, io.ErrUnexpectedEOF)
				}
				size++
				if control == 0 {
					break
				}
				switch {
				case control == 1 || control == 2: // stored: a 16-bit size less one, then the bytes
					b, err := full(2)
					if err != nil {
						return streams, blocks, err
					}
					n := int(binary.BigEndian.Uint16(b)) + 1
					if err := skip(n); err != nil {
						return streams, blocks, err
					}
					size += 2 + n
				case control >= 0x80: // LZMA: unpacked and packed sizes, the properties byte on a reset
					b, err := full(4)
					if err != nil {
						return streams, blocks, err
					}
					n := int(binary.BigEndian.Uint16(b[2:])) + 1
					size += 4
					if control >= 0xc0 {
						if err := skip(1); err != nil {
							return streams, blocks, err
						}
						size++
					}
					if err := skip(n); err != nil {
						return streams, blocks, err
					}
					size += n
				default:
					return streams, blocks, fmt.Errorf("stream %d, block %d: an LZMA2 chunk of kind %#x",
						streams, inStream, control)
				}
			}
			if pad := (4 - size%4) % 4; pad > 0 {
				b, err := full(pad)
				if err != nil || !bytes.Equal(b, make([]byte, pad)) {
					return streams, blocks, fmt.Errorf("stream %d, block %d: corrupt block padding", streams, inStream)
				}
			}
			if err := skip(checkSize); err != nil {
				return streams, blocks, err
			}
		}
		// The index: its count of records, two numbers each, padding, CRC32.
		used := 1
		count, n, err := xzVLI(r)
		if err != nil {
			return streams, blocks, fmt.Errorf("stream %d: index: %v", streams, err)
		}
		used += n
		if count != uint64(inStream) {
			return streams, blocks, fmt.Errorf("stream %d: its index lists %d blocks, but it holds %d",
				streams, count, inStream)
		}
		for i := 0; i < 2*inStream; i++ {
			_, n, err := xzVLI(r)
			if err != nil {
				return streams, blocks, fmt.Errorf("stream %d: index: %v", streams, err)
			}
			used += n
		}
		if err := skip((4-used%4)%4 + 4); err != nil {
			return streams, blocks, fmt.Errorf("stream %d: index: %v", streams, err)
		}
		footer, err := full(12)
		if err != nil || footer[10] != 'Y' || footer[11] != 'Z' {
			return streams, blocks, fmt.Errorf("stream %d: no stream footer", streams)
		}
		// Stream padding: zero bytes, a multiple of four, then the end or another stream.
		zeros := 0
		for {
			b, err := r.ReadByte()
			if err == io.EOF {
				if zeros%4 != 0 {
					return streams, blocks, errors.New("stream padding not a multiple of four bytes")
				}
				return streams, blocks, nil
			}
			if err != nil {
				return streams, blocks, err
			}
			if b != 0 {
				_ = r.UnreadByte()
				break
			}
			zeros++
		}
		if zeros%4 != 0 {
			return streams, blocks, errors.New("stream padding not a multiple of four bytes")
		}
	}
}

// xzBlockHeader checks one block header: its CRC32, its flags, and that its last filter is LZMA2, the only
// one whose chunks this walk can skip (and the only last filter the reader decodes).
func xzBlockHeader(h []byte) error {
	size := len(h)
	if crc32.ChecksumIEEE(h[:size-4]) != binary.LittleEndian.Uint32(h[size-4:]) {
		return errors.New("a corrupt block header")
	}
	flags := h[1]
	if flags&0x3c != 0 {
		return errors.New("a block header with reserved flags")
	}
	body := bytes.NewReader(h[2 : size-4])
	if flags&0x40 != 0 {
		if _, _, err := xzVLI(body); err != nil {
			return err
		}
	}
	if flags&0x80 != 0 {
		if _, _, err := xzVLI(body); err != nil {
			return err
		}
	}
	var last uint64
	for i := 0; i <= int(flags&3); i++ {
		id, _, err := xzVLI(body)
		if err != nil {
			return err
		}
		props, _, err := xzVLI(body)
		if err != nil {
			return err
		}
		if props > uint64(body.Len()) {
			return errors.New("a corrupt block header")
		}
		if _, err := body.Seek(int64(props), io.SeekCurrent); err != nil {
			return err
		}
		last = id
	}
	if last != 0x21 {
		return fmt.Errorf("a block whose last filter is %#x, not LZMA2", last)
	}
	return nil
}

// xzVLI reads one of xz's variable-length integers (at most 9 bytes, 7 bits each), and how many bytes it took.
func xzVLI(r io.ByteReader) (uint64, int, error) {
	var v uint64
	for i := 0; i < 9; i++ {
		b, err := r.ReadByte()
		if err != nil {
			return 0, i, io.ErrUnexpectedEOF
		}
		v |= uint64(b&0x7f) << (7 * i)
		if b&0x80 == 0 {
			if i > 0 && b == 0 {
				return 0, i + 1, errors.New("a number not in its shortest form")
			}
			return v, i + 1, nil
		}
	}
	return 0, 9, errors.New("a number longer than nine bytes")
}