Read xz with a real dictionary maximum (review 65, hq issue 345)
mesh/merge-gate pass: builds build-agent, mesh-controller, route-proxy → ace, g14, novox, shanks; no bus step; every machine composes with the change as it…
mesh/repo-check pass: its merge-check.sh passed
mesh/delivery superseded: a newer head of the same pull request

ulikunitz/xz's ReaderConfig.DictCap is a minimum: a stream asking for a
1 GiB or 4 GiB dictionary was given it. github.com/therootcompany/xz, a port
of xz-embedded, takes a maximum and refuses a block whose LZMA2 dictionary is
larger before allocating it. 64 MiB, xz's largest preset.
This commit is contained in:
jochen
2026-10-12 15:39:34 +02:00
parent 10f76cafc4
commit 01fc9e1d73
64 changed files with 3342 additions and 7679 deletions
+11 -10
View File
@@ -20,7 +20,7 @@ import (
"time"
"github.com/novox/mesh-controller/internal/catalogue"
"github.com/ulikunitz/xz"
"github.com/therootcompany/xz"
)
// An archive whose source is somebody else's published file, pinned by URL and sha256 (novox/hq ADR 0316
@@ -194,14 +194,15 @@ func unpackUpstream(ctx context.Context, file, into, within string) error {
case bytes.HasPrefix(magic, []byte("BZh")):
stream = bzip2.NewReader(buffered)
case bytes.HasPrefix(magic, []byte{0xfd, '7', 'z', 'X', 'Z', 0x00}):
// Read in Go (github.com/ulikunitz/xz), never by a tool the build seat's image may not carry: the seat's
// image is the catalogue's build-agent recipe, which installs git and docker-cli alone, and a release
// pinned as .tar.xz failed there on 2026-10-12 (hq issue 345). The dictionary is bounded, and what it
// unpacks to is bounded by untar (maxUpstream, maxEntries).
config := xz.ReaderConfig{DictCap: xzDictCap}
zipped, err := config.NewReader(buffered)
// Read in Go (github.com/therootcompany/xz, a port of xz-embedded), never by a tool the build seat's image
// may not carry: the seat's image is the catalogue's build-agent recipe, which installs git and docker-cli
// alone, and a release pinned as .tar.xz failed there on 2026-10-12 (hq issue 345). xzDictMax is a real
// maximum: a block whose LZMA2 dictionary is larger is refused before its dictionary is allocated
// (ErrMemlimit). What the stream unpacks to is bounded by untar (maxUpstream, maxEntries).
zipped, err := xz.NewReader(buffered, xzDictMax)
if err != nil {
return fmt.Errorf("it is not an xz stream this build seat reads: %v", err)
return fmt.Errorf("it is not an xz stream this build seat reads (a dictionary of %d MiB at most): %v",
xzDictMax>>20, err)
}
stream = zipped
default:
@@ -211,9 +212,9 @@ func unpackUpstream(ctx context.Context, file, into, within string) error {
return untar(stream, into, within)
}
// xzDictCap is the largest xz dictionary read: xz's own presets use at most 64 MiB (-9); a stream asking for
// xzDictMax is the largest xz dictionary read: xz's own presets use at most 64 MiB (-9); a stream asking for
// more is refused rather than given the memory.
const xzDictCap = 64 << 20
const xzDictMax = 64 << 20
// untar writes a tar stream's files and directories within `within` into a directory.
func untar(stream io.Reader, into, within string) error {