Read xz with a real dictionary maximum (review 65, hq issue 345)

ulikunitz/xz's ReaderConfig.DictCap is a minimum: a stream asking for a
1 GiB or 4 GiB dictionary was given it. github.com/therootcompany/xz, a port
of xz-embedded, takes a maximum and refuses a block whose LZMA2 dictionary is
larger before allocating it. 64 MiB, xz's largest preset.

cmd/mesh-controller/delivery_conditions.go

@@ -47,10 +47,14 @@ type stalledLine struct {
 	// issue 443): a time inside a finished sentence cannot be said again in another zone. A mesh-delivery from before
 	// says none, and Waiting is said as it reads.
 	Checks []waitingCheck `json:"checks,omitempty"`
-	For    string         `json:"for"`
-	Bound  string         `json:"bound"`
-	H2     string         `json:"h2"`
-	Says   string         `json:"says"`
+	// Held is the state of the delivery of a line of the state `unanswered`, ready or rejected, when one is held
+	// (novox/hq issues 441, 451): its check answered once, so a recheck is the step. Empty for a head with no
+	// delivery, and from a mesh-delivery that does not say it yet; the words are never read for it.
+	Held  string `json:"held,omitempty"`
+	For   string `json:"for"`
+	Bound string `json:"bound"`
+	H2    string `json:"h2"`
+	Says  string `json:"says"`
 }
 
 // waitingCheck is one merge check a stalled line waits on, as mesh-delivery gives it: its context, its state —

cmd/mesh-controller/handover_test.go

@@ -167,3 +167,54 @@ func TestASetuidSearchAsksOnlyAKnownNodeAndFailsOnARefusal(t *testing.T) {
 		t.Fatalf("a start: %v, printed %q", err, out.String())
 	}
 }
+
+// The directory judgement's line asks only a known node, one name and nothing else, fails on a refusal, and says it
+// changed nothing (novox/hq issue 358).
+func TestADirectoryJudgementAsksOnlyAKnownNodeAndSaysItIsReadOnly(t *testing.T) {
+	t.Setenv(link.CallerVar, "jo through mesh-cli on anchor")
+	asked := 0
+	ask := func(answer link.HandOverAnswer) func(node, by string) (link.HandOverAnswer, error) {
+		return func(node, by string) (link.HandOverAnswer, error) {
+			asked++
+			if node != "novox" || by != "jo through mesh-cli on anchor" {
+				t.Fatalf("asked %q %q", node, by)
+			}
+			return answer, nil
+		}
+	}
+	known := func(string) error { return nil }
+	var out bytes.Buffer
+	for _, args := range [][]string{nil, {"novox", "extra"}, {"-x"}} {
+		if err := judgeDirectoriesAsked(args, known, ask(link.HandOverAnswer{Said: "x"}), &out); err == nil || asked != 0 {
+			t.Fatalf("%v was asked: %v", args, err)
+		}
+	}
+	if err := judgeDirectoriesAsked([]string{"novox"}, func(string) error { return errors.New("no node called novox") },
+		ask(link.HandOverAnswer{Said: "x"}), &out); err == nil || asked != 0 {
+		t.Fatalf("an unknown node: %v", err)
+	}
+	if err := judgeDirectoriesAsked([]string{"novox"}, known, ask(link.HandOverAnswer{Refused: "no; nothing was judged"}),
+		&out); err == nil || !strings.Contains(err.Error(), "novox refused") || out.Len() != 0 {
+		t.Fatalf("a refusal: %v, printed %q", err, out.String())
+	}
+	said := "2 directories declared: 1 placed, 1 would be refused\n/var/lib/a: placed\n/var/lib/b: would be refused: …"
+	if err := judgeDirectoriesAsked([]string{"novox"}, known, ask(link.HandOverAnswer{Said: said}), &out); err != nil ||
+		out.String() != "novox (read-only: nothing on novox was changed)\n"+said+"\n" {
+		t.Fatalf("a judgement: %v, printed %q", err, out.String())
+	}
+}
+
+// The judgement is a seat verb an agent may call — it reads, and changes nothing on the node — and runs exactly
+// `node judge-directories <node>` (novox/hq issue 358).
+func TestTheDirectoryJudgementIsAVerbThatOnlyReads(t *testing.T) {
+	argv, err := argvFor("judge-directories", map[string]any{"node": "novox"})
+	if err != nil || strings.Join(argv, " ") != "node judge-directories novox" {
+		t.Fatalf("the verb runs %v, %v", argv, err)
+	}
+	if _, err := argvFor("judge-directories", map[string]any{}); err == nil {
+		t.Fatal("the verb ran with no node")
+	}
+	if !readingVerbs["judge-directories"] {
+		t.Fatal("the judgement is not counted a reading verb")
+	}
+}

cmd/mesh-controller/nodes.go

@@ -127,9 +127,14 @@ func nodeCommand(ctx context.Context, args []string) error {
 		// and whoever may call a verb includes agents.
 		return nodeSetuidSearch(ctx, open, args[1:])
 
+	case "judge-directories":
+		// Which of a node's declared directories the walk that follows no link another account could make would
+		// refuse (novox/hq issue 358). Read-only on the node and here, so a verb may run it too.
+		return nodeJudgeDirectories(ctx, open, args[1:])
+
 	default:
-		return fmt.Errorf("node has no %q; it has add, list, show, public-domain, account, agent-account, hand-over "+
-			"and setuid-search", args[0])
+		return fmt.Errorf("node has no %q; it has add, list, show, public-domain, account, agent-account, hand-over, "+
+			"setuid-search and judge-directories", args[0])
 	}
 }
 
@@ -937,3 +942,57 @@ func printJSONTo(w io.Writer, v any) error {
 	fmt.Fprintln(w, string(body))
 	return nil
 }
+
+const judgeDirectoriesUsage = "node judge-directories <node> — read-only: which of the directories <node> is " +
+	"declared the node-engine's walk that follows no link another account could make would refuse, one line each " +
+	"(novox/hq issue 358). Nothing is made, chmodded or chowned"
+
+// nodeJudgeDirectories asks the node's engine to judge its declared directories, signed with the mesh's key as a
+// hand-over is. Read-only: the engine changes nothing and keeps nothing.
+func nodeJudgeDirectories(ctx context.Context, open *stores, args []string) error {
+	known := func(node string) error {
+		_, err := open.inventory.NodeByName(ctx, node)
+		return err
+	}
+	ask := func(node, by string) (link.HandOverAnswer, error) {
+		ident, err := open.Identity(ctx)
+		if err != nil {
+			return link.HandOverAnswer{}, fmt.Errorf("the mesh's signing key cannot be read, so nothing was asked of %s: %w",
+				node, err)
+		}
+		address, err := broker.BusAddress()
+		if err != nil {
+			return link.HandOverAnswer{}, err
+		}
+		js, err := broker.Dial(address)
+		if err != nil {
+			return link.HandOverAnswer{}, fmt.Errorf("cannot reach the bus, so nothing was asked of %s: %w", node, err)
+		}
+		defer js.Close()
+		return link.AskJudgeDirectories(ctx, js.Conn(), ident, node, by, link.HandOverWithin)
+	}
+	return judgeDirectoriesAsked(args, known, ask, os.Stdout)
+}
+
+// judgeDirectoriesAsked is the line with its two acts given: whether the mesh knows the node, and the ask. The
+// engine's refusal is this command's failure.
+func judgeDirectoriesAsked(args []string, known func(node string) error,
+	ask func(node, by string) (link.HandOverAnswer, error), out io.Writer) error {
+	if len(args) != 1 || args[0] == "" || strings.HasPrefix(args[0], "-") {
+		return errors.New(judgeDirectoriesUsage)
+	}
+	node := args[0]
+	if err := known(node); err != nil {
+		return fmt.Errorf("nothing was asked: %w", err)
+	}
+	answer, err := ask(node, handOverBy())
+	if err != nil {
+		return err
+	}
+	if answer.Refused != "" {
+		return fmt.Errorf("%s refused: %s", node, answer.Refused)
+	}
+	fmt.Fprintf(out, "%s (read-only: nothing on %s was changed)\n", node, node)
+	fmt.Fprintln(out, answer.Said)
+	return nil
+}

cmd/mesh-controller/plain_words.go

@@ -885,6 +885,11 @@ func stalledWords(l stalledLine, o conditions.Observation) (headline, explanatio
 // within its bound (novox/hq issue 438): mesh-delivery holds no delivery of it, so there is nothing to stop, release
 // or close, and no action is offered. The operator's acts are a new commit, which asks the check again, or a look
 // at the build queue, where a check that never ran may still wait.
+//
+// A head whose delivery is held in ready or rejected is said the same way (novox/hq issue 441), and mesh-delivery
+// names that state as data (Held). Its check answered once, so the build queue is the wrong place to look: the step
+// is a recheck of the delivery, named by its id, which asks the check again (novox/hq issue 451). It is advised, not
+// offered as an action, since the controller is not granted the recheck.
 func unansweredWords(l stalledLine, o conditions.Observation) (headline, explanation, resolved, needs string,
 	actions []conditions.Action) {
 	repository, _, _ := strings.Cut(l.ID, "@")
@@ -899,13 +904,23 @@ func unansweredWords(l stalledLine, o conditions.Observation) (headline, explana
 	if d, err := time.ParseDuration(l.Bound); err == nil {
 		limit = ", past its limit of " + humanDuration(d)
 	}
-	which := "Its merge check"
+	check := "merge check"
 	if said := uncheckedWaitWords(l); len(said) > 0 {
-		which = "Its merge check (" + strings.Join(said, ", ") + ")"
+		check = "merge check (" + strings.Join(said, ", ") + ")"
 	}
-	if o.Resolver == conditions.ResolverOperator {
-		needs = "push a new commit to its branch, which asks the check again, or see whether its check still waits " +
-			"in the build queue: mesh-controller.queue."
+	which := "Its " + check
+	switch l.Held {
+	case "ready", "rejected":
+		which = "Its delivery is " + l.Held + ", but on the forge its " + check
+		if o.Resolver == conditions.ResolverOperator {
+			needs = "recheck its delivery, which asks its check again — mesh-delivery.recheck with id " + l.ID +
+				" and why — or push a new commit to its branch, " + FromMeshMCPServer
+		}
+	default:
+		if o.Resolver == conditions.ResolverOperator {
+			needs = "push a new commit to its branch, which asks the check again, or see whether its check still " +
+				"waits in the build queue: mesh-controller.queue."
+		}
 	}
 	return fmt.Sprintf("%s's merge check has not answered %s", pull, long),
 		fmt.Sprintf("%s is open on a branch that requires the merge check. %s was asked and has not answered %s%s. "+

cmd/mesh-controller/plain_words_test.go

@@ -452,3 +452,49 @@ func TestAnUnansweredMergeChecksTimeIsSaidInLocalTime(t *testing.T) {
 		t.Fatalf("a pending check without its time reads %q", o.Explanation)
 	}
 }
+
+// **An unanswered merge check on a ready or rejected delivery is told to recheck it** (novox/hq issue 451):
+// mesh-delivery says such a line too (issue 441) and names the delivery's state as data, `held`. That check already
+// answered once, so the build queue is the wrong place to look; a recheck of the delivery, named by its id, asks the
+// check again. A line with no delivery held keeps the advice of issue 438, and so does a line from a mesh-delivery
+// that does not say `held` yet: the controller never guesses the state from the words.
+func TestAnUnansweredMergeCheckOnAReadyOrRejectedDeliveryOffersARecheck(t *testing.T) {
+	line := func(held string) stalledLine {
+		var l stalledLine
+		raw := `{"id":"novox/mesh-controller@c11222026a3b","number":212,"state":"unanswered",` +
+			`"waiting":["mesh/merge-gate pending since 2026-10-11T00:11:39Z"],"for":"1h1m0s","bound":"1h0m0s",` +
+			`"h2":"none: its delivery is ` + held + `, a state the operator moves — the operator's",` +
+			`"says":"novox/mesh-controller#212 is open on main, which requires the merge check, and its delivery is ` +
+			held + ` here, but on the forge its check has not answered: mesh/merge-gate pending since ` +
+			`2026-10-11T00:11:39Z. The pull request waits on that status and nothing asks it again. A recheck of the ` +
+			`delivery asks its check again, as does a new commit on its branch"`
+		if held != "" {
+			raw += `,"held":"` + held + `"`
+		}
+		if err := json.Unmarshal([]byte(raw+`}`), &l); err != nil {
+			t.Fatal(err)
+		}
+		return l
+	}
+	for _, held := range []string{"ready", "rejected"} {
+		o := stalledObservations([]stalledLine{line(held)})[0]
+		t.Logf("%s: %s\nneeds: %s", held, o.Explanation, o.Needs)
+		if !strings.Contains(o.Needs, "mesh-delivery.recheck") || !strings.Contains(o.Needs, "novox/mesh-controller@c11222026a3b") {
+			t.Fatalf("a %s delivery's unanswered check is not offered its recheck: %q", held, o.Needs)
+		}
+		if strings.Contains(o.Needs, "build queue") || strings.Contains(o.Needs, "mesh-controller.queue") {
+			t.Fatalf("a %s delivery's unanswered check, which answered once, is sent to the build queue: %q", held, o.Needs)
+		}
+		if !strings.Contains(o.Explanation, "Its delivery is "+held) {
+			t.Fatalf("a %s delivery's explanation does not say its state: %q", held, o.Explanation)
+		}
+		if len(o.Actions) != 0 {
+			t.Fatalf("it offers an action the controller is not granted: %+v", o.Actions)
+		}
+	}
+	// No delivery held, or a mesh-delivery that does not say: the advice of issue 438, and no recheck.
+	o := stalledObservations([]stalledLine{line("")})[0]
+	if !strings.Contains(o.Needs, "in the build queue: mesh-controller.queue") || strings.Contains(o.Needs, "recheck") {
+		t.Fatalf("a line with no delivery held says to %q", o.Needs)
+	}
+}

cmd/mesh-controller/queue.go

@@ -8,6 +8,7 @@ import (
 	"fmt"
 	"io"
 	"os"
+	"regexp"
 	"sort"
 	"strings"
 	"time"
@@ -67,18 +68,95 @@ func listQueue(ctx context.Context, args []string, w io.Writer) error {
 	if err != nil {
 		return err
 	}
+	now := time.Now()
 	if *asJSON {
-		body, err := json.MarshalIndent(q, "", "  ")
+		body, err := json.MarshalIndent(queueAnswerOf(q, now), "", "  ")
 		if err != nil {
 			return err
 		}
 		fmt.Fprintln(w, string(body))
 		return nil
 	}
-	fmt.Fprint(w, queueText(q, time.Now()))
+	fmt.Fprint(w, queueText(q, now))
 	return nil
 }
 
+// queueAnswer is the queue as data (novox/hq issue 455): what a caller reads to say how far back an ask
+// waits — "waiting in the build queue behind 2" — which the table alone could not tell it. Text is the
+// table, written from the same read: the verb answers it as its output and the rest as its answer, so the
+// two never disagree.
+type queueAnswer struct {
+	Seat       string         `json:"seat"`
+	Waiting    int            `json:"waiting"`
+	InFlight   int            `json:"in-flight"`
+	Dead       int            `json:"dead"`
+	MaxDeliver int            `json:"max-deliver"`
+	Asks       []queuedAnswer `json:"asks"`
+	Text       string         `json:"text"`
+}
+
+// queuedAnswer is one ask as data. Never what the request carries beyond its source — not `held`, every
+// artifact the mesh has built, nor a check's checkouts beside it.
+type queuedAnswer struct {
+	ID    string `json:"id"`
+	State string `json:"state"`
+	// Kind is "check" for a pull request's merge check, "build" for anything else.
+	Kind       string `json:"kind"`
+	Repository string `json:"repository"`
+	Path       string `json:"path,omitempty"`
+	Ref        string `json:"ref,omitempty"`
+	// Commit is the ref when it names a commit; empty when the ask follows a branch or the default.
+	Commit string `json:"commit,omitempty"`
+	// Machine is the one building an ask in flight; WasOn the one that started it and handed it back.
+	Machine string    `json:"machine,omitempty"`
+	WasOn   string    `json:"was-on,omitempty"`
+	AskedAt time.Time `json:"asked-at,omitzero"`
+	// Since is when the ask came to its state: its start on the machine for one in flight that a
+	// machine said it started, otherwise when it was asked.
+	Since time.Time `json:"since,omitzero"`
+	// Position is its place among the waiting, 1 being handed out next; 0 for an ask not waiting.
+	Position int    `json:"position,omitempty"`
+	Seq      uint64 `json:"seq"`
+}
+
+// aCommitRef is a ref that names a commit by its full hash (SHA-1 or SHA-256).
+var aCommitRef = regexp.MustCompile(`^(?:[0-9a-f]{40}|[0-9a-f]{64})$`)
+
+// queueAnswerOf is the queue as data, in the table's order: in flight, the waiting from the next one to
+// be handed out, then the dead. A pure function of one read (novox/hq issue 455).
+func queueAnswerOf(q link.Queue, now time.Time) queueAnswer {
+	waiting, running, dead := q.Of(link.AskWaiting), q.Of(link.AskInFlight), q.Of(link.AskDead)
+	out := queueAnswer{Seat: q.Seat, Waiting: len(waiting), InFlight: len(running), Dead: len(dead),
+		MaxDeliver: q.MaxDeliver, Asks: []queuedAnswer{}, Text: queueText(q, now)}
+	one := func(a link.QueuedAsk, position int) queuedAnswer {
+		kind := "build"
+		if a.Request.Check != nil {
+			kind = "check"
+		}
+		since := a.AskedAt
+		if a.State == link.AskInFlight && a.On != "" && !a.Started.IsZero() {
+			since = a.Started
+		}
+		commit := ""
+		if aCommitRef.MatchString(a.Ref) {
+			commit = a.Ref
+		}
+		return queuedAnswer{ID: a.ID, State: a.State, Kind: kind, Repository: link.WithoutPassword(a.Repository), Path: a.Path,
+			Ref: a.Ref, Commit: commit, Machine: a.On, WasOn: a.Was, AskedAt: a.AskedAt, Since: since,
+			Position: position, Seq: a.Seq}
+	}
+	for _, a := range running {
+		out.Asks = append(out.Asks, one(a, 0))
+	}
+	for i, a := range waiting {
+		out.Asks = append(out.Asks, one(a, i+1))
+	}
+	for _, a := range dead {
+		out.Asks = append(out.Asks, one(a, 0))
+	}
+	return out
+}
+
 // queueText is the queue as a person reads it: a summary line, then each kind in queue order.
 // Never what an ask carries as `held` — that is every artifact the mesh has built.
 func queueText(q link.Queue, now time.Time) string {
@@ -92,7 +170,7 @@ func queueText(q link.Queue, now time.Time) string {
 		return "asked " + now.Sub(t).Round(time.Second).String() + " ago"
 	}
 	what := func(a link.QueuedAsk) string {
-		s := a.Repository
+		s := link.WithoutPassword(a.Repository)
 		if a.Path != "" {
 			s += " at " + a.Path
 		}

cmd/mesh-controller/queue_answer_test.go

@@ -0,0 +1,137 @@
+package main
+
+import (
+	"encoding/json"
+	"reflect"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/novox/mesh-controller/internal/link"
+)
+
+// The queue answers as data as well as text (novox/hq issue 455): each ask's repository, commit, kind,
+// machine, since and position among the waiting, read from the same queue the table is written from.
+
+const aCommit = "0123456789abcdef0123456789abcdef01234567"
+
+func aQueueOfEachKind(now time.Time) link.Queue {
+	asked := now.Add(-5 * time.Minute)
+	started := now.Add(-2 * time.Minute)
+	return link.Queue{Seat: link.TheBuildMachine, Delivered: 2, AckPending: 1, MaxDeliver: 5, Asks: []link.QueuedAsk{
+		{Seq: 1, ID: "build-dead", Repository: "ssh://forge/novox/x", AskedAt: asked, State: link.AskDead},
+		{Seq: 2, ID: "build-running", Repository: "ssh://forge/novox/mesh-host", Ref: aCommit, AskedAt: asked,
+			State: link.AskInFlight, On: "ace", Started: started,
+			Request: link.BuildRequest{ID: "build-running", Check: &link.CheckRequest{Owner: "novox", Repo: "mesh-host"}}},
+		{Seq: 3, ID: "build-first", Repository: "ssh://forge/novox/mesh-catalog", Ref: "main", AskedAt: asked,
+			State: link.AskWaiting},
+		{Seq: 4, ID: "build-second", Repository: "ssh://forge/novox/mesh-catalog", Path: "modules/gitea", Ref: aCommit,
+			AskedAt: now.Add(-time.Minute), State: link.AskWaiting,
+			Request: link.BuildRequest{ID: "build-second", Check: &link.CheckRequest{Owner: "novox", Repo: "mesh-catalog"}}},
+	}}
+}
+
+func TestTheQueueVerbAnswersEachAskAsData(t *testing.T) {
+	now := time.Date(2026, 10, 12, 12, 0, 0, 0, time.UTC)
+	q := aQueueOfEachKind(now)
+	got := queueAnswerOf(q, now)
+	if got.Waiting != 2 || got.InFlight != 1 || got.Dead != 1 || got.Seat != link.TheBuildMachine {
+		t.Fatalf("counts: %+v", got)
+	}
+	if got.Text != queueText(q, now) {
+		t.Fatalf("the data's text is not the table read from the same queue:\n%s", got.Text)
+	}
+	byID := map[string]queuedAnswer{}
+	for _, a := range got.Asks {
+		byID[a.ID] = a
+	}
+	running := byID["build-running"]
+	if running.Kind != "check" || running.Machine != "ace" || running.Commit != aCommit || running.Position != 0 ||
+		!running.Since.Equal(now.Add(-2*time.Minute)) || running.State != link.AskInFlight {
+		t.Errorf("in flight: %+v", running)
+	}
+	first, second := byID["build-first"], byID["build-second"]
+	if first.Position != 1 || first.Kind != "build" || first.Commit != "" || first.Ref != "main" || first.Machine != "" ||
+		!first.Since.Equal(now.Add(-5*time.Minute)) {
+		t.Errorf("first waiting: %+v", first)
+	}
+	if second.Position != 2 || second.Kind != "check" || second.Commit != aCommit || second.Path != "modules/gitea" ||
+		second.Repository != "ssh://forge/novox/mesh-catalog" {
+		t.Errorf("second waiting: %+v", second)
+	}
+	if dead := byID["build-dead"]; dead.Position != 0 || dead.State != link.AskDead {
+		t.Errorf("dead: %+v", dead)
+	}
+	// In the table's order: in flight, then the waiting by position, then the dead.
+	var order []string
+	for _, a := range got.Asks {
+		order = append(order, a.ID)
+	}
+	if want := []string{"build-running", "build-first", "build-second", "build-dead"}; !reflect.DeepEqual(order, want) {
+		t.Errorf("order %v, want %v", order, want)
+	}
+}
+
+// The verb asks its command for JSON, and its answer carries the data while its output stays the table a
+// person reads, so a text caller reads what it always did.
+func TestTheQueueVerbKeepsTheTableAndCarriesTheData(t *testing.T) {
+	argv, err := argvFor("queue", nil)
+	if err != nil || !reflect.DeepEqual(argv, []string{"queue", "--json"}) {
+		t.Fatalf("queue's command line: %v %v", argv, err)
+	}
+	now := time.Now()
+	q := aQueueOfEachKind(now)
+	body, err := json.Marshal(queueAnswerOf(q, now))
+	if err != nil {
+		t.Fatal(err)
+	}
+	answer := answerOf(argv, body, "", true)
+	if answer.Output != queueText(q, now) {
+		t.Fatalf("the output is not the table:\n%s", answer.Output)
+	}
+	data, ok := answer.Answer.(map[string]any)
+	if !ok {
+		t.Fatalf("no data: %#v", answer.Answer)
+	}
+	if _, doubled := data["text"]; doubled {
+		t.Error("the table is said twice: once as the output and again inside the data")
+	}
+	asks, _ := data["asks"].([]any)
+	if len(asks) != 4 || data["waiting"] != float64(2) {
+		t.Fatalf("data: %v", data)
+	}
+	second := asks[2].(map[string]any)
+	if second["position"] != float64(2) || second["kind"] != "check" || second["commit"] != aCommit {
+		t.Errorf("second waiting: %v", second)
+	}
+	for _, a := range asks {
+		if _, held := a.(map[string]any)["held"]; held {
+			t.Error("an ask carries what the mesh holds")
+		}
+	}
+	// A failure is said as text, as before.
+	failed := answerOf(argv, nil, "mesh-controller: the bus cannot be reached\n", false)
+	if failed.Answer != nil || !strings.Contains(failed.Output, "cannot be reached") {
+		t.Errorf("a failure: %+v", failed)
+	}
+}
+
+// A repository named with a password never shows it, in the table or the data.
+func TestTheQueueNeverShowsARepositorysPassword(t *testing.T) {
+	if got := link.WithoutPassword("https://agent:s3cret@forge.example/novox/x.git"); strings.Contains(got, "s3cret") ||
+		!strings.Contains(got, "agent@forge.example/novox/x.git") {
+		t.Errorf("redacted: %s", got)
+	}
+	now := time.Now()
+	q := link.Queue{Seat: link.TheBuildMachine, Asks: []link.QueuedAsk{{Seq: 1, ID: "build-1",
+		Repository: "https://agent:s3cret@forge.example/novox/x.git", State: link.AskWaiting}}}
+	body, _ := json.Marshal(queueAnswerOf(q, now))
+	if strings.Contains(string(body), "s3cret") {
+		t.Errorf("the queue said a password: %s", body)
+	}
+	for _, kept := range []string{"ssh://git@forge:222/novox/x.git", "novox/x", "git@forge:novox/x.git", ""} {
+		if got := link.WithoutPassword(kept); got != kept {
+			t.Errorf("%q became %q", kept, got)
+		}
+	}
+}

cmd/mesh-controller/queue_test.go

@@ -326,7 +326,7 @@ func TestTheQueueVerbsRunTheirCommands(t *testing.T) {
 		args map[string]any
 		want []string
 	}{
-		{"queue", nil, []string{"queue"}},
+		{"queue", nil, []string{"queue", "--json"}},
 		{"cancel", map[string]any{"id": "build-1"}, []string{"cancel", "build-1"}},
 		{"clear", nil, []string{"clear"}},
 		{"clear", map[string]any{"dead": "true"}, []string{"clear", "--dead"}},

cmd/mesh-controller/seatverbs.go

@@ -314,6 +314,11 @@ func (a *verbArguments) commandLine() ([]string, error) {
 			return nil, err
 		}
 		return []string{"node", "show", str("node")}, nil
+	case "judge-directories":
+		if err := need("node"); err != nil {
+			return nil, err
+		}
+		return []string{"node", "judge-directories", str("node")}, nil
 	case "modules":
 		return []string{"module", "list", "--json"}, nil
 	case "seats":
@@ -426,7 +431,8 @@ func (a *verbArguments) commandLine() ([]string, error) {
 		return argv, nil
 	// The build queue (novox/hq ADR 0219).
 	case "queue":
-		return []string{"queue"}, nil
+		// As data, the table its output (novox/hq issue 455).
+		return []string{"queue", "--json"}, nil
 	case "cancel", "kill":
 		if err := need("id"); err != nil {
 			return nil, err
@@ -977,7 +983,14 @@ var jsonVerbs = map[string]bool{"status": true, "seats": true, "plan": true, "co
 	// What the records say the registries may keep (novox/hq ADR 0251).
 	"artifacts": true, "collect": true, "images": true, "mirrors": true,
 	// The delivery's owner's verbs answer JSON where they read (plan, order, check, walks) — novox/hq ADR 0239.
-	"delivery": true}
+	"delivery": true,
+	// The build queue, each ask with its position among the waiting (novox/hq issue 455).
+	"queue": true}
+
+// tableBesideVerbs are the JSON verbs whose document carries the table a person reads as its `text`: the
+// verb answers that table as its output, as it did before it answered data, and the rest as `answer` —
+// both from the one read, so they never disagree (novox/hq issue 455).
+var tableBesideVerbs = map[string]bool{"queue": true}
 
 // overviewVerbs are the JSON verbs whose answer is said once, as data, and not again as the text the
 // command printed: the overviews, which grow with the mesh (novox/hq issue 314). Every reader of theirs
@@ -1097,6 +1110,12 @@ func answerOf(argv []string, stdout []byte, stderr string, ok bool) verbAnswer {
 		var parsed any
 		if json.Unmarshal(bytes.TrimSpace(stdout), &parsed) == nil {
 			answer.Answer = parsed
+			if doc, isDoc := parsed.(map[string]any); isDoc && tableBesideVerbs[argv[0]] {
+				if table, said := doc["text"].(string); said {
+					answer.Output = table + stderr
+					delete(doc, "text")
+				}
+			}
 			if overviewVerbs[argv[0]] {
 				// Once, as data: the same document again as text doubled an answer that already
 				// outgrew one message of the bus (novox/hq issue 314).
@@ -1479,7 +1498,9 @@ func seatAnnouncement(handlers map[string]link.ToolHandler) micro.Info {
 }
 
 // nodeReads are the `node` subcommands a verb may run: the ones that only read.
-var nodeReads = map[string]bool{"list": true, "show": true}
+var nodeReads = map[string]bool{"list": true, "show": true,
+	// Read-only on the node too: the engine judges its directories and changes nothing (novox/hq issue 358).
+	"judge-directories": true}
 
 // flagsOnly says a command line's rest names no subcommand: empty, or beginning with a flag. For a command
 // with no subcommands every word is a flag, its value or a name it reads.

cmd/mesh-controller/seatverbs_schema_test.go

@@ -256,7 +256,7 @@ func stringLit(e ast.Expr) (string, bool) {
 var accountedFlags = map[string]map[string]string{
 	"status": {"json": "set by the verb: the answer is data"},
 	"seats":  {"json": "set by the verb: the answer is data"},
-	"queue":  {"json": "not set: the verb answers the table a person reads"},
+	"queue":  {"json": "set by the verb: the answer is data, its output the table (novox/hq issue 455)"},
 	"plan":   {"json": "set by the verb unless files is asked"},
 	"push": {
 		"wait": "set by the verb to 0: a tool call cannot hold a connection for a whole apply",

cmd/mesh-controller/status_summary.go

@@ -172,7 +172,7 @@ func composeStatus(open *stores) func(context.Context) ([]byte, error) {
 var readingVerbs = map[string]bool{
 	"tools": true, "calls": true, "status": true, "nodes": true, "node": true, "modules": true,
 	"seats": true, "builds": true, "plan": true, "queue": true, "durations": true, "hand-acts": true,
-	"doctor": true, "conditions": true, "healers": true,
+	"doctor": true, "conditions": true, "healers": true, "judge-directories": true,
 }
 
 // nudgingListener is the enrolment, nudging the summary when a machine said something new.

go.mod

@@ -8,6 +8,7 @@ require (
 	github.com/nats-io/nats-server/v2 v2.11.17
 	github.com/nats-io/nats.go v1.54.0
 	github.com/novox/mesh-host v0.0.0
+	github.com/therootcompany/xz v1.0.1
 	golang.org/x/crypto v0.57.0
 	golang.org/x/net v0.58.0
 	golang.org/x/sys v0.48.0

go.sum

@@ -38,6 +38,8 @@ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UV
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
 github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
+github.com/therootcompany/xz v1.0.1 h1:CmOtsn1CbtmyYiusbfmhmkpAAETj0wBIH6kCYaX+xzw=
+github.com/therootcompany/xz v1.0.1/go.mod h1:3K3UH1yCKgBneZYhuQUvJ9HPD19UEXEI0BWbMn8qNMY=
 golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M=
 golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA=
 golang.org/x/net v0.58.0 h1:ynWG7rqYi4ccpTEuPZ2QGWHktVEM9DMCj9yzDE0Q7To=

internal/broker/nats.go

@@ -336,6 +336,13 @@ func AskHandOverSubject(node string) string { return "mesh.node." + node + ".ask
 // a hand-over is (link.SetuidSearchContext), for the same reason.
 func AskSetuidSearchSubject(node string) string { return "mesh.node." + node + ".ask.setuid-search" }
 
+// AskJudgeDirectoriesSubject is where the controller asks one machine's node-engine to judge, read-only, which of
+// its declared directories the walk that follows no link another account could make would refuse (novox/hq issue
+// 358): a request answered once, signed as a hand-over is (link.JudgeDirectoriesContext), for the same reason.
+func AskJudgeDirectoriesSubject(node string) string {
+	return "mesh.node." + node + ".ask.judge-directories"
+}
+
 // inbox is a principal's own reply space. No user is ever granted a bare `_INBOX.>` (design 25
 // §4): with one account, inbox privacy is the permission list or it is nothing, so each user's
 // inbox is derived from its own identity and its permissions name that prefix and no other.
@@ -614,7 +621,9 @@ func PermissionsFor(p Principal) (Permissions, error) {
 			// issue 356), which it answers on the request's reply: the one request a node is asked.
 			AskHandOverSubject(p.Node),
 			// And asking it for a fresh search for setuid programs (novox/hq issue 361), answered the same way.
-			AskSetuidSearchSubject(p.Node)}
+			AskSetuidSearchSubject(p.Node),
+			// And asking it to judge its declared directories, read-only (novox/hq issue 358), answered the same way.
+			AskJudgeDirectoriesSubject(p.Node)}
 		// The node-engine witnesses the core builds it places (novox/hq to-be 45 §8, ADR 0236; the
 		// contract is lease/witness.go): it asks its own machine's node tools PING, and, where the
 		// machine runs the controller, reads the lease's one key — read, never written.

internal/broker/nats_test.go

@@ -114,7 +114,9 @@ func TestOnlyWhatCanBeAskedMayAnswer(t *testing.T) {
 	node, _ := PermissionsFor(Principal{Kind: KindNode, Node: "one", PasswordHash: "x"})
 	if !node.AllowResponses || !slices.Contains(node.Subscribe, AskHandOverSubject("one")) ||
 		!slices.Contains(node.Subscribe, AskSetuidSearchSubject("one")) ||
-		slices.Contains(node.Subscribe, AskSetuidSearchSubject("two")) {
+		slices.Contains(node.Subscribe, AskSetuidSearchSubject("two")) ||
+		!slices.Contains(node.Subscribe, AskJudgeDirectoriesSubject("one")) ||
+		slices.Contains(node.Subscribe, AskJudgeDirectoriesSubject("two")) {
 		t.Fatalf("a node cannot answer the hand-over it is asked: %v %v", node.AllowResponses, node.Subscribe)
 	}
 	person, _ := PermissionsFor(Principal{Kind: KindPerson, Node: "one", Module: "jo", PasswordHash: "x"})

internal/broker/testdata/composed.conf

@@ -34,7 +34,7 @@ accounts {
       } }
       { user: "node.one", password: "$2a$11$nnnnnnnnnnnnnnnnnnnnnn", permissions: {
         publish: { allow: ["$JS.ACK.NODES.one.>", "$JS.API.CONSUMER.INFO.NODES.one", "$SRV.PING.node-tools.one", "mesh.control.one.>"] }
-        subscribe: { allow: ["_DELIVER.one", "_DELIVER.one.>", "_INBOX.node.one.>", "mesh.node.one.ask.hand-over", "mesh.node.one.ask.report", "mesh.node.one.ask.setuid-search", "mesh.node.one.declare"] }
+        subscribe: { allow: ["_DELIVER.one", "_DELIVER.one.>", "_INBOX.node.one.>", "mesh.node.one.ask.hand-over", "mesh.node.one.ask.judge-directories", "mesh.node.one.ask.report", "mesh.node.one.ask.setuid-search", "mesh.node.one.declare"] }
         allow_responses: { max: 1, ttl: "1m" }
       } }
       { user: "one.nats", password: "$2a$11$bbbbbbbbbbbbbbbbbbbbbb", permissions: {

internal/broker/writers.go

@@ -96,6 +96,11 @@ var WritersTable = []WriterRow{
 	{State: "a fresh setuid search asked of a machine", Writer: "controller, at its terminal",
 		KeptIn: "the machine, which throws its last search away", Others: "the engine verifies the mesh's signature and starts it, or refuses",
 		Subjects: []string{"mesh.node.*.ask.setuid-search"}, Writes: isController},
+	// A read-only judgement of a machine's declared directories asked of its engine (novox/hq issue 358): signed as a
+	// hand-over is, under a signing context of its own. It changes nothing on the machine.
+	{State: "a judgement of a machine's directories asked of it", Writer: "controller",
+		KeptIn: "nowhere: the engine answers it and keeps nothing", Others: "the engine verifies the mesh's signature and answers, or refuses",
+		Subjects: []string{"mesh.node.*.ask.judge-directories"}, Writes: isController},
 	{State: "a machine's applied state and its report", Writer: "the node-engine's apply queue",
 		KeptIn: "the machine; the report on the bus", Others: "the reconcile and a delivery enqueue, never apply",
 		// And its health statement between reports (novox/hq ADR 0240): the same writer stating the same

internal/broker/writers_test.go

@@ -17,6 +17,7 @@ var designRows = []string{
 	"a machine's declaration",
 	"a hand-over asked of a machine",
 	"a fresh setuid search asked of a machine",
+	"a judgement of a machine's directories asked of it",
 	"a machine's applied state and its report",
 	"the controller lease",
 	"plans and their tiers",
@@ -186,3 +187,21 @@ func TestASetuidSearchAskHasOnePublisher(t *testing.T) {
 		}
 	}
 }
+
+// **A judgement of a machine's directories asked of it has one publisher, the controller** (novox/hq issue 358), as a
+// hand-over has.
+func TestADirectoryJudgementAskHasOnePublisher(t *testing.T) {
+	for _, p := range []Principal{
+		{Kind: KindNode, Node: "laptop"},
+		{Kind: KindNodeTools, Node: "laptop", Module: RuntimeModule},
+		{Kind: KindModule, Node: "laptop", Module: "notes"},
+	} {
+		err := CheckWriters(p, []string{AskJudgeDirectoriesSubject("laptop")})
+		if err == nil || !strings.Contains(err.Error(), "a judgement of a machine's directories asked of it") {
+			t.Errorf("%s may ask a judgement of directories: %v", p.Username(), err)
+		}
+	}
+	if err := CheckWriters(Principal{Kind: KindController}, []string{AskJudgeDirectoriesSubject("laptop")}); err != nil {
+		t.Fatalf("the controller may not ask a judgement: %v", err)
+	}
+}

internal/builder/builder.go

@@ -971,8 +971,19 @@ func one(ctx context.Context, run Runner, publish Publisher,
 		return catalogue.Built{Name: a.Name, Kind: a.Kind, Reference: reference}, nil
 
 	case catalogue.ArtifactArchive:
-		src.ownHas(cleanEntry(a.From) + "/**")
-		body, err := pack(filepath.Join(tree, a.From))
+		var body []byte
+		var err error
+		if a.URL != "" {
+			// An upstream file pinned by URL and sha256 (novox/hq ADR 0316): its build source is the pin, which
+			// is in module.json, always part of it.
+			body, err = upstreamArchive(ctx, workspace, a, say)
+			if err != nil {
+				return catalogue.Built{}, fmt.Errorf("%s: %s: %w", module, a.Name, err)
+			}
+		} else {
+			src.ownHas(cleanEntry(a.From) + "/**")
+			body, err = pack(filepath.Join(tree, a.From))
+		}
 		if err != nil {
 			return catalogue.Built{}, fmt.Errorf("%s: packing %s failed: %w", module, a.Name, err)
 		}

internal/builder/upstream_file.go

@@ -0,0 +1,300 @@
+package builder
+
+import (
+	"archive/tar"
+	"bufio"
+	"bytes"
+	"compress/bzip2"
+	"compress/gzip"
+	"context"
+	"crypto/sha256"
+	"encoding/hex"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"os"
+	"path"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"github.com/novox/mesh-controller/internal/catalogue"
+	"github.com/therootcompany/xz"
+)
+
+// An archive whose source is somebody else's published file, pinned by URL and sha256 (novox/hq ADR 0316
+// §2, §6, to-be 56 §7): a mail client's release, an add-on's xpi.
+//
+// **Fetched by the build seat, checked against the pin before anything is read from it, and packed as any
+// archive is packed.** What a machine receives is the mesh's own archive from the artifact store, by digest,
+// exactly as for a directory of the repository; no machine ever asks the vendor. A file whose bytes are not
+// the pin's fails the build and names both digests, because the address serving something else is the one
+// thing the pin exists to catch.
+//
+// **Kept in the build seat's workspace under its sha256**, so it is fetched once: a vendor that serves only
+// its newest version at one address (Owl) would otherwise break every later build of the same pin.
+
+// maxUpstream is the largest upstream file fetched, and the largest a release may unpack to: the node-engine
+// unpacks no archive larger than this either (mesh-host internal/apply/archive.go, maxArchive).
+const maxUpstream = 512 << 20
+
+// maxEntries is the most entries an upstream archive may hold: millions of empty ones would fill the build
+// seat's inodes and the node's. Betterbird's release holds 64.
+const maxEntries = 100_000
+
+// noDowngrade refuses a redirect off https: the digest still guards the bytes, and a fetch in the clear says
+// to whoever is on the path what the build seat fetches.
+func noDowngrade(req *http.Request, via []*http.Request) error {
+	if req.URL.Scheme != "https" {
+		return fmt.Errorf("%s redirected to %s, which is not https", via[0].URL, req.URL)
+	}
+	if len(via) >= 10 {
+		return errors.New("more than 10 redirects")
+	}
+	return nil
+}
+
+// upstreamClient fetches upstream files: a variable so a test serves its own.
+var upstreamClient = &http.Client{Timeout: 15 * time.Minute}
+
+// upstreamDir is where the build seat keeps the upstream files it fetched, by sha256.
+func upstreamDir(workspace string) string { return filepath.Join(workspace, "upstream") }
+
+// upstreamArchive is the packed archive of an artifact pinned by URL: the file fetched (or kept from an
+// earlier build) and checked, then laid out as the artifact says and packed.
+func upstreamArchive(ctx context.Context, workspace string, a catalogue.Artifact,
+	say func(step, format string, args ...any)) ([]byte, error) {
+	file, err := upstreamFile(ctx, workspace, a, say)
+	if err != nil {
+		return nil, err
+	}
+	laid := filepath.Join(workspace, "upstream-"+a.Name)
+	if err := os.RemoveAll(laid); err != nil {
+		return nil, err
+	}
+	defer os.RemoveAll(laid)
+	if err := os.MkdirAll(laid, 0o755); err != nil {
+		return nil, err
+	}
+	if a.File != "" {
+		body, err := os.ReadFile(file)
+		if err != nil {
+			return nil, err
+		}
+		if err := os.WriteFile(filepath.Join(laid, a.File), body, 0o644); err != nil {
+			return nil, err
+		}
+	} else if err := unpackUpstream(ctx, file, laid, a.Within); err != nil {
+		return nil, fmt.Errorf("%s (%s): %w", a.URL, a.Version, err)
+	}
+	return pack(laid)
+}
+
+// upstreamFile is the path of the pinned file in the workspace: kept from an earlier build when its bytes
+// are still the pin's, else fetched now and kept only when they are.
+func upstreamFile(ctx context.Context, workspace string, a catalogue.Artifact,
+	say func(step, format string, args ...any)) (string, error) {
+	want := strings.ToLower(a.SHA256)
+	dir := upstreamDir(workspace)
+	kept := filepath.Join(dir, want)
+	if got, err := sumOf(kept); err == nil {
+		if got == want {
+			say("upstream", "%s %s: kept from an earlier build (sha256:%s)", a.Name, a.Version, short(want))
+			return kept, nil
+		}
+		// Changed on disk since it was kept: fetched again rather than trusted.
+		os.Remove(kept)
+	}
+	if err := os.MkdirAll(dir, 0o755); err != nil {
+		return "", err
+	}
+	say("upstream", "fetching %s (%s)", a.URL, a.Version)
+	request, err := http.NewRequestWithContext(ctx, http.MethodGet, a.URL, nil)
+	if err != nil {
+		return "", err
+	}
+	client := *upstreamClient
+	client.CheckRedirect = noDowngrade
+	response, err := client.Do(request)
+	if err != nil {
+		return "", fmt.Errorf("cannot fetch %s: %w", a.URL, err)
+	}
+	defer response.Body.Close()
+	if response.StatusCode != http.StatusOK {
+		return "", fmt.Errorf("%s answered %s", a.URL, response.Status)
+	}
+	part, err := os.CreateTemp(dir, want+".*.part")
+	if err != nil {
+		return "", err
+	}
+	defer os.Remove(part.Name())
+	hash := sha256.New()
+	n, err := io.Copy(io.MultiWriter(part, hash), io.LimitReader(response.Body, maxUpstream+1))
+	if cerr := part.Close(); err == nil {
+		err = cerr
+	}
+	if err != nil {
+		return "", fmt.Errorf("fetching %s: %w", a.URL, err)
+	}
+	if n > maxUpstream {
+		return "", fmt.Errorf("%s is larger than %d bytes, more than a machine unpacks; nothing was kept", a.URL, maxUpstream)
+	}
+	got := hex.EncodeToString(hash.Sum(nil))
+	if got != want {
+		return "", fmt.Errorf("%s was pinned as sha256:%s and what it served is sha256:%s (%d bytes); nothing "+
+			"was kept. The address serves another file than the one reviewed: compare it, then move the pin "+
+			"in a reviewed change (novox/hq ADR 0316)", a.URL, want, got, n)
+	}
+	if err := os.Rename(part.Name(), kept); err != nil {
+		return "", err
+	}
+	say("upstream", "%s %s: %d bytes, sha256:%s as pinned", a.Name, a.Version, n, short(want))
+	return kept, nil
+}
+
+// sumOf is the sha256 of a file, in hexadecimal.
+func sumOf(file string) (string, error) {
+	f, err := os.Open(file)
+	if err != nil {
+		return "", err
+	}
+	defer f.Close()
+	hash := sha256.New()
+	if _, err := io.Copy(hash, f); err != nil {
+		return "", err
+	}
+	return hex.EncodeToString(hash.Sum(nil)), nil
+}
+
+// unpackUpstream unpacks a tar archive, compressed with gzip, bzip2 or xz (told by its first bytes), into a
+// directory: of its entries only those within `within` (all, when empty), with that prefix taken off.
+//
+// **Only files and directories, and nothing outside.** A link, a device or an entry naming a path outside
+// the archive, or outside `within`, refuses the build by name: the node-engine unpacks only files and
+// directories, and an archive quietly unpacked without what it held would be a release that is not the one
+// pinned.
+func unpackUpstream(ctx context.Context, file, into, within string) error {
+	f, err := os.Open(file)
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+	buffered := bufio.NewReader(f)
+	magic, _ := buffered.Peek(6)
+	var stream io.Reader
+	switch {
+	case bytes.HasPrefix(magic, []byte{0x1f, 0x8b}):
+		zipped, err := gzip.NewReader(buffered)
+		if err != nil {
+			return err
+		}
+		defer zipped.Close()
+		stream = zipped
+	case bytes.HasPrefix(magic, []byte("BZh")):
+		stream = bzip2.NewReader(buffered)
+	case bytes.HasPrefix(magic, []byte{0xfd, '7', 'z', 'X', 'Z', 0x00}):
+		// Read in Go (github.com/therootcompany/xz, a port of xz-embedded), never by a tool the build seat's image
+		// may not carry: the seat's image is the catalogue's build-agent recipe, which installs git and docker-cli
+		// alone, and a release pinned as .tar.xz failed there on 2026-10-12 (hq issue 345). xzDictMax is a real
+		// maximum: a block whose LZMA2 dictionary is larger is refused before its dictionary is allocated
+		// (ErrMemlimit). What the stream unpacks to is bounded by untar (maxUpstream, maxEntries).
+		zipped, err := xz.NewReader(buffered, xzDictMax)
+		if err != nil {
+			return fmt.Errorf("it is not an xz stream this build seat reads (a dictionary of %d MiB at most): %v",
+				xzDictMax>>20, err)
+		}
+		stream = zipped
+	default:
+		return errors.New("it is not a tar archive compressed with gzip, bzip2 or xz; a single file is kept " +
+			"whole by naming it in `file`")
+	}
+	return untar(stream, into, within)
+}
+
+// xzDictMax is the largest xz dictionary read: xz's own presets use at most 64 MiB (-9); a stream asking for
+// more is refused rather than given the memory.
+const xzDictMax = 64 << 20
+
+// untar writes a tar stream's files and directories within `within` into a directory.
+func untar(stream io.Reader, into, within string) error {
+	reader := tar.NewReader(stream)
+	prefix := ""
+	if within != "" {
+		prefix = path.Clean(within) + "/"
+	}
+	var total int64
+	files, entries := 0, 0
+	for {
+		header, err := reader.Next()
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			return err
+		}
+		if entries++; entries > maxEntries {
+			return fmt.Errorf("it holds more than %d entries", maxEntries)
+		}
+		if header.Typeflag == tar.TypeXGlobalHeader {
+			// A pax global header (git archive writes one) names no file.
+			continue
+		}
+		name := path.Clean(strings.TrimPrefix(header.Name, "./"))
+		if path.IsAbs(name) || name == ".." || strings.HasPrefix(name, "../") {
+			return fmt.Errorf("%s names a path outside the archive", header.Name)
+		}
+		if name == "." || name+"/" == prefix ||
+			(header.Typeflag == tar.TypeDir && strings.HasPrefix(prefix, name+"/")) {
+			// The archive's root, the directory taken, or one of its parents.
+			continue
+		}
+		if prefix != "" {
+			if !strings.HasPrefix(name, prefix) {
+				return fmt.Errorf("%s is outside %s, the directory the artifact is taken from", header.Name, within)
+			}
+			name = strings.TrimPrefix(name, prefix)
+		}
+		target := filepath.Join(into, filepath.FromSlash(name))
+		switch header.Typeflag {
+		case tar.TypeDir:
+			if err := os.MkdirAll(target, 0o755); err != nil {
+				return err
+			}
+		case tar.TypeReg:
+			total += header.Size
+			if total > maxUpstream {
+				return fmt.Errorf("it unpacks to more than %d bytes, more than a machine unpacks", maxUpstream)
+			}
+			if err := os.MkdirAll(filepath.Dir(target), 0o755); err != nil {
+				return err
+			}
+			mode := os.FileMode(0o644)
+			if header.Mode&0o111 != 0 {
+				mode = 0o755
+			}
+			out, err := os.OpenFile(target, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, mode)
+			if err != nil {
+				return err
+			}
+			if _, err := io.Copy(out, io.LimitReader(reader, header.Size)); err != nil {
+				out.Close()
+				return err
+			}
+			if err := out.Close(); err != nil {
+				return err
+			}
+			files++
+		default:
+			return fmt.Errorf("%s is a %c, and an archive carries only files and directories", header.Name,
+				header.Typeflag)
+		}
+	}
+	if files == 0 {
+		if within != "" {
+			return fmt.Errorf("it holds no file within %s", within)
+		}
+		return errors.New("it holds no file")
+	}
+	return nil
+}

internal/builder/upstream_file_test.go

@@ -0,0 +1,469 @@
+package builder
+
+import (
+	"archive/tar"
+	"bytes"
+	"compress/gzip"
+	"context"
+	"crypto/sha256"
+	"encoding/binary"
+	"encoding/hex"
+	"fmt"
+	"hash/crc32"
+	"io"
+	"net"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"runtime"
+	"sort"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/novox/mesh-controller/internal/catalogue"
+)
+
+// An archive pinned by URL and sha256 (novox/hq ADR 0316 §2, §6; How it is checked: "an upstream artifact
+// whose download does not match its sha256 fails the build and names both digests").
+
+type tarEntry struct {
+	name, body string
+	mode       int64
+	kind       byte
+}
+
+func tarOf(t *testing.T, entries []tarEntry) []byte {
+	t.Helper()
+	var raw bytes.Buffer
+	w := tar.NewWriter(&raw)
+	for _, e := range entries {
+		kind := e.kind
+		if kind == 0 {
+			kind = tar.TypeReg
+		}
+		h := &tar.Header{Name: e.name, Mode: e.mode, Typeflag: kind, Size: int64(len(e.body))}
+		if kind == tar.TypeSymlink {
+			h.Linkname, h.Size = "/usr/share/hunspell", 0
+		}
+		if kind == tar.TypeDir {
+			h.Size = 0
+		}
+		if err := w.WriteHeader(h); err != nil {
+			t.Fatal(err)
+		}
+		if kind == tar.TypeReg {
+			if _, err := w.Write([]byte(e.body)); err != nil {
+				t.Fatal(err)
+			}
+		}
+	}
+	if err := w.Close(); err != nil {
+		t.Fatal(err)
+	}
+	return raw.Bytes()
+}
+
+func gzipped(t *testing.T, body []byte) []byte {
+	t.Helper()
+	var out bytes.Buffer
+	z := gzip.NewWriter(&out)
+	if _, err := z.Write(body); err != nil {
+		t.Fatal(err)
+	}
+	if err := z.Close(); err != nil {
+		t.Fatal(err)
+	}
+	return out.Bytes()
+}
+
+func sumHex(b []byte) string {
+	s := sha256.Sum256(b)
+	return hex.EncodeToString(s[:])
+}
+
+// vendor serves files by path over https and counts what it was asked.
+type vendor struct {
+	files map[string][]byte
+	asked int
+}
+
+func serveVendor(t *testing.T, v *vendor) string {
+	t.Helper()
+	server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		v.asked++
+		body, ok := v.files[r.URL.Path]
+		if !ok {
+			http.NotFound(w, r)
+			return
+		}
+		w.Write(body)
+	}))
+	t.Cleanup(server.Close)
+	before := upstreamClient
+	// Under a public name, as a manifest must say it: example.com, which the test server's certificate
+	// names, dialled at the server.
+	transport := server.Client().Transport.(*http.Transport).Clone()
+	addr := server.Listener.Addr().String()
+	transport.DialContext = func(ctx context.Context, network, _ string) (net.Conn, error) {
+		return (&net.Dialer{}).DialContext(ctx, network, addr)
+	}
+	upstreamClient = &http.Client{Transport: transport}
+	t.Cleanup(func() { upstreamClient = before })
+	return "https://example.com"
+}
+
+// entriesOf lists a packed archive: each file with its mode and content.
+func entriesOf(t *testing.T, packed []byte) map[string]string {
+	t.Helper()
+	z, err := gzip.NewReader(bytes.NewReader(packed))
+	if err != nil {
+		t.Fatal(err)
+	}
+	r := tar.NewReader(z)
+	out := map[string]string{}
+	for {
+		h, err := r.Next()
+		if err == io.EOF {
+			return out
+		}
+		if err != nil {
+			t.Fatal(err)
+		}
+		body, _ := io.ReadAll(r)
+		out[h.Name] = string(body) + " " + os.FileMode(h.Mode).String()
+	}
+}
+
+func release(t *testing.T) []byte {
+	return gzipped(t, tarOf(t, []tarEntry{
+		{name: "betterbird/", kind: tar.TypeDir, mode: 0o755},
+		{name: "betterbird/application.ini", body: "[App]\nVersion=153.4.0\n", mode: 0o644},
+		{name: "betterbird/betterbird", body: "#!/bin/sh\n", mode: 0o755},
+		{name: "betterbird/defaults/pref/channel-prefs.js", body: "pref(\"app.update.channel\", \"default\");\n", mode: 0o644},
+	}))
+}
+
+func pinnedManifest(base, sum string) string {
+	return `{"module":"betterbird","version":"1",
+  "build":{"artifacts":[
+    {"name":"archive","kind":"archive","url":"` + base + `/release.tar.gz","sha256":"` + sum + `","version":"153.4.0esr-bb10","within":"betterbird"}]},
+  "resources":[{"id":"installation","type":"archive","path":"/opt/betterbird","artifact":"archive"}]}`
+}
+
+func TestAnArchivePinnedByURLIsFetchedCheckedAndPackedWithoutItsOwnDirectory(t *testing.T) {
+	body := release(t)
+	v := &vendor{files: map[string][]byte{"/release.tar.gz": body}}
+	base := serveVendor(t, v)
+	r, workspace := aRepository(t, pinnedManifest(base, sumHex(body)), nil)
+	got, err := Build(context.Background(), r.run, r, "https://forge.invalid/catalogue.git", "", "", workspace, nil,
+		Npmrc{}, GitCredential{}, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	digest, _ := got.Manifest.Resources[0]["digest"].(string)
+	if !strings.HasPrefix(digest, "sha256:") || r.archives["betterbird/archive"] != digest {
+		t.Fatalf("the release was not published and pinned as the module's archive: %v, published %v",
+			got.Manifest.Resources[0], r.archives)
+	}
+
+	packed, err := upstreamArchive(context.Background(), workspace, catalogue.Artifact{Name: "archive", Kind: "archive",
+		URL: base + "/release.tar.gz", SHA256: sumHex(body), Version: "153.4.0esr-bb10", Within: "betterbird"},
+		func(string, string, ...any) {})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if "sha256:"+sumHex(packed) != digest {
+		t.Fatalf("a second packing of one pin gave another digest: %s and %s", digest, sumHex(packed))
+	}
+	entries := entriesOf(t, packed)
+	want := map[string]string{
+		"application.ini":                "[App]\nVersion=153.4.0\n -rw-r--r--",
+		"betterbird":                     "#!/bin/sh\n -rwxr-xr-x",
+		"defaults/pref/channel-prefs.js": "pref(\"app.update.channel\", \"default\");\n -rw-r--r--",
+	}
+	if len(entries) != len(want) {
+		t.Fatalf("the archive holds %v, want %v", entries, want)
+	}
+	for name, w := range want {
+		if entries[name] != w {
+			t.Errorf("%s is %q in the archive, want %q", name, entries[name], w)
+		}
+	}
+}
+
+func TestAnUpstreamFileIsFetchedOnceAndKeptByItsDigest(t *testing.T) {
+	// The vendor of an add-on serves only its newest version at one address: a pin that was fetched once
+	// must still build after the address moved on.
+	body := release(t)
+	v := &vendor{files: map[string][]byte{"/release.tar.gz": body}}
+	base := serveVendor(t, v)
+	r, workspace := aRepository(t, pinnedManifest(base, sumHex(body)), nil)
+	first, err := Build(context.Background(), r.run, r, "https://forge.invalid/catalogue.git", "", "", workspace, nil,
+		Npmrc{}, GitCredential{}, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	v.files["/release.tar.gz"] = []byte("the vendor's next release")
+	second, err := Build(context.Background(), r.run, r, "https://forge.invalid/catalogue.git", "", "", workspace, nil,
+		Npmrc{}, GitCredential{}, nil)
+	if err != nil {
+		t.Fatalf("a pin fetched before failed to build once the address served something else: %v", err)
+	}
+	if v.asked != 1 {
+		t.Fatalf("the vendor was asked %d times for one pin, want once", v.asked)
+	}
+	if first.Manifest.Resources[0]["digest"] != second.Manifest.Resources[0]["digest"] {
+		t.Fatal("two builds of one pin gave two digests")
+	}
+}
+
+func TestAnUpstreamFileWhoseDownloadDoesNotMatchItsPinFailsAndNamesBothDigests(t *testing.T) {
+	pinned := release(t)
+	served := gzipped(t, tarOf(t, []tarEntry{{name: "betterbird/betterbird", body: "not what was reviewed", mode: 0o755}}))
+	v := &vendor{files: map[string][]byte{"/release.tar.gz": served}}
+	base := serveVendor(t, v)
+	r, workspace := aRepository(t, pinnedManifest(base, sumHex(pinned)), nil)
+	_, err := Build(context.Background(), r.run, r, "https://forge.invalid/catalogue.git", "", "", workspace, nil,
+		Npmrc{}, GitCredential{}, nil)
+	if err == nil {
+		t.Fatal("a download that is not the pinned file was built")
+	}
+	for _, digest := range []string{sumHex(pinned), sumHex(served)} {
+		if !strings.Contains(err.Error(), "sha256:"+digest) {
+			t.Errorf("the refusal does not name sha256:%s: %v", digest, err)
+		}
+	}
+	if len(r.archives) != 0 {
+		t.Fatalf("something was published: %v", r.archives)
+	}
+	kept, _ := filepath.Glob(filepath.Join(upstreamDir(workspace), "*"))
+	if len(kept) != 0 {
+		t.Fatalf("the refused download was kept: %v", kept)
+	}
+}
+
+func TestAnUpstreamFileThatIsNotAnArchiveIsKeptWholeUnderItsName(t *testing.T) {
+	xpi := []byte("PK\x03\x04 an add-on")
+	v := &vendor{files: map[string][]byte{"/owl.xpi": xpi}}
+	base := serveVendor(t, v)
+	packed, err := upstreamArchive(context.Background(), t.TempDir(), catalogue.Artifact{Name: "owl", Kind: "archive",
+		URL: base + "/owl.xpi", SHA256: sumHex(xpi), Version: "1.5.6.1", File: "owl.xpi"}, func(string, string, ...any) {})
+	if err != nil {
+		t.Fatal(err)
+	}
+	entries := entriesOf(t, packed)
+	if len(entries) != 1 || entries["owl.xpi"] != string(xpi)+" -rw-r--r--" {
+		t.Fatalf("the add-on was not kept whole as owl.xpi: %v", entries)
+	}
+}
+
+func TestAnUpstreamArchiveHoldingALinkOrAnEntryOutsideItsDirectoryIsRefused(t *testing.T) {
+	cases := map[string][]tarEntry{
+		// The community package links the dictionaries; a release carrying a link is not one a machine unpacks.
+		"is a 2": {{name: "betterbird/betterbird", body: "x", mode: 0o755},
+			{name: "betterbird/dictionaries", kind: tar.TypeSymlink}},
+		"outside betterbird": {{name: "betterbird/betterbird", body: "x", mode: 0o755},
+			{name: "README", body: "x", mode: 0o644}},
+		"outside the archive": {{name: "../etc/passwd", body: "x", mode: 0o644}},
+		"no file within":      {{name: "betterbird/", kind: tar.TypeDir, mode: 0o755}},
+	}
+	for want, entries := range cases {
+		body := gzipped(t, tarOf(t, entries))
+		v := &vendor{files: map[string][]byte{"/r.tar.gz": body}}
+		base := serveVendor(t, v)
+		_, err := upstreamArchive(context.Background(), t.TempDir(), catalogue.Artifact{Name: "archive", Kind: "archive",
+			URL: base + "/r.tar.gz", SHA256: sumHex(body), Version: "1", Within: "betterbird"}, func(string, string, ...any) {})
+		if err == nil || !strings.Contains(err.Error(), want) {
+			t.Errorf("an archive whose entry is %q was not refused for it: %v", want, err)
+		}
+	}
+}
+
+func TestAnUpstreamArchiveCompressedWithXzIsRead(t *testing.T) {
+	// Betterbird publishes its Linux release as .tar.xz; the build seat's image carries xz for it.
+	if _, err := exec.LookPath("xz"); err != nil {
+		t.Skip("NOT TESTED here: no xz on this machine (the build seat's image carries it)")
+	}
+	plain := tarOf(t, []tarEntry{{name: "betterbird/betterbird", body: "#!/bin/sh\n", mode: 0o755}})
+	cmd := exec.Command("xz", "-zc")
+	cmd.Stdin = bytes.NewReader(plain)
+	compressed, err := cmd.Output()
+	if err != nil {
+		t.Fatal(err)
+	}
+	v := &vendor{files: map[string][]byte{"/r.tar.xz": compressed}}
+	base := serveVendor(t, v)
+	packed, err := upstreamArchive(context.Background(), t.TempDir(), catalogue.Artifact{Name: "archive", Kind: "archive",
+		URL: base + "/r.tar.xz", SHA256: sumHex(compressed), Version: "1", Within: "betterbird"}, func(string, string, ...any) {})
+	if err != nil {
+		t.Fatal(err)
+	}
+	var names []string
+	for n := range entriesOf(t, packed) {
+		names = append(names, n)
+	}
+	sort.Strings(names)
+	if strings.Join(names, ",") != "betterbird" {
+		t.Fatalf("the xz release unpacked to %v", names)
+	}
+}
+
+func TestAnUpstreamTarWithAGlobalHeaderNestedWithinAndTrailingDataIsRead(t *testing.T) {
+	// git archive writes a pax global header; a release may unpack under a/b with a/ listed first; xz may be
+	// followed by bytes after the tar's end, which must be read before xz is waited for.
+	var raw bytes.Buffer
+	w := tar.NewWriter(&raw)
+	w.WriteHeader(&tar.Header{Typeflag: tar.TypeXGlobalHeader, Name: "pax_global_header", PAXRecords: map[string]string{"comment": "abc"}})
+	w.WriteHeader(&tar.Header{Name: "a/", Typeflag: tar.TypeDir, Mode: 0o755})
+	w.WriteHeader(&tar.Header{Name: "a/b/", Typeflag: tar.TypeDir, Mode: 0o755})
+	w.WriteHeader(&tar.Header{Name: "a/b/f", Typeflag: tar.TypeReg, Mode: 0o644, Size: 1})
+	w.Write([]byte("x"))
+	w.Close()
+	body := gzipped(t, raw.Bytes())
+	v := &vendor{files: map[string][]byte{"/r.tar.gz": body}}
+	base := serveVendor(t, v)
+	packed, err := upstreamArchive(context.Background(), t.TempDir(), catalogue.Artifact{Name: "a", Kind: "archive",
+		URL: base + "/r.tar.gz", SHA256: sumHex(body), Version: "1", Within: "a/b"}, func(string, string, ...any) {})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if e := entriesOf(t, packed); len(e) != 1 || e["f"] == "" {
+		t.Fatalf("%v", e)
+	}
+	if _, err := exec.LookPath("xz"); err != nil {
+		t.Skip("NOT TESTED here: no xz")
+	}
+	// A megabyte after the tar's end: more than a pipe holds.
+	trailing := append(append([]byte{}, raw.Bytes()...), bytes.Repeat([]byte{0}, 1<<20)...)
+	cmd := exec.Command("xz", "-zc")
+	cmd.Stdin = bytes.NewReader(trailing)
+	compressed, err := cmd.Output()
+	if err != nil {
+		t.Fatal(err)
+	}
+	v.files["/r.tar.xz"] = compressed
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+	if _, err := upstreamArchive(ctx, t.TempDir(), catalogue.Artifact{Name: "a", Kind: "archive", URL: base + "/r.tar.xz",
+		SHA256: sumHex(compressed), Version: "1", Within: "a/b"}, func(string, string, ...any) {}); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestAnUpstreamArchiveWithTooManyEntriesIsRefused(t *testing.T) {
+	var raw bytes.Buffer
+	w := tar.NewWriter(&raw)
+	for i := 0; i <= maxEntries; i++ {
+		w.WriteHeader(&tar.Header{Name: fmt.Sprintf("d%d/", i), Typeflag: tar.TypeDir, Mode: 0o755})
+	}
+	w.Close()
+	body := gzipped(t, raw.Bytes())
+	v := &vendor{files: map[string][]byte{"/r.tar.gz": body}}
+	base := serveVendor(t, v)
+	_, err := upstreamArchive(context.Background(), t.TempDir(), catalogue.Artifact{Name: "a", Kind: "archive",
+		URL: base + "/r.tar.gz", SHA256: sumHex(body), Version: "1"}, func(string, string, ...any) {})
+	if err == nil || !strings.Contains(err.Error(), "entries") {
+		t.Fatalf("%v", err)
+	}
+}
+
+func TestARedirectOffHttpsIsNotFollowed(t *testing.T) {
+	plain := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Write([]byte("x")) }))
+	t.Cleanup(plain.Close)
+	secure := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		http.Redirect(w, r, plain.URL+"/r", http.StatusFound)
+	}))
+	t.Cleanup(secure.Close)
+	before := upstreamClient
+	upstreamClient = secure.Client()
+	t.Cleanup(func() { upstreamClient = before })
+	_, err := upstreamFile(context.Background(), t.TempDir(), catalogue.Artifact{Name: "a", URL: secure.URL + "/r",
+		SHA256: sumHex([]byte("x")), Version: "1", File: "a"}, func(string, string, ...any) {})
+	if err == nil || !strings.Contains(err.Error(), "not https") {
+		t.Fatalf("%v", err)
+	}
+}
+
+func TestAnXzReleaseIsReadWithoutAnyXzToolOnTheBuildSeat(t *testing.T) {
+	// The build seat's image (the catalogue's build-agent recipe) carries git and docker-cli alone: Betterbird's
+	// .tar.xz failed there on 2026-10-12 with "this build seat has no xz to read it" (hq issue 345). It is
+	// read in Go, with nothing on PATH.
+	t.Setenv("PATH", "")
+	body, err := os.ReadFile(filepath.Join("testdata", "release.tar.xz"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	v := &vendor{files: map[string][]byte{"/r.tar.xz": body}}
+	base := serveVendor(t, v)
+	packed, err := upstreamArchive(context.Background(), t.TempDir(), catalogue.Artifact{Name: "archive", Kind: "archive",
+		URL: base + "/r.tar.xz", SHA256: sumHex(body), Version: "1", Within: "betterbird"}, func(string, string, ...any) {})
+	if err != nil {
+		t.Fatal(err)
+	}
+	e := entriesOf(t, packed)
+	if len(e) != 2 || e["betterbird"] != "#!/bin/sh\n -rwxr-xr-x" || e["defaults/p.js"] != "pref(\"a\", 1);\n -rw-r--r--" {
+		t.Fatalf("%v", e)
+	}
+}
+
+// withDictionary is an xz stream whose first block's LZMA2 dictionary property is props, its header's CRC32
+// recomputed: a stream asking for a dictionary of its own choosing.
+func withDictionary(t *testing.T, stream []byte, props byte) []byte {
+	t.Helper()
+	out := append([]byte{}, stream...)
+	const at = 12 // the stream header
+	size := (int(out[at]) + 1) * 4
+	header := out[at : at+size]
+	flags := header[1]
+	i := 2
+	readVarint := func() {
+		for header[i]&0x80 != 0 {
+			i++
+		}
+		i++
+	}
+	if flags&0x40 != 0 {
+		readVarint()
+	}
+	if flags&0x80 != 0 {
+		readVarint()
+	}
+	if header[i] != 0x21 || header[i+1] != 0x01 {
+		t.Fatalf("the fixture's first filter is not LZMA2: % x", header)
+	}
+	header[i+2] = props
+	binary.LittleEndian.PutUint32(header[size-4:], crc32.ChecksumIEEE(header[:size-4]))
+	return out
+}
+
+func TestAnXzStreamAskingForTooLargeADictionaryIsRefused(t *testing.T) {
+	body, err := os.ReadFile(filepath.Join("testdata", "release.tar.xz"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	// 1 GiB (props 36) and 4 GiB less one (props 40): over the 64 MiB maximum, refused before allocation.
+	for _, props := range []byte{36, 40} {
+		crafted := withDictionary(t, body, props)
+		v := &vendor{files: map[string][]byte{"/r.tar.xz": crafted}}
+		base := serveVendor(t, v)
+		var before, after runtime.MemStats
+		runtime.GC()
+		runtime.ReadMemStats(&before)
+		_, err := upstreamArchive(context.Background(), t.TempDir(), catalogue.Artifact{Name: "archive", Kind: "archive",
+			URL: base + "/r.tar.xz", SHA256: sumHex(crafted), Version: "1", Within: "betterbird"}, func(string, string, ...any) {})
+		runtime.ReadMemStats(&after)
+		if err == nil || !strings.Contains(err.Error(), "dictionary") {
+			t.Errorf("props %d: a dictionary over the maximum was not refused: %v", props, err)
+		}
+		if grew := after.TotalAlloc - before.TotalAlloc; grew > 256<<20 {
+			t.Errorf("props %d: %d bytes were allocated for a refused stream", props, grew)
+		}
+	}
+	// The fixture as it is (a small dictionary) still reads.
+	if crafted := withDictionary(t, body, 0); len(crafted) == 0 {
+		t.Fatal("no fixture")
+	}
+}

internal/catalogue/build.go

@@ -2,7 +2,10 @@ package catalogue
 
 import (
 	"fmt"
+	"net"
+	"net/url"
 	"path"
+	"regexp"
 	"sort"
 	"strings"
 )
@@ -312,7 +315,7 @@ func (b *Build) problems(module string) []string {
 					module, a.Name, a.System, spokenSystems()))
 			}
 		} else {
-			if a.From == "" {
+			if a.From == "" && a.URL == "" {
 				problems = append(problems, fmt.Sprintf(
 					"%s: %q says nothing about what it is built from", module, a.Name))
 			}
@@ -350,6 +353,85 @@ func (b *Build) problems(module string) []string {
 			problems = append(problems, fmt.Sprintf(
 				"%s: %q mirrors %q, which names no tag or digest", module, a.Name, a.From))
 		}
+		problems = append(problems, upstreamFileProblems(module, a)...)
+	}
+	return problems
+}
+
+// reservedSuffixes are names no public vendor publishes under: an installation's own and documentation's.
+var reservedSuffixes = []string{".internal", ".local", ".lan", ".home", ".corp", ".localhost", ".home.arpa",
+	".test", ".invalid", ".example", ".intranet", ".private"}
+
+// notAPublicHost says why a host is not a public vendor's name, or "".
+func notAPublicHost(host string) string {
+	h := strings.ToLower(strings.TrimSuffix(host, "."))
+	if net.ParseIP(h) != nil {
+		return "an address rather than a name"
+	}
+	if !strings.Contains(h, ".") {
+		return "a name with no domain"
+	}
+	for _, s := range reservedSuffixes {
+		if strings.HasSuffix(h, s) {
+			return "a name under " + s + ", which is nobody's public domain"
+		}
+	}
+	return ""
+}
+
+// sha256Hex is a pinned sha256 as a manifest writes it: 64 lower-case hexadecimal characters.
+var sha256Hex = regexp.MustCompile(`^[0-9a-f]{64}$`)
+
+// upstreamFileProblems is what is wrong with an archive pinned by URL (novox/hq ADR 0316 §2, §6): it is an
+// archive, fetched over https, pinned by a sha256 and a version, and it names no directory of the repository
+// besides; a file name or a directory within is a plain path that stays where it is put.
+func upstreamFileProblems(module string, a Artifact) []string {
+	pinned := a.URL != "" || a.SHA256 != "" || a.Version != "" || a.File != "" || a.Within != ""
+	if !pinned {
+		return nil
+	}
+	var problems []string
+	say := func(format string, args ...any) {
+		problems = append(problems, fmt.Sprintf("%s: %q ", module, a.Name)+fmt.Sprintf(format, args...))
+	}
+	if a.Kind != ArtifactArchive {
+		say("is a %q and names an upstream file (url, sha256, version, file, within): only an archive is "+
+			"mirrored from one (novox/hq ADR 0316)", a.Kind)
+		return problems
+	}
+	if a.URL == "" {
+		say("names a sha256, a version, a file or a directory within, and no url to fetch it from")
+		return problems
+	}
+	if a.From != "" {
+		say("is built from %q and fetched from %s: an archive is one or the other", a.From, a.URL)
+	}
+	if u, err := url.Parse(a.URL); err != nil || u.Scheme != "https" || u.Host == "" {
+		say("is fetched from %q, and an upstream file is fetched over https from a named host", a.URL)
+	} else if why := notAPublicHost(u.Hostname()); why != "" {
+		say("is fetched from %s, %s: an upstream file is somebody else's release, published under a public name", a.URL, why)
+	}
+	if !sha256Hex.MatchString(a.SHA256) {
+		say("is fetched from %s without a pinned sha256 (64 lower-case hexadecimal characters): what that "+
+			"address serves changes under the mesh, and only the digest says it is what was reviewed", a.URL)
+	}
+	if strings.TrimSpace(a.Version) == "" {
+		say("pins %s and says no version, so a person reading the pin cannot tell which release it is", a.URL)
+	}
+	if a.File != "" && a.Within != "" {
+		say("names a file (%q) and a directory within (%q): a file is kept whole, a directory is taken out "+
+			"of an archive", a.File, a.Within)
+	}
+	for field, p := range map[string]string{"file": a.File, "within": a.Within} {
+		if p == "" {
+			continue
+		}
+		clean := path.Clean(p)
+		if clean != p || strings.HasPrefix(p, "/") || clean == "." || clean == ".." || strings.HasPrefix(clean, "../") ||
+			(field == "file" && strings.Contains(p, "/")) {
+			say("names %s %q, which is not a plain relative %s", field, p, map[string]string{
+				"file": "file name", "within": "directory"}[field])
+		}
 	}
 	return problems
 }

internal/catalogue/installation.go

@@ -71,6 +71,9 @@ var hostname = regexp.MustCompile(
 		`(?:be|nl|de|fr|uk|eu|com|net|org|io|dev|app|cloud|site|online|me|co|ch|at|lu|` +
 		`internal|example|tld|test|invalid)`)
 
+// upstreamVersion is where an artifact pinned by URL says which release it is (Artifact.Version).
+var upstreamVersion = regexp.MustCompile(`^build\.artifacts\[[0-9]+\]\.version$`)
+
 // address is a dotted quad.
 var address = regexp.MustCompile(`(?:[0-9]{1,3}\.){3}[0-9]{1,3}`)
 
@@ -120,7 +123,10 @@ func InstallationProblems(m Manifest) []string {
 			problems = append(problems, fmt.Sprintf("%s names %s at %s", m.Module, name, at))
 		}
 		for _, ip := range addressesIn(value) {
-			if meant[ip] {
+			// An upstream pin's version may have four dotted numbers (Owl 1.5.6.1), which is no address; a name
+			// in it is still judged above. Its URL is judged whole: the vendor's host goes through the artifact's
+			// names-on-purpose like any name a definition means (novox/hq ADR 0316).
+			if meant[ip] || upstreamVersion.MatchString(at) {
 				continue
 			}
 			problems = append(problems, fmt.Sprintf("%s names the public address %s at %s", m.Module, ip, at))

internal/catalogue/manifest.go

@@ -976,6 +976,35 @@ type Artifact struct {
 	// success, and was invisible to the launcher (novox/hq 04-ISSUES/142).
 	Binary string `json:"binary,omitempty"`
 
+	// URL is where an archive's upstream file is fetched from, for an archive that is a release
+	// somebody else published rather than a directory of this repository (novox/hq ADR 0316 §2, §6).
+	// With it come SHA256 and Version, and From stays empty.
+	//
+	// **Pinned by the digest of what the vendor published, mirrored by the build seat.** A release of
+	// about 80 MB copied into a repository is kept for ever in its history at every upgrade, and an
+	// add-on whose licence forbids copying it to third parties may not be in a repository at all. So the
+	// manifest names the file, the build seat fetches it, refuses it when its sha256 is not the pin's and
+	// names both digests, and packs it as any archive is packed: what reaches a machine is the mesh's own
+	// archive in the artifact store, fetched by digest like every other, and a machine never asks the
+	// vendor. Upgrading is changing the pin, reviewed like any other change.
+	URL string `json:"url,omitempty"`
+	// SHA256 is the sha256 of the file at URL, as 64 lower-case hexadecimal characters.
+	SHA256 string `json:"sha256,omitempty"`
+	// Version is what the vendor calls the release pinned, said where a person reads the pin; the build
+	// does not read it.
+	Version string `json:"version,omitempty"`
+	// File is the name an upstream file that is not an archive is kept under: the archive built from it
+	// holds that one file (an add-on's xpi). Empty means the file is a tar archive (gzip, bzip2 or xz)
+	// whose files are the artifact.
+	File string `json:"file,omitempty"`
+	// Within is the directory inside an upstream tar archive whose content is the artifact, for a
+	// release that unpacks into a directory of its own name (`betterbird/`). An entry outside it refuses
+	// the build. Empty means the archive's root.
+	Within string `json:"within,omitempty"`
+	// NamesOnPurpose are the names this artifact means to name, each with why: the vendor's host an upstream
+	// file is fetched from, which the installation check judges like any name (InstallationProblems).
+	NamesOnPurpose map[string]string `json:"names-on-purpose,omitempty"`
+
 	// Entrypoints are the compiled files a tool host should load from this module, relative to the
 	// bundle's root.
 	//

internal/catalogue/upstream_file_test.go

@@ -0,0 +1,101 @@
+package catalogue
+
+import (
+	"strings"
+	"testing"
+)
+
+// An archive pinned by URL and sha256 (novox/hq ADR 0316 §2, §6): the vendor's file, mirrored by the build
+// seat, never a copy in the repository.
+
+const pinnedSum = "38dd24890a66b4cf9a45bc24e15192e00c26b929e1bb8df7974c16a4539129e2"
+
+func upstreamManifest(artifact string) string {
+	return `{"module":"a","version":"1","build":{"artifacts":[` + artifact + `]}}`
+}
+
+func TestAnArchivePinnedByURLAndSha256IsAccepted(t *testing.T) {
+	for _, a := range []string{
+		`{"name":"archive","kind":"archive","url":"https://vendor.example.org/release-1.tar.xz","sha256":"` + pinnedSum +
+			`","version":"1.0","within":"release"}`,
+		`{"name":"addon","kind":"archive","url":"https://vendor.example.org/addon.xpi","sha256":"` + pinnedSum +
+			`","version":"1.5.6.1","file":"addon.xpi"}`,
+	} {
+		if _, err := ParseManifest([]byte(upstreamManifest(a))); err != nil {
+			t.Errorf("%s was refused: %v", a, err)
+		}
+	}
+}
+
+func TestAnUpstreamFileThatIsNotPinnedIsRefused(t *testing.T) {
+	cases := map[string]string{
+		// No digest: what the address serves changes under the mesh.
+		`{"name":"x","kind":"archive","url":"https://vendor.invalid/r.tar.xz","version":"1"}`:                "without a pinned sha256",
+		`{"name":"x","kind":"archive","url":"https://vendor.invalid/r.tar.xz","sha256":"ABC","version":"1"}`: "without a pinned sha256",
+		// No version: a person reading the pin cannot tell which release.
+		`{"name":"x","kind":"archive","url":"https://vendor.invalid/r.tar.xz","sha256":"` + pinnedSum + `"}`: "says no version",
+		// Not https.
+		`{"name":"x","kind":"archive","url":"http://vendor.invalid/r.tar.xz","sha256":"` + pinnedSum + `","version":"1"}`: "over https",
+		// Both a directory of the repository and a URL.
+		`{"name":"x","kind":"archive","from":"files","url":"https://vendor.invalid/r.tar.xz","sha256":"` + pinnedSum +
+			`","version":"1"}`: "one or the other",
+		// A pin without a URL.
+		`{"name":"x","kind":"archive","from":"files","sha256":"` + pinnedSum + `"}`: "no url",
+		// Only an archive is mirrored from a file.
+		`{"name":"x","kind":"image","from":"Dockerfile","url":"https://vendor.invalid/r.tar.xz","sha256":"` + pinnedSum +
+			`","version":"1"}`: "only an archive",
+		// A file name that is a path, a directory within that leaves the archive, and both at once.
+		`{"name":"x","kind":"archive","url":"https://vendor.invalid/a.xpi","sha256":"` + pinnedSum +
+			`","version":"1","file":"../a.xpi"}`: "not a plain relative file name",
+		`{"name":"x","kind":"archive","url":"https://vendor.invalid/r.tar.xz","sha256":"` + pinnedSum +
+			`","version":"1","within":"../up"}`: "not a plain relative directory",
+		`{"name":"x","kind":"archive","url":"https://vendor.invalid/r.tar.xz","sha256":"` + pinnedSum +
+			`","version":"1","within":"r","file":"r.tar.xz"}`: "a file is kept whole",
+	}
+	for artifact, want := range cases {
+		_, err := ParseManifest([]byte(upstreamManifest(artifact)))
+		if err == nil {
+			t.Errorf("%s was accepted", artifact)
+			continue
+		}
+		if !strings.Contains(err.Error(), want) {
+			t.Errorf("%s was refused for another reason than %q: %v", artifact, want, err)
+		}
+	}
+}
+
+func TestAnUpstreamFilesVendorIsNamedOnPurposeAndAnInstallationsIsRefused(t *testing.T) {
+	// The vendor's host is judged like any name, and passes through names-on-purpose; a version of four dotted
+	// numbers is not an address, and a name in it is still judged.
+	m, err := ParseManifest([]byte(upstreamManifest(`{"name":"owl","kind":"archive","url":"https://vendor.be/owl/owl.xpi","sha256":"` +
+		pinnedSum + `","version":"1.5.6.1","file":"owl.xpi","names-on-purpose":{"vendor.be":"Owl's vendor publishes it there"}}`)))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if problems := InstallationProblems(m); len(problems) != 0 {
+		t.Fatalf("the declared vendor and the version were taken for an installation: %v", problems)
+	}
+	m.Build.Artifacts[0].NamesOnPurpose = nil
+	if problems := InstallationProblems(m); len(problems) != 1 || !strings.Contains(problems[0], "vendor.be") {
+		t.Fatalf("an undeclared vendor host is judged: %v", problems)
+	}
+	m.Build.Artifacts[0].Version = "git.novox.be 10.0.0.1"
+	if problems := InstallationProblems(m); len(problems) != 2 {
+		t.Fatalf("a name in the version is judged: %v", problems)
+	}
+	m.Build.Artifacts[0].URL, m.Build.Artifacts[0].Version = "https://mail.novox.be/owl.xpi", "1"
+	if problems := InstallationProblems(m); len(problems) != 1 || !strings.Contains(problems[0], "mail.novox.be") {
+		t.Fatalf("an installation's host in the url is judged: %v", problems)
+	}
+}
+
+func TestAnUpstreamFileFromAnAddressOrAReservedNameIsRefused(t *testing.T) {
+	for _, u := range []string{"https://git.novox.internal/r.tar.xz", "https://10.1.2.3/r.tar.xz",
+		"https://203.0.113.9/r.tar.xz", "https://[2001:db8::1]/r.tar.xz", "https://nas.lan/r.tar.xz", "https://vendor/r.tar.xz"} {
+		_, err := ParseManifest([]byte(upstreamManifest(`{"name":"x","kind":"archive","url":"` + u + `","sha256":"` +
+			pinnedSum + `","version":"1"}`)))
+		if err == nil || !strings.Contains(err.Error(), "published under a public name") {
+			t.Errorf("%s: %v", u, err)
+		}
+	}
+}

internal/catalogue/verbs.go

@@ -333,7 +333,9 @@ var ControllerVerbs = []Verb{
 	// The build queue, controlled by hand (novox/hq ADR 0219). Every verb that drops an ask leaves a
 	// failed outcome for it, so a plan waiting on it fails visibly instead of hanging.
 	{Name: "queue", Description: "Every ask in the build queue: waiting, in flight (on which machine, for how long), " +
-		"and dead (handed out as often as allowed and never settled) — each with its id, repository, path, ref and when it was asked.",
+		"and dead (handed out as often as allowed and never settled) — each with its id, repository, path, ref and when it was asked. " +
+		"Answers the table as text and, as data, each ask's state, kind (check or build), repository, commit, machine, since " +
+		"and position among the waiting (novox/hq issue 455).",
 		Input: schema(nil, nil)},
 	{Name: "cancel", Description: "Drop one waiting or dead ask from the build queue; its outcome is recorded failed, " +
 		"cancelled by hand, and a plan that asked for it fails. One in flight is refused: `kill` ends it where it runs.",
@@ -490,6 +492,13 @@ var ControllerVerbs = []Verb{
 			"why":        "with consumer or older-than: why — required, and recorded in the hand-act log",
 			"cause":      "with consumer or older-than: the cause in a word (cleanup-waiting when absent)",
 		}, nil, "confirm")},
+	{Name: "judge-directories", Description: "Read-only: which of the directories one machine is declared its " +
+		"node-engine's walk that follows no link another account could make would refuse (novox/hq issue 358), one line " +
+		"each: placed, would be refused and why (the link and who could replace it), refused already by the placement " +
+		"guard, or below a home. The engine judges the declaration it keeps and makes, chmods and chowns nothing.",
+		Input: schema(map[string]string{
+			"node": "the machine's name",
+		}, []string{"node"})},
 	// What a consumer gave up on (novox/hq issue 330): kept in DEAD_LETTERS until a person acts on it.
 	{Name: "root-free", Description: "Whether each machine named is root-free now (novox/hq ADR 0259 §8): no agent " +
 		"there can become root without a person. Judged when asked, never from a condition: free only when the machine " +

internal/link/handover.go

@@ -148,6 +148,28 @@ func AskSetuidSearch(ctx context.Context, conn *nats.Conn, signer Signer, node,
 			"there", record: "a start"})
 }
 
+// JudgeDirectoriesContext is prefixed to a directory judgement ask's bytes before signing (novox/hq issue 358): never
+// another ask's signature, nor a declaration's. The engine holds the same words.
+const JudgeDirectoriesContext = "novox-mesh judge-directories v1\n"
+
+// AskJudgeDirectories asks one machine's node-engine to judge, read-only, which of the directories it is declared
+// the walk that follows no link another account could make would refuse (novox/hq issue 358): the ask a hand-over
+// is, naming no path, signed under JudgeDirectoriesContext. The engine changes nothing and answers one line per
+// directory.
+func AskJudgeDirectories(ctx context.Context, conn *nats.Conn, signer Signer, node, by string,
+	timeout time.Duration) (HandOverAnswer, error) {
+	if conn == nil {
+		return HandOverAnswer{}, errors.New("this controller is not on the bus")
+	}
+	body, err := signAsk(ctx, signer, JudgeDirectoriesContext, HandOverAsk{Node: node, By: by})
+	if err != nil {
+		return HandOverAnswer{}, err
+	}
+	return askSigned(ctx, conn, broker.AskJudgeDirectoriesSubject(node), body, node, timeout, askWords{
+		what: "a judgement of its directories", nothing: "nothing was judged, and nothing is changed by one",
+		issue: "issue 358", unknown: "ask again: a judgement changes nothing", record: "a judgement"})
+}
+
 // SignHandOver fills the ask's expiry and nonce and signs it with the mesh's key, over HandOverContext and the
 // ask's bytes exactly as they travel.
 func SignHandOver(ctx context.Context, signer Signer, ask HandOverAsk) ([]byte, error) {

internal/link/judge_directories_test.go

@@ -0,0 +1,60 @@
+package link
+
+import (
+	"context"
+	"crypto/ed25519"
+	"encoding/json"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/nats-io/nats.go"
+
+	"github.com/novox/mesh-controller/internal/broker"
+	"github.com/novox/mesh-controller/internal/testbus"
+)
+
+// A judgement of a machine's directories is asked of the machine on its own subject, signed under its own context
+// (never a hand-over's), naming no path, and the engine's answer comes back (novox/hq issue 358). The engine holds the
+// same subject and context (mesh-host internal/link, TestTheDirectoryJudgementAskKeepsItsSubjectAndContext).
+func TestADirectoryJudgementIsAskedOfTheMachineSignedUnderItsOwnContext(t *testing.T) {
+	if broker.AskJudgeDirectoriesSubject("laptop") != "mesh.node.laptop.ask.judge-directories" ||
+		JudgeDirectoriesContext != "novox-mesh judge-directories v1\n" {
+		t.Fatalf("the subject %q, the context %q", broker.AskJudgeDirectoriesSubject("laptop"), JudgeDirectoriesContext)
+	}
+	conn, err := nats.Connect(testbus.URL(t))
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Cleanup(conn.Close)
+	signer, public := testSigner(t)
+	var heard HandOverAsk
+	engine, err := conn.Subscribe(broker.AskJudgeDirectoriesSubject("laptop"), func(msg *nats.Msg) {
+		var signed SignedHandOver
+		_ = json.Unmarshal(msg.Data, &signed)
+		if ed25519.Verify(public, append([]byte(HandOverContext), signed.Ask...), signed.Signature) ||
+			ed25519.Verify(public, append([]byte(SetuidSearchContext), signed.Ask...), signed.Signature) ||
+			!ed25519.Verify(public, append([]byte(JudgeDirectoriesContext), signed.Ask...), signed.Signature) {
+			_ = msg.Respond([]byte(`{"refused":"not signed as a judgement"}`))
+			return
+		}
+		_ = json.Unmarshal(signed.Ask, &heard)
+		body, _ := json.Marshal(HandOverAnswer{Said: "3 directories declared, asked by " + heard.By})
+		_ = msg.Respond(body)
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Cleanup(func() { _ = engine.Unsubscribe() })
+	a, err := AskJudgeDirectories(context.Background(), conn, signer, "laptop", "jo", 5*time.Second)
+	if err != nil || a.Said != "3 directories declared, asked by jo" {
+		t.Fatalf("answered %+v, %v", a, err)
+	}
+	if heard.Node != "laptop" || heard.Path != "" || heard.Nonce == "" || heard.Expires.IsZero() {
+		t.Fatalf("the engine heard %+v", heard)
+	}
+	if _, err := AskJudgeDirectories(context.Background(), conn, signer, "anchor", "jo", 5*time.Second); err == nil ||
+		!strings.Contains(err.Error(), "nothing was judged") {
+		t.Fatalf("a machine with no engine listening: %v", err)
+	}
+}

internal/link/queue.go

@@ -7,6 +7,7 @@ import (
 	"fmt"
 	"regexp"
 	"sort"
+	"strings"
 	"time"
 
 	"github.com/nats-io/nats.go"
@@ -268,6 +269,30 @@ func ReadQueue(ctx context.Context, js *broker.JetStream, seat string) (Queue, e
 	return q, nil
 }
 
+// WithoutPassword is a repository as a listing may say it: a URL's password, should one ever be written
+// into it, left out (novox/hq issue 455). Only for what is shown: the ask itself keeps what it was given,
+// which a cancel's outcome and a rebuild's match are read from.
+func WithoutPassword(repository string) string {
+	scheme, rest, found := strings.Cut(repository, "://")
+	if !found {
+		return repository
+	}
+	authority, path, _ := strings.Cut(rest, "/")
+	at := strings.LastIndex(authority, "@")
+	if at < 0 {
+		return repository
+	}
+	user, _, password := strings.Cut(authority[:at], ":")
+	if !password {
+		return repository
+	}
+	out := scheme + "://" + user + authority[at:]
+	if len(rest) > len(authority) {
+		out += "/" + path
+	}
+	return out
+}
+
 // ReadBuildEvents is every start and outcome the seat announced since a moment, from the events
 // stream, with a consumer of its own that is gone when this returns.
 func ReadBuildEvents(ctx context.Context, js *broker.JetStream, seat string, since time.Time) (BuildHeard, error) {

module.json

@@ -89,6 +89,7 @@
     "cleanup",
     "dead-letters",
     "root-free",
+    "judge-directories",
     "data",
     "build",
     "artifacts",

vendor/github.com/therootcompany/xz/AUTHORS

@@ -0,0 +1,8 @@
+# Package xz authors
+
+Michael Cross <https://github.com/xi2>
+
+# XZ Embedded authors
+
+Lasse Collin <lasse.collin@tukaani.org>
+Igor Pavlov <http://7-zip.org/>

vendor/github.com/therootcompany/xz/LICENSE

@@ -0,0 +1,86 @@
+Creative Commons CC0 1.0 Universal
+
+CREATIVE COMMONS CORPORATION IS NOT A LAW FIRM AND DOES NOT PROVIDE LEGAL
+SERVICES. DISTRIBUTION OF THIS DOCUMENT DOES NOT CREATE AN ATTORNEY-CLIENT
+RELATIONSHIP. CREATIVE COMMONS PROVIDES THIS INFORMATION ON AN "AS-IS" BASIS.
+CREATIVE COMMONS MAKES NO WARRANTIES REGARDING THE USE OF THIS DOCUMENT OR THE
+INFORMATION OR WORKS PROVIDED HEREUNDER, AND DISCLAIMS LIABILITY FOR DAMAGES
+RESULTING FROM THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS PROVIDED
+HEREUNDER.
+
+Statement of Purpose
+
+The laws of most jurisdictions throughout the world automatically confer
+exclusive Copyright and Related Rights (defined below) upon the creator and
+subsequent owner(s) (each and all, an "owner") of an original work of
+authorship and/or a database (each, a "Work").
+
+Certain owners wish to permanently relinquish those rights to a Work for the
+purpose of contributing to a commons of creative, cultural and scientific
+works ("Commons") that the public can reliably and without fear of later
+claims of infringement build upon, modify, incorporate in other works, reuse
+and redistribute as freely as possible in any form whatsoever and for any
+purposes, including without limitation commercial purposes. These owners may
+contribute to the Commons to promote the ideal of a free culture and the
+further production of creative, cultural and scientific works, or to gain
+reputation or greater distribution for their Work in part through the use and
+efforts of others.
+
+For these and/or other purposes and motivations, and without any expectation
+of additional consideration or compensation, the person associating CC0 with a
+Work (the "Affirmer"), to the extent that he or she is an owner of Copyright
+and Related Rights in the Work, voluntarily elects to apply CC0 to the Work
+and publicly distribute the Work under its terms, with knowledge of his or her
+Copyright and Related Rights in the Work and the meaning and intended legal
+effect of CC0 on those rights.
+
+1. Copyright and Related Rights. A Work made available under CC0 may be protected by copyright and related or neighboring rights ("Copyright and Related Rights"). Copyright and Related Rights include, but are not limited to, the following: 
+
+i. the right to reproduce, adapt, distribute, perform, display, communicate,
+and translate a Work;
+
+ii. moral rights retained by the original author(s) and/or performer(s);
+
+iii. publicity and privacy rights pertaining to a person's image or
+likeness depicted in a Work;
+
+iv. rights protecting against unfair competition in regards to a Work, subject
+to the limitations in paragraph 4(a), below;
+
+v. rights protecting the extraction, dissemination, use and reuse of data in a
+Work;
+
+vi. database rights (such as those arising under Directive 96/9/EC of the
+European Parliament and of the Council of 11 March 1996 on the legal
+protection of databases, and under any national implementation thereof,
+including any amended or successor version of such directive); and
+
+vii. other similar, equivalent or corresponding rights throughout the world
+based on applicable law or treaty, and any national implementations thereof.
+
+2. Waiver. To the greatest extent permitted by, but not in contravention of, applicable law, Affirmer hereby overtly, fully, permanently, irrevocably and unconditionally waives, abandons, and surrenders all of Affirmer's Copyright and Related Rights and associated claims and causes of action, whether now known or unknown (including existing as well as future claims and causes of action), in the Work (i) in all territories worldwide, (ii) for the maximum duration provided by applicable law or treaty (including future time extensions), (iii) in any current or future medium and for any number of copies, and (iv) for any purpose whatsoever, including without limitation commercial, advertising or promotional purposes (the "Waiver"). Affirmer makes the Waiver for the benefit of each member of the public at large and to the detriment of Affirmer's heirs and successors, fully intending that such Waiver shall not be subject to revocation, rescission, cancellation, termination, or any other legal or equitable action to disrupt the quiet enjoyment of the Work by the public as contemplated by Affirmer's express Statement of Purpose. 
+
+3. Public License Fallback. Should any part of the Waiver for any reason be judged legally invalid or ineffective under applicable law, then the Waiver shall be preserved to the maximum extent permitted taking into account Affirmer's express Statement of Purpose. In addition, to the extent the Waiver is so judged Affirmer hereby grants to each affected person a royalty-free, non transferable, non sublicensable, non exclusive, irrevocable and unconditional license to exercise Affirmer's Copyright and Related Rights in the Work (i) in all territories worldwide, (ii) for the maximum duration provided by applicable law or treaty (including future time extensions), (iii) in any current or future medium and for any number of copies, and (iv) for any purpose whatsoever, including without limitation commercial, advertising or promotional purposes (the "License"). The License shall be deemed effective as of the date CC0 was applied by Affirmer to the Work. Should any part of the License for any reason be judged legally invalid or ineffective under applicable law, such partial invalidity or ineffectiveness shall not invalidate the remainder of the License, and in such case Affirmer hereby affirms that he or she will not (i) exercise any of his or her remaining Copyright and Related Rights in the Work or (ii) assert any associated claims and causes of action with respect to the Work, in either case contrary to Affirmer's express Statement of Purpose. 
+
+4. Limitations and Disclaimers. 
+
+a. No trademark or patent rights held by Affirmer are waived, abandoned,
+surrendered, licensed or otherwise affected by this document.
+
+b. Affirmer offers the Work as-is and makes no representations or warranties
+of any kind concerning the Work, express, implied, statutory or otherwise,
+including without limitation warranties of title, merchantability, fitness for
+a particular purpose, non infringement, or the absence of latent or other
+defects, accuracy, or the present or absence of errors, whether or not
+discoverable, all to the greatest extent permissible under applicable law.
+
+c. Affirmer disclaims responsibility for clearing rights of other persons that
+may apply to the Work or any use thereof, including without limitation any
+person's Copyright and Related Rights in the Work. Further, Affirmer
+disclaims responsibility for obtaining any necessary consents, permissions or
+other rights required for any use of the Work.
+
+d. Affirmer understands and acknowledges that Creative Commons is not a party
+to this document and has no duty or obligation with respect to this CC0 or use
+of the Work.
+

vendor/github.com/therootcompany/xz/README.md

@@ -0,0 +1,19 @@
+# Xz
+
+Package xz implements XZ decompression natively in Go.
+
+Documentation at <https://pkg.go.dev/github.com/therootcompany/xz>.
+
+Download and install with `go get github.com/xi2/xz`.
+
+If you need compression as well as decompression, you might want to
+look at <https://github.com/ulikunitz/xz>.
+
+# LICENSE
+
+This was originally released into the public domain by the AUTHORS.
+
+Here it is licensed more explicitly as Creative Commons CC0 1.0 Universal
+(CC0-1.0) so that it can be detected by automated tooling, and satisfy the legal
+requirements for vendor integration for cases in which a "public domain"
+statement is not sufficient.

vendor/github.com/therootcompany/xz/dec_bcj.go

@@ -0,0 +1,461 @@
+/*
+ * Branch/Call/Jump (BCJ) filter decoders
+ *
+ * Authors: Lasse Collin <lasse.collin@tukaani.org>
+ *          Igor Pavlov <http://7-zip.org/>
+ *
+ * Translation to Go: Michael Cross <https://github.com/xi2>
+ *
+ * This file has been put into the public domain.
+ * You can do whatever you want with this file.
+ */
+
+package xz
+
+/* from linux/lib/xz/xz_dec_bcj.c *************************************/
+
+type xzDecBCJ struct {
+	/* Type of the BCJ filter being used */
+	typ xzFilterID
+	/*
+	 * Return value of the next filter in the chain. We need to preserve
+	 * this information across calls, because we must not call the next
+	 * filter anymore once it has returned xzStreamEnd
+	 */
+	ret xzRet
+	/*
+	 * Absolute position relative to the beginning of the uncompressed
+	 * data (in a single .xz Block).
+	 */
+	pos int
+	/* x86 filter state */
+	x86PrevMask uint32
+	/* Temporary space to hold the variables from xzBuf */
+	out    []byte
+	outPos int
+	temp   struct {
+		/* Amount of already filtered data in the beginning of buf */
+		filtered int
+		/*
+		 * Buffer to hold a mix of filtered and unfiltered data. This
+		 * needs to be big enough to hold Alignment + 2 * Look-ahead:
+		 *
+		 * Type         Alignment   Look-ahead
+		 * x86              1           4
+		 * PowerPC          4           0
+		 * IA-64           16           0
+		 * ARM              4           0
+		 * ARM-Thumb        2           2
+		 * SPARC            4           0
+		 */
+		buf      []byte // slice buf will be backed by bufArray
+		bufArray [16]byte
+	}
+}
+
+/*
+ * This is used to test the most significant byte of a memory address
+ * in an x86 instruction.
+ */
+func bcjX86TestMSByte(b byte) bool {
+	return b == 0x00 || b == 0xff
+}
+
+func bcjX86Filter(s *xzDecBCJ, buf []byte) int {
+	var maskToAllowedStatus = []bool{
+		true, true, true, false, true, false, false, false,
+	}
+	var maskToBitNum = []byte{0, 1, 2, 2, 3, 3, 3, 3}
+	var i int
+	var prevPos int = -1
+	var prevMask uint32 = s.x86PrevMask
+	var src uint32
+	var dest uint32
+	var j uint32
+	var b byte
+	if len(buf) <= 4 {
+		return 0
+	}
+	for i = 0; i < len(buf)-4; i++ {
+		if buf[i]&0xfe != 0xe8 {
+			continue
+		}
+		prevPos = i - prevPos
+		if prevPos > 3 {
+			prevMask = 0
+		} else {
+			prevMask = (prevMask << (uint(prevPos) - 1)) & 7
+			if prevMask != 0 {
+				b = buf[i+4-int(maskToBitNum[prevMask])]
+				if !maskToAllowedStatus[prevMask] || bcjX86TestMSByte(b) {
+					prevPos = i
+					prevMask = prevMask<<1 | 1
+					continue
+				}
+			}
+		}
+		prevPos = i
+		if bcjX86TestMSByte(buf[i+4]) {
+			src = getLE32(buf[i+1:])
+			for {
+				dest = src - uint32(s.pos+i+5)
+				if prevMask == 0 {
+					break
+				}
+				j = uint32(maskToBitNum[prevMask]) * 8
+				b = byte(dest >> (24 - j))
+				if !bcjX86TestMSByte(b) {
+					break
+				}
+				src = dest ^ (1<<(32-j) - 1)
+			}
+			dest &= 0x01FFFFFF
+			dest |= 0 - dest&0x01000000
+			putLE32(dest, buf[i+1:])
+			i += 4
+		} else {
+			prevMask = prevMask<<1 | 1
+		}
+	}
+	prevPos = i - prevPos
+	if prevPos > 3 {
+		s.x86PrevMask = 0
+	} else {
+		s.x86PrevMask = prevMask << (uint(prevPos) - 1)
+	}
+	return i
+}
+
+func bcjPowerPCFilter(s *xzDecBCJ, buf []byte) int {
+	var i int
+	var instr uint32
+	for i = 0; i+4 <= len(buf); i += 4 {
+		instr = getBE32(buf[i:])
+		if instr&0xFC000003 == 0x48000001 {
+			instr &= 0x03FFFFFC
+			instr -= uint32(s.pos + i)
+			instr &= 0x03FFFFFC
+			instr |= 0x48000001
+			putBE32(instr, buf[i:])
+		}
+	}
+	return i
+}
+
+var bcjIA64BranchTable = [...]byte{
+	0, 0, 0, 0, 0, 0, 0, 0,
+	0, 0, 0, 0, 0, 0, 0, 0,
+	4, 4, 6, 6, 0, 0, 7, 7,
+	4, 4, 0, 0, 4, 4, 0, 0,
+}
+
+func bcjIA64Filter(s *xzDecBCJ, buf []byte) int {
+	var branchTable = bcjIA64BranchTable[:]
+	/*
+	 * The local variables take a little bit stack space, but it's less
+	 * than what LZMA2 decoder takes, so it doesn't make sense to reduce
+	 * stack usage here without doing that for the LZMA2 decoder too.
+	 */
+	/* Loop counters */
+	var i int
+	var j int
+	/* Instruction slot (0, 1, or 2) in the 128-bit instruction word */
+	var slot uint32
+	/* Bitwise offset of the instruction indicated by slot */
+	var bitPos uint32
+	/* bit_pos split into byte and bit parts */
+	var bytePos uint32
+	var bitRes uint32
+	/* Address part of an instruction */
+	var addr uint32
+	/* Mask used to detect which instructions to convert */
+	var mask uint32
+	/* 41-bit instruction stored somewhere in the lowest 48 bits */
+	var instr uint64
+	/* Instruction normalized with bit_res for easier manipulation */
+	var norm uint64
+	for i = 0; i+16 <= len(buf); i += 16 {
+		mask = uint32(branchTable[buf[i]&0x1f])
+		for slot, bitPos = 0, 5; slot < 3; slot, bitPos = slot+1, bitPos+41 {
+			if (mask>>slot)&1 == 0 {
+				continue
+			}
+			bytePos = bitPos >> 3
+			bitRes = bitPos & 7
+			instr = 0
+			for j = 0; j < 6; j++ {
+				instr |= uint64(buf[i+j+int(bytePos)]) << (8 * uint(j))
+			}
+			norm = instr >> bitRes
+			if (norm>>37)&0x0f == 0x05 && (norm>>9)&0x07 == 0 {
+				addr = uint32((norm >> 13) & 0x0fffff)
+				addr |= (uint32(norm>>36) & 1) << 20
+				addr <<= 4
+				addr -= uint32(s.pos + i)
+				addr >>= 4
+				norm &= ^(uint64(0x8fffff) << 13)
+				norm |= uint64(addr&0x0fffff) << 13
+				norm |= uint64(addr&0x100000) << (36 - 20)
+				instr &= 1<<bitRes - 1
+				instr |= norm << bitRes
+				for j = 0; j < 6; j++ {
+					buf[i+j+int(bytePos)] = byte(instr >> (8 * uint(j)))
+				}
+			}
+		}
+	}
+	return i
+}
+
+func bcjARMFilter(s *xzDecBCJ, buf []byte) int {
+	var i int
+	var addr uint32
+	for i = 0; i+4 <= len(buf); i += 4 {
+		if buf[i+3] == 0xeb {
+			addr = uint32(buf[i]) | uint32(buf[i+1])<<8 |
+				uint32(buf[i+2])<<16
+			addr <<= 2
+			addr -= uint32(s.pos + i + 8)
+			addr >>= 2
+			buf[i] = byte(addr)
+			buf[i+1] = byte(addr >> 8)
+			buf[i+2] = byte(addr >> 16)
+		}
+	}
+	return i
+}
+
+func bcjARMThumbFilter(s *xzDecBCJ, buf []byte) int {
+	var i int
+	var addr uint32
+	for i = 0; i+4 <= len(buf); i += 2 {
+		if buf[i+1]&0xf8 == 0xf0 && buf[i+3]&0xf8 == 0xf8 {
+			addr = uint32(buf[i+1]&0x07)<<19 |
+				uint32(buf[i])<<11 |
+				uint32(buf[i+3]&0x07)<<8 |
+				uint32(buf[i+2])
+			addr <<= 1
+			addr -= uint32(s.pos + i + 4)
+			addr >>= 1
+			buf[i+1] = byte(0xf0 | (addr>>19)&0x07)
+			buf[i] = byte(addr >> 11)
+			buf[i+3] = byte(0xf8 | (addr>>8)&0x07)
+			buf[i+2] = byte(addr)
+			i += 2
+		}
+	}
+	return i
+}
+
+func bcjSPARCFilter(s *xzDecBCJ, buf []byte) int {
+	var i int
+	var instr uint32
+	for i = 0; i+4 <= len(buf); i += 4 {
+		instr = getBE32(buf[i:])
+		if instr>>22 == 0x100 || instr>>22 == 0x1ff {
+			instr <<= 2
+			instr -= uint32(s.pos + i)
+			instr >>= 2
+			instr = (0x40000000 - instr&0x400000) |
+				0x40000000 | (instr & 0x3FFFFF)
+			putBE32(instr, buf[i:])
+		}
+	}
+	return i
+}
+
+/*
+ * Apply the selected BCJ filter. Update *pos and s.pos to match the amount
+ * of data that got filtered.
+ */
+func bcjApply(s *xzDecBCJ, buf []byte, pos *int) {
+	var filtered int
+	buf = buf[*pos:]
+	switch s.typ {
+	case idBCJX86:
+		filtered = bcjX86Filter(s, buf)
+	case idBCJPowerPC:
+		filtered = bcjPowerPCFilter(s, buf)
+	case idBCJIA64:
+		filtered = bcjIA64Filter(s, buf)
+	case idBCJARM:
+		filtered = bcjARMFilter(s, buf)
+	case idBCJARMThumb:
+		filtered = bcjARMThumbFilter(s, buf)
+	case idBCJSPARC:
+		filtered = bcjSPARCFilter(s, buf)
+	default:
+		/* Never reached */
+	}
+	*pos += filtered
+	s.pos += filtered
+}
+
+/*
+ * Flush pending filtered data from temp to the output buffer.
+ * Move the remaining mixture of possibly filtered and unfiltered
+ * data to the beginning of temp.
+ */
+func bcjFlush(s *xzDecBCJ, b *xzBuf) {
+	var copySize int
+	copySize = len(b.out) - b.outPos
+	if copySize > s.temp.filtered {
+		copySize = s.temp.filtered
+	}
+	copy(b.out[b.outPos:], s.temp.buf[:copySize])
+	b.outPos += copySize
+	s.temp.filtered -= copySize
+	copy(s.temp.buf, s.temp.buf[copySize:])
+	s.temp.buf = s.temp.buf[:len(s.temp.buf)-copySize]
+}
+
+/*
+ * Decode raw stream which has a BCJ filter as the first filter.
+ *
+ * The BCJ filter functions are primitive in sense that they process the
+ * data in chunks of 1-16 bytes. To hide this issue, this function does
+ * some buffering.
+ */
+func xzDecBCJRun(s *xzDecBCJ, b *xzBuf, chain func(*xzBuf) xzRet) xzRet {
+	var outStart int
+	/*
+	 * Flush pending already filtered data to the output buffer. Return
+	 * immediately if we couldn't flush everything, or if the next
+	 * filter in the chain had already returned xzStreamEnd.
+	 */
+	if s.temp.filtered > 0 {
+		bcjFlush(s, b)
+		if s.temp.filtered > 0 {
+			return xzOK
+		}
+		if s.ret == xzStreamEnd {
+			return xzStreamEnd
+		}
+	}
+	/*
+	 * If we have more output space than what is currently pending in
+	 * temp, copy the unfiltered data from temp to the output buffer
+	 * and try to fill the output buffer by decoding more data from the
+	 * next filter in the chain. Apply the BCJ filter on the new data
+	 * in the output buffer. If everything cannot be filtered, copy it
+	 * to temp and rewind the output buffer position accordingly.
+	 *
+	 * This needs to be always run when len(temp.buf) == 0 to handle a special
+	 * case where the output buffer is full and the next filter has no
+	 * more output coming but hasn't returned xzStreamEnd yet.
+	 */
+	if len(s.temp.buf) < len(b.out)-b.outPos || len(s.temp.buf) == 0 {
+		outStart = b.outPos
+		copy(b.out[b.outPos:], s.temp.buf)
+		b.outPos += len(s.temp.buf)
+		s.ret = chain(b)
+		if s.ret != xzStreamEnd && s.ret != xzOK {
+			return s.ret
+		}
+		bcjApply(s, b.out[:b.outPos], &outStart)
+		/*
+		 * As an exception, if the next filter returned xzStreamEnd,
+		 * we can do that too, since the last few bytes that remain
+		 * unfiltered are meant to remain unfiltered.
+		 */
+		if s.ret == xzStreamEnd {
+			return xzStreamEnd
+		}
+		s.temp.buf = s.temp.bufArray[:b.outPos-outStart]
+		b.outPos -= len(s.temp.buf)
+		copy(s.temp.buf, b.out[b.outPos:])
+		/*
+		 * If there wasn't enough input to the next filter to fill
+		 * the output buffer with unfiltered data, there's no point
+		 * to try decoding more data to temp.
+		 */
+		if b.outPos+len(s.temp.buf) < len(b.out) {
+			return xzOK
+		}
+	}
+	/*
+	 * We have unfiltered data in temp. If the output buffer isn't full
+	 * yet, try to fill the temp buffer by decoding more data from the
+	 * next filter. Apply the BCJ filter on temp. Then we hopefully can
+	 * fill the actual output buffer by copying filtered data from temp.
+	 * A mix of filtered and unfiltered data may be left in temp; it will
+	 * be taken care on the next call to this function.
+	 */
+	if b.outPos < len(b.out) {
+		/* Make b.out temporarily point to s.temp. */
+		s.out = b.out
+		s.outPos = b.outPos
+		b.out = s.temp.bufArray[:]
+		b.outPos = len(s.temp.buf)
+		s.ret = chain(b)
+		s.temp.buf = s.temp.bufArray[:b.outPos]
+		b.out = s.out
+		b.outPos = s.outPos
+		if s.ret != xzOK && s.ret != xzStreamEnd {
+			return s.ret
+		}
+		bcjApply(s, s.temp.buf, &s.temp.filtered)
+		/*
+		 * If the next filter returned xzStreamEnd, we mark that
+		 * everything is filtered, since the last unfiltered bytes
+		 * of the stream are meant to be left as is.
+		 */
+		if s.ret == xzStreamEnd {
+			s.temp.filtered = len(s.temp.buf)
+		}
+		bcjFlush(s, b)
+		if s.temp.filtered > 0 {
+			return xzOK
+		}
+	}
+	return s.ret
+}
+
+/*
+ * Allocate memory for BCJ decoders. xzDecBCJReset must be used before
+ * calling xzDecBCJRun.
+ */
+func xzDecBCJCreate() *xzDecBCJ {
+	return new(xzDecBCJ)
+}
+
+/*
+ * Decode the Filter ID of a BCJ filter and check the start offset is
+ * valid. Returns xzOK if the given Filter ID and offset is
+ * supported. Otherwise xzOptionsError is returned.
+ */
+func xzDecBCJReset(s *xzDecBCJ, id xzFilterID, offset int) xzRet {
+	switch id {
+	case idBCJX86:
+	case idBCJPowerPC:
+	case idBCJIA64:
+	case idBCJARM:
+	case idBCJARMThumb:
+	case idBCJSPARC:
+	default:
+		/* Unsupported Filter ID */
+		return xzOptionsError
+	}
+	// check offset is a multiple of alignment
+	switch id {
+	case idBCJPowerPC, idBCJARM, idBCJSPARC:
+		if offset%4 != 0 {
+			return xzOptionsError
+		}
+	case idBCJIA64:
+		if offset%16 != 0 {
+			return xzOptionsError
+		}
+	case idBCJARMThumb:
+		if offset%2 != 0 {
+			return xzOptionsError
+		}
+	}
+	s.typ = id
+	s.ret = xzOK
+	s.pos = offset
+	s.x86PrevMask = 0
+	s.temp.filtered = 0
+	s.temp.buf = nil
+	return xzOK
+}

vendor/github.com/therootcompany/xz/dec_delta.go

@@ -0,0 +1,55 @@
+/*
+ * Delta decoder
+ *
+ * Author: Lasse Collin <lasse.collin@tukaani.org>
+ *
+ * Translation to Go: Michael Cross <https://github.com/xi2>
+ *
+ * This file has been put into the public domain.
+ * You can do whatever you want with this file.
+ */
+
+package xz
+
+type xzDecDelta struct {
+	delta    [256]byte
+	pos      byte
+	distance int // in range [1, 256]
+}
+
+/*
+ * Decode raw stream which has a delta filter as the first filter.
+ */
+func xzDecDeltaRun(s *xzDecDelta, b *xzBuf, chain func(*xzBuf) xzRet) xzRet {
+	outStart := b.outPos
+	ret := chain(b)
+	for i := outStart; i < b.outPos; i++ {
+		tmp := b.out[i] + s.delta[byte(s.distance+int(s.pos))]
+		s.delta[s.pos] = tmp
+		b.out[i] = tmp
+		s.pos--
+	}
+	return ret
+}
+
+/*
+ * Allocate memory for a delta decoder. xzDecDeltaReset must be used
+ * before calling xzDecDeltaRun.
+ */
+func xzDecDeltaCreate() *xzDecDelta {
+	return new(xzDecDelta)
+}
+
+/*
+ * Returns xzOK if the given distance is valid. Otherwise
+ * xzOptionsError is returned.
+ */
+func xzDecDeltaReset(s *xzDecDelta, distance int) xzRet {
+	if distance < 1 || distance > 256 {
+		return xzOptionsError
+	}
+	s.delta = [256]byte{}
+	s.pos = 0
+	s.distance = distance
+	return xzOK
+}

vendor/github.com/therootcompany/xz/dec_stream.go

@@ -0,0 +1,932 @@
+/*
+ * .xz Stream decoder
+ *
+ * Author: Lasse Collin <lasse.collin@tukaani.org>
+ *
+ * Translation to Go: Michael Cross <https://github.com/xi2>
+ *
+ * This file has been put into the public domain.
+ * You can do whatever you want with this file.
+ */
+
+package xz
+
+import (
+	"bytes"
+	"crypto/sha256"
+	"hash"
+	"hash/crc32"
+	"hash/crc64"
+)
+
+/* from linux/lib/xz/xz_stream.h **************************************/
+
+/*
+ * See the .xz file format specification at
+ * http://tukaani.org/xz/xz-file-format.txt
+ * to understand the container format.
+ */
+const (
+	streamHeaderSize = 12
+	headerMagic      = "\xfd7zXZ\x00"
+	footerMagic      = "YZ"
+)
+
+/*
+ * Variable-length integer can hold a 63-bit unsigned integer or a special
+ * value indicating that the value is unknown.
+ */
+type vliType uint64
+
+const (
+	vliUnknown vliType = ^vliType(0)
+	/* Maximum encoded size of a VLI */
+	vliBytesMax = 8 * 8 / 7 // (Sizeof(vliType) * 8 / 7)
+)
+
+/* from linux/lib/xz/xz_dec_stream.c **********************************/
+
+/* Hash used to validate the Index field */
+type xzDecHash struct {
+	unpadded     vliType
+	uncompressed vliType
+	sha256       hash.Hash
+}
+
+// type of xzDec.sequence
+type xzDecSeq int
+
+const (
+	seqStreamHeader xzDecSeq = iota
+	seqBlockStart
+	seqBlockHeader
+	seqBlockUncompress
+	seqBlockPadding
+	seqBlockCheck
+	seqIndex
+	seqIndexPadding
+	seqIndexCRC32
+	seqStreamFooter
+)
+
+// type of xzDec.index.sequence
+type xzDecIndexSeq int
+
+const (
+	seqIndexCount xzDecIndexSeq = iota
+	seqIndexUnpadded
+	seqIndexUncompressed
+)
+
+/**
+ * xzDec - Opaque type to hold the XZ decoder state
+ */
+type xzDec struct {
+	/* Position in decMain */
+	sequence xzDecSeq
+	/* Position in variable-length integers and Check fields */
+	pos int
+	/* Variable-length integer decoded by decVLI */
+	vli vliType
+	/* Saved inPos and outPos */
+	inStart  int
+	outStart int
+	/* CRC32 checksum hash used in Index */
+	crc32 hash.Hash
+	/* Hashes used in Blocks */
+	checkCRC32  hash.Hash
+	checkCRC64  hash.Hash
+	checkSHA256 hash.Hash
+	/* for checkTypes CRC32/CRC64/SHA256, check is one of the above 3 hashes */
+	check hash.Hash
+	/* Embedded stream header struct containing CheckType */
+	*Header
+	/*
+	 * True if the next call to xzDecRun is allowed to return
+	 * xzBufError.
+	 */
+	allowBufError bool
+	/* Information stored in Block Header */
+	blockHeader struct {
+		/*
+		 * Value stored in the Compressed Size field, or
+		 * vliUnknown if Compressed Size is not present.
+		 */
+		compressed vliType
+		/*
+		 * Value stored in the Uncompressed Size field, or
+		 * vliUnknown if Uncompressed Size is not present.
+		 */
+		uncompressed vliType
+		/* Size of the Block Header field */
+		size int
+	}
+	/* Information collected when decoding Blocks */
+	block struct {
+		/* Observed compressed size of the current Block */
+		compressed vliType
+		/* Observed uncompressed size of the current Block */
+		uncompressed vliType
+		/* Number of Blocks decoded so far */
+		count vliType
+		/*
+		 * Hash calculated from the Block sizes. This is used to
+		 * validate the Index field.
+		 */
+		hash xzDecHash
+	}
+	/* Variables needed when verifying the Index field */
+	index struct {
+		/* Position in decIndex */
+		sequence xzDecIndexSeq
+		/* Size of the Index in bytes */
+		size vliType
+		/* Number of Records (matches block.count in valid files) */
+		count vliType
+		/*
+		 * Hash calculated from the Records (matches block.hash in
+		 * valid files).
+		 */
+		hash xzDecHash
+	}
+	/*
+	 * Temporary buffer needed to hold Stream Header, Block Header,
+	 * and Stream Footer. The Block Header is the biggest (1 KiB)
+	 * so we reserve space according to that. bufArray has to be aligned
+	 * to a multiple of four bytes; the variables before it
+	 * should guarantee this.
+	 */
+	temp struct {
+		pos      int
+		buf      []byte // slice buf will be backed by bufArray
+		bufArray [1024]byte
+	}
+	// chain is the function (or to be more precise, closure) which
+	// does the decompression and will call into the lzma2 and other
+	// filter code as needed. It is constructed by decBlockHeader
+	chain func(b *xzBuf) xzRet
+	// lzma2 holds the state of the last filter (which must be LZMA2)
+	lzma2 *xzDecLZMA2
+	// pointers to allocated BCJ/Delta filters
+	bcjs   []*xzDecBCJ
+	deltas []*xzDecDelta
+	// number of currently in use BCJ/Delta filters from the above
+	bcjsUsed   int
+	deltasUsed int
+}
+
+/* Sizes of the Check field with different Check IDs */
+var checkSizes = [...]byte{
+	0,
+	4, 4, 4,
+	8, 8, 8,
+	16, 16, 16,
+	32, 32, 32,
+	64, 64, 64,
+}
+
+/*
+ * Fill s.temp by copying data starting from b.in[b.inPos]. Caller
+ * must have set s.temp.pos to indicate how much data we are supposed
+ * to copy into s.temp.buf. Return true once s.temp.pos has reached
+ * len(s.temp.buf).
+ */
+func fillTemp(s *xzDec, b *xzBuf) bool {
+	copySize := len(b.in) - b.inPos
+	tempRemaining := len(s.temp.buf) - s.temp.pos
+	if copySize > tempRemaining {
+		copySize = tempRemaining
+	}
+	copy(s.temp.buf[s.temp.pos:], b.in[b.inPos:])
+	b.inPos += copySize
+	s.temp.pos += copySize
+	if s.temp.pos == len(s.temp.buf) {
+		s.temp.pos = 0
+		return true
+	}
+	return false
+}
+
+/* Decode a variable-length integer (little-endian base-128 encoding) */
+func decVLI(s *xzDec, in []byte, inPos *int) xzRet {
+	var byte byte
+	if s.pos == 0 {
+		s.vli = 0
+	}
+	for *inPos < len(in) {
+		byte = in[*inPos]
+		*inPos++
+		s.vli |= vliType(byte&0x7f) << uint(s.pos)
+		if byte&0x80 == 0 {
+			/* Don't allow non-minimal encodings. */
+			if byte == 0 && s.pos != 0 {
+				return xzDataError
+			}
+			s.pos = 0
+			return xzStreamEnd
+		}
+		s.pos += 7
+		if s.pos == 7*vliBytesMax {
+			return xzDataError
+		}
+	}
+	return xzOK
+}
+
+/*
+ * Decode the Compressed Data field from a Block. Update and validate
+ * the observed compressed and uncompressed sizes of the Block so that
+ * they don't exceed the values possibly stored in the Block Header
+ * (validation assumes that no integer overflow occurs, since vliType
+ * is uint64). Update s.check if presence of the CRC32/CRC64/SHA256
+ * field was indicated in Stream Header.
+ *
+ * Once the decoding is finished, validate that the observed sizes match
+ * the sizes possibly stored in the Block Header. Update the hash and
+ * Block count, which are later used to validate the Index field.
+ */
+func decBlock(s *xzDec, b *xzBuf) xzRet {
+	var ret xzRet
+	s.inStart = b.inPos
+	s.outStart = b.outPos
+	ret = s.chain(b)
+	s.block.compressed += vliType(b.inPos - s.inStart)
+	s.block.uncompressed += vliType(b.outPos - s.outStart)
+	/*
+	 * There is no need to separately check for vliUnknown since
+	 * the observed sizes are always smaller than vliUnknown.
+	 */
+	if s.block.compressed > s.blockHeader.compressed ||
+		s.block.uncompressed > s.blockHeader.uncompressed {
+		return xzDataError
+	}
+	switch s.CheckType {
+	case CheckCRC32, CheckCRC64, CheckSHA256:
+		_, _ = s.check.Write(b.out[s.outStart:b.outPos])
+	}
+	if ret == xzStreamEnd {
+		if s.blockHeader.compressed != vliUnknown &&
+			s.blockHeader.compressed != s.block.compressed {
+			return xzDataError
+		}
+		if s.blockHeader.uncompressed != vliUnknown &&
+			s.blockHeader.uncompressed != s.block.uncompressed {
+			return xzDataError
+		}
+		s.block.hash.unpadded +=
+			vliType(s.blockHeader.size) + s.block.compressed
+		s.block.hash.unpadded += vliType(checkSizes[s.CheckType])
+		s.block.hash.uncompressed += s.block.uncompressed
+		var buf [2 * 8]byte // 2*Sizeof(vliType)
+		putLE64(uint64(s.block.hash.unpadded), buf[:])
+		putLE64(uint64(s.block.hash.uncompressed), buf[8:])
+		_, _ = s.block.hash.sha256.Write(buf[:])
+		s.block.count++
+	}
+	return ret
+}
+
+/* Update the Index size and the CRC32 hash. */
+func indexUpdate(s *xzDec, b *xzBuf) {
+	inUsed := b.inPos - s.inStart
+	s.index.size += vliType(inUsed)
+	_, _ = s.crc32.Write(b.in[s.inStart : s.inStart+inUsed])
+}
+
+/*
+ * Decode the Number of Records, Unpadded Size, and Uncompressed Size
+ * fields from the Index field. That is, Index Padding and CRC32 are not
+ * decoded by this function.
+ *
+ * This can return xzOK (more input needed), xzStreamEnd (everything
+ * successfully decoded), or xzDataError (input is corrupt).
+ */
+func decIndex(s *xzDec, b *xzBuf) xzRet {
+	var ret xzRet
+	for {
+		ret = decVLI(s, b.in, &b.inPos)
+		if ret != xzStreamEnd {
+			indexUpdate(s, b)
+			return ret
+		}
+		switch s.index.sequence {
+		case seqIndexCount:
+			s.index.count = s.vli
+			/*
+			 * Validate that the Number of Records field
+			 * indicates the same number of Records as
+			 * there were Blocks in the Stream.
+			 */
+			if s.index.count != s.block.count {
+				return xzDataError
+			}
+			s.index.sequence = seqIndexUnpadded
+		case seqIndexUnpadded:
+			s.index.hash.unpadded += s.vli
+			s.index.sequence = seqIndexUncompressed
+		case seqIndexUncompressed:
+			s.index.hash.uncompressed += s.vli
+			var buf [2 * 8]byte // 2*Sizeof(vliType)
+			putLE64(uint64(s.index.hash.unpadded), buf[:])
+			putLE64(uint64(s.index.hash.uncompressed), buf[8:])
+			_, _ = s.index.hash.sha256.Write(buf[:])
+			s.index.count--
+			s.index.sequence = seqIndexUnpadded
+		}
+		if !(s.index.count > 0) {
+			break
+		}
+	}
+	return xzStreamEnd
+}
+
+/*
+ * Validate that the next 4 bytes match s.crc32.Sum(nil). s.pos must
+ * be zero when starting to validate the first byte.
+ */
+func crcValidate(s *xzDec, b *xzBuf) xzRet {
+	sum := s.crc32.Sum(nil)
+	// CRC32 - reverse slice
+	sum[0], sum[1], sum[2], sum[3] = sum[3], sum[2], sum[1], sum[0]
+	for {
+		if b.inPos == len(b.in) {
+			return xzOK
+		}
+		if sum[s.pos] != b.in[b.inPos] {
+			return xzDataError
+		}
+		b.inPos++
+		s.pos++
+		if !(s.pos < 4) {
+			break
+		}
+	}
+	s.crc32.Reset()
+	s.pos = 0
+	return xzStreamEnd
+}
+
+/*
+ * Validate that the next 4/8/32 bytes match s.check.Sum(nil). s.pos
+ * must be zero when starting to validate the first byte.
+ */
+func checkValidate(s *xzDec, b *xzBuf) xzRet {
+	sum := s.check.Sum(nil)
+	if s.CheckType == CheckCRC32 || s.CheckType == CheckCRC64 {
+		// CRC32/64 - reverse slice
+		for i, j := 0, len(sum)-1; i < j; i, j = i+1, j-1 {
+			sum[i], sum[j] = sum[j], sum[i]
+		}
+	}
+	for {
+		if b.inPos == len(b.in) {
+			return xzOK
+		}
+		if sum[s.pos] != b.in[b.inPos] {
+			return xzDataError
+		}
+		b.inPos++
+		s.pos++
+		if !(s.pos < len(sum)) {
+			break
+		}
+	}
+	s.check.Reset()
+	s.pos = 0
+	return xzStreamEnd
+}
+
+/*
+ * Skip over the Check field when the Check ID is not supported.
+ * Returns true once the whole Check field has been skipped over.
+ */
+func checkSkip(s *xzDec, b *xzBuf) bool {
+	for s.pos < int(checkSizes[s.CheckType]) {
+		if b.inPos == len(b.in) {
+			return false
+		}
+		b.inPos++
+		s.pos++
+	}
+	s.pos = 0
+	return true
+}
+
+/* polynomial table used in decStreamHeader below */
+var xzCRC64Table = crc64.MakeTable(crc64.ECMA)
+
+/* Decode the Stream Header field (the first 12 bytes of the .xz Stream). */
+func decStreamHeader(s *xzDec) xzRet {
+	if string(s.temp.buf[:len(headerMagic)]) != headerMagic {
+		return xzFormatError
+	}
+	if crc32.ChecksumIEEE(s.temp.buf[len(headerMagic):len(headerMagic)+2]) !=
+		getLE32(s.temp.buf[len(headerMagic)+2:]) {
+		return xzDataError
+	}
+	if s.temp.buf[len(headerMagic)] != 0 {
+		return xzOptionsError
+	}
+	/*
+	 * Of integrity checks, we support none (Check ID = 0),
+	 * CRC32 (Check ID = 1), CRC64 (Check ID = 4) and SHA256 (Check ID = 10)
+	 * However, we will accept other check types too, but then the check
+	 * won't be verified and a warning (xzUnsupportedCheck) will be given.
+	 */
+	s.CheckType = CheckID(s.temp.buf[len(headerMagic)+1])
+	if s.CheckType > checkMax {
+		return xzOptionsError
+	}
+	switch s.CheckType {
+	case CheckNone:
+		// CheckNone: no action needed
+	case CheckCRC32:
+		if s.checkCRC32 == nil {
+			s.checkCRC32 = crc32.NewIEEE()
+		} else {
+			s.checkCRC32.Reset()
+		}
+		s.check = s.checkCRC32
+	case CheckCRC64:
+		if s.checkCRC64 == nil {
+			s.checkCRC64 = crc64.New(xzCRC64Table)
+		} else {
+			s.checkCRC64.Reset()
+		}
+		s.check = s.checkCRC64
+	case CheckSHA256:
+		if s.checkSHA256 == nil {
+			s.checkSHA256 = sha256.New()
+		} else {
+			s.checkSHA256.Reset()
+		}
+		s.check = s.checkSHA256
+	default:
+		return xzUnsupportedCheck
+	}
+	return xzOK
+}
+
+/* Decode the Stream Footer field (the last 12 bytes of the .xz Stream) */
+func decStreamFooter(s *xzDec) xzRet {
+	if string(s.temp.buf[10:10+len(footerMagic)]) != footerMagic {
+		return xzDataError
+	}
+	if crc32.ChecksumIEEE(s.temp.buf[4:10]) != getLE32(s.temp.buf) {
+		return xzDataError
+	}
+	/*
+	 * Validate Backward Size. Note that we never added the size of the
+	 * Index CRC32 field to s->index.size, thus we use s->index.size / 4
+	 * instead of s->index.size / 4 - 1.
+	 */
+	if s.index.size>>2 != vliType(getLE32(s.temp.buf[4:])) {
+		return xzDataError
+	}
+	if s.temp.buf[8] != 0 || CheckID(s.temp.buf[9]) != s.CheckType {
+		return xzDataError
+	}
+	/*
+	 * Use xzStreamEnd instead of xzOK to be more convenient
+	 * for the caller.
+	 */
+	return xzStreamEnd
+}
+
+/* Decode the Block Header and initialize the filter chain. */
+func decBlockHeader(s *xzDec) xzRet {
+	var ret xzRet
+	/*
+	 * Validate the CRC32. We know that the temp buffer is at least
+	 * eight bytes so this is safe.
+	 */
+	crc := getLE32(s.temp.buf[len(s.temp.buf)-4:])
+	s.temp.buf = s.temp.buf[:len(s.temp.buf)-4]
+	if crc32.ChecksumIEEE(s.temp.buf) != crc {
+		return xzDataError
+	}
+	s.temp.pos = 2
+	/*
+	 * Catch unsupported Block Flags.
+	 */
+	if s.temp.buf[1]&0x3C != 0 {
+		return xzOptionsError
+	}
+	/* Compressed Size */
+	if s.temp.buf[1]&0x40 != 0 {
+		if decVLI(s, s.temp.buf, &s.temp.pos) != xzStreamEnd {
+			return xzDataError
+		}
+		if s.vli >= 1<<63-8 {
+			// the whole block must stay smaller than 2^63 bytes
+			// the block header cannot be smaller than 8 bytes
+			return xzDataError
+		}
+		if s.vli == 0 {
+			// compressed size must be non-zero
+			return xzDataError
+		}
+		s.blockHeader.compressed = s.vli
+	} else {
+		s.blockHeader.compressed = vliUnknown
+	}
+	/* Uncompressed Size */
+	if s.temp.buf[1]&0x80 != 0 {
+		if decVLI(s, s.temp.buf, &s.temp.pos) != xzStreamEnd {
+			return xzDataError
+		}
+		s.blockHeader.uncompressed = s.vli
+	} else {
+		s.blockHeader.uncompressed = vliUnknown
+	}
+	// get total number of filters (1-4)
+	filterTotal := int(s.temp.buf[1]&0x03) + 1
+	// slice to hold decoded filters
+	filterList := make([]struct {
+		id    xzFilterID
+		props uint32
+	}, filterTotal)
+	// decode the non-last filters which cannot be LZMA2
+	for i := 0; i < filterTotal-1; i++ {
+		/* Valid Filter Flags always take at least two bytes. */
+		if len(s.temp.buf)-s.temp.pos < 2 {
+			return xzDataError
+		}
+		s.temp.pos += 2
+		switch id := xzFilterID(s.temp.buf[s.temp.pos-2]); id {
+		case idDelta:
+			// delta filter
+			if s.temp.buf[s.temp.pos-1] != 0x01 {
+				return xzOptionsError
+			}
+			/* Filter Properties contains distance - 1 */
+			if len(s.temp.buf)-s.temp.pos < 1 {
+				return xzDataError
+			}
+			props := uint32(s.temp.buf[s.temp.pos])
+			s.temp.pos++
+			filterList[i] = struct {
+				id    xzFilterID
+				props uint32
+			}{id: id, props: props}
+		case idBCJX86, idBCJPowerPC, idBCJIA64,
+			idBCJARM, idBCJARMThumb, idBCJSPARC:
+			// bcj filter
+			var props uint32
+			switch s.temp.buf[s.temp.pos-1] {
+			case 0x00:
+				props = 0
+			case 0x04:
+				if len(s.temp.buf)-s.temp.pos < 4 {
+					return xzDataError
+				}
+				props = getLE32(s.temp.buf[s.temp.pos:])
+				s.temp.pos += 4
+			default:
+				return xzOptionsError
+			}
+			filterList[i] = struct {
+				id    xzFilterID
+				props uint32
+			}{id: id, props: props}
+		default:
+			return xzOptionsError
+		}
+	}
+	/*
+	 * decode the last filter which must be LZMA2
+	 */
+	if len(s.temp.buf)-s.temp.pos < 2 {
+		return xzDataError
+	}
+	/* Filter ID = LZMA2 */
+	if xzFilterID(s.temp.buf[s.temp.pos]) != idLZMA2 {
+		return xzOptionsError
+	}
+	s.temp.pos++
+	/* Size of Properties = 1-byte Filter Properties */
+	if s.temp.buf[s.temp.pos] != 0x01 {
+		return xzOptionsError
+	}
+	s.temp.pos++
+	/* Filter Properties contains LZMA2 dictionary size. */
+	if len(s.temp.buf)-s.temp.pos < 1 {
+		return xzDataError
+	}
+	props := uint32(s.temp.buf[s.temp.pos])
+	s.temp.pos++
+	filterList[filterTotal-1] = struct {
+		id    xzFilterID
+		props uint32
+	}{id: idLZMA2, props: props}
+	/*
+	 * Process the filter list and create s.chain, going from last
+	 * filter (LZMA2) to first filter
+	 *
+	 * First, LZMA2.
+	 */
+	ret = xzDecLZMA2Reset(s.lzma2, byte(filterList[filterTotal-1].props))
+	if ret != xzOK {
+		return ret
+	}
+	s.chain = func(b *xzBuf) xzRet {
+		return xzDecLZMA2Run(s.lzma2, b)
+	}
+	/*
+	 * Now the non-last filters
+	 */
+	for i := filterTotal - 2; i >= 0; i-- {
+		switch id := filterList[i].id; id {
+		case idDelta:
+			// delta filter
+			var delta *xzDecDelta
+			if s.deltasUsed < len(s.deltas) {
+				delta = s.deltas[s.deltasUsed]
+			} else {
+				delta = xzDecDeltaCreate()
+				s.deltas = append(s.deltas, delta)
+			}
+			s.deltasUsed++
+			ret = xzDecDeltaReset(delta, int(filterList[i].props)+1)
+			if ret != xzOK {
+				return ret
+			}
+			chain := s.chain
+			s.chain = func(b *xzBuf) xzRet {
+				return xzDecDeltaRun(delta, b, chain)
+			}
+		case idBCJX86, idBCJPowerPC, idBCJIA64,
+			idBCJARM, idBCJARMThumb, idBCJSPARC:
+			// bcj filter
+			var bcj *xzDecBCJ
+			if s.bcjsUsed < len(s.bcjs) {
+				bcj = s.bcjs[s.bcjsUsed]
+			} else {
+				bcj = xzDecBCJCreate()
+				s.bcjs = append(s.bcjs, bcj)
+			}
+			s.bcjsUsed++
+			ret = xzDecBCJReset(bcj, id, int(filterList[i].props))
+			if ret != xzOK {
+				return ret
+			}
+			chain := s.chain
+			s.chain = func(b *xzBuf) xzRet {
+				return xzDecBCJRun(bcj, b, chain)
+			}
+		}
+	}
+	/* The rest must be Header Padding. */
+	for s.temp.pos < len(s.temp.buf) {
+		if s.temp.buf[s.temp.pos] != 0x00 {
+			return xzOptionsError
+		}
+		s.temp.pos++
+	}
+	s.temp.pos = 0
+	s.block.compressed = 0
+	s.block.uncompressed = 0
+	return xzOK
+}
+
+func decMain(s *xzDec, b *xzBuf) xzRet {
+	var ret xzRet
+	/*
+	 * Store the start position for the case when we are in the middle
+	 * of the Index field.
+	 */
+	s.inStart = b.inPos
+	for {
+		switch s.sequence {
+		case seqStreamHeader:
+			/*
+			 * Stream Header is copied to s.temp, and then
+			 * decoded from there. This way if the caller
+			 * gives us only little input at a time, we can
+			 * still keep the Stream Header decoding code
+			 * simple. Similar approach is used in many places
+			 * in this file.
+			 */
+			if !fillTemp(s, b) {
+				return xzOK
+			}
+			/*
+			 * If decStreamHeader returns
+			 * xzUnsupportedCheck, it is still possible
+			 * to continue decoding. Thus, update s.sequence
+			 * before calling decStreamHeader.
+			 */
+			s.sequence = seqBlockStart
+			ret = decStreamHeader(s)
+			if ret != xzOK {
+				return ret
+			}
+			fallthrough
+		case seqBlockStart:
+			/* We need one byte of input to continue. */
+			if b.inPos == len(b.in) {
+				return xzOK
+			}
+			/* See if this is the beginning of the Index field. */
+			if b.in[b.inPos] == 0 {
+				s.inStart = b.inPos
+				b.inPos++
+				s.sequence = seqIndex
+				break
+			}
+			/*
+			 * Calculate the size of the Block Header and
+			 * prepare to decode it.
+			 */
+			s.blockHeader.size = (int(b.in[b.inPos]) + 1) * 4
+			s.temp.buf = s.temp.bufArray[:s.blockHeader.size]
+			s.temp.pos = 0
+			s.sequence = seqBlockHeader
+			fallthrough
+		case seqBlockHeader:
+			if !fillTemp(s, b) {
+				return xzOK
+			}
+			ret = decBlockHeader(s)
+			if ret != xzOK {
+				return ret
+			}
+			s.sequence = seqBlockUncompress
+			fallthrough
+		case seqBlockUncompress:
+			ret = decBlock(s, b)
+			if ret != xzStreamEnd {
+				return ret
+			}
+			s.sequence = seqBlockPadding
+			fallthrough
+		case seqBlockPadding:
+			/*
+			 * Size of Compressed Data + Block Padding
+			 * must be a multiple of four. We don't need
+			 * s->block.compressed for anything else
+			 * anymore, so we use it here to test the size
+			 * of the Block Padding field.
+			 */
+			for s.block.compressed&3 != 0 {
+				if b.inPos == len(b.in) {
+					return xzOK
+				}
+				if b.in[b.inPos] != 0 {
+					return xzDataError
+				}
+				b.inPos++
+				s.block.compressed++
+			}
+			s.sequence = seqBlockCheck
+			fallthrough
+		case seqBlockCheck:
+			switch s.CheckType {
+			case CheckCRC32, CheckCRC64, CheckSHA256:
+				ret = checkValidate(s, b)
+				if ret != xzStreamEnd {
+					return ret
+				}
+			default:
+				if !checkSkip(s, b) {
+					return xzOK
+				}
+			}
+			s.sequence = seqBlockStart
+		case seqIndex:
+			ret = decIndex(s, b)
+			if ret != xzStreamEnd {
+				return ret
+			}
+			s.sequence = seqIndexPadding
+			fallthrough
+		case seqIndexPadding:
+			for (s.index.size+vliType(b.inPos-s.inStart))&3 != 0 {
+				if b.inPos == len(b.in) {
+					indexUpdate(s, b)
+					return xzOK
+				}
+				if b.in[b.inPos] != 0 {
+					return xzDataError
+				}
+				b.inPos++
+			}
+			/* Finish the CRC32 value and Index size. */
+			indexUpdate(s, b)
+			/* Compare the hashes to validate the Index field. */
+			if !bytes.Equal(
+				s.block.hash.sha256.Sum(nil), s.index.hash.sha256.Sum(nil)) {
+				return xzDataError
+			}
+			s.sequence = seqIndexCRC32
+			fallthrough
+		case seqIndexCRC32:
+			ret = crcValidate(s, b)
+			if ret != xzStreamEnd {
+				return ret
+			}
+			s.temp.buf = s.temp.bufArray[:streamHeaderSize]
+			s.sequence = seqStreamFooter
+			fallthrough
+		case seqStreamFooter:
+			if !fillTemp(s, b) {
+				return xzOK
+			}
+			return decStreamFooter(s)
+		}
+	}
+	/* Never reached */
+}
+
+/**
+ * xzDecRun - Run the XZ decoder
+ * @s:         Decoder state allocated using xzDecInit
+ * @b:         Input and output buffers
+ *
+ * See xzRet for details of return values.
+ *
+ * xzDecRun is a wrapper for decMain to handle some special cases.
+ *
+ * We must return xzBufError when it seems clear that we are not
+ * going to make any progress anymore. This is to prevent the caller
+ * from calling us infinitely when the input file is truncated or
+ * otherwise corrupt. Since zlib-style API allows that the caller
+ * fills the input buffer only when the decoder doesn't produce any
+ * new output, we have to be careful to avoid returning xzBufError
+ * too easily: xzBufError is returned only after the second
+ * consecutive call to xzDecRun that makes no progress.
+ */
+func xzDecRun(s *xzDec, b *xzBuf) xzRet {
+	inStart := b.inPos
+	outStart := b.outPos
+	ret := decMain(s, b)
+	if ret == xzOK && inStart == b.inPos && outStart == b.outPos {
+		if s.allowBufError {
+			ret = xzBufError
+		}
+		s.allowBufError = true
+	} else {
+		s.allowBufError = false
+	}
+	return ret
+}
+
+/**
+ * xzDecInit - Allocate and initialize a XZ decoder state
+ * @dictMax:    Maximum size of the LZMA2 dictionary (history buffer) for
+ *              decoding. LZMA2 dictionary is always 2^n bytes
+ *              or 2^n + 2^(n-1) bytes (the latter sizes are less common
+ *              in practice), so other values for dictMax don't make sense.
+ *
+ * dictMax specifies the maximum allowed dictionary size that xzDecRun
+ * may allocate once it has parsed the dictionary size from the stream
+ * headers. This way excessive allocations can be avoided while still
+ * limiting the maximum memory usage to a sane value to prevent running the
+ * system out of memory when decompressing streams from untrusted sources.
+ *
+ * xzDecInit returns a pointer to an xzDec, which is ready to be used with
+ * xzDecRun.
+ */
+func xzDecInit(dictMax uint32, header *Header) *xzDec {
+	s := new(xzDec)
+	s.crc32 = crc32.NewIEEE()
+	s.Header = header
+	s.block.hash.sha256 = sha256.New()
+	s.index.hash.sha256 = sha256.New()
+	s.lzma2 = xzDecLZMA2Create(dictMax)
+	xzDecReset(s)
+	return s
+}
+
+/**
+ * xzDecReset - Reset an already allocated decoder state
+ * @s:          Decoder state allocated using xzDecInit
+ *
+ * This function can be used to reset the decoder state without
+ * reallocating memory with xzDecInit.
+ */
+func xzDecReset(s *xzDec) {
+	s.sequence = seqStreamHeader
+	s.allowBufError = false
+	s.pos = 0
+	s.crc32.Reset()
+	s.check = nil
+	s.CheckType = checkUnset
+	s.block.compressed = 0
+	s.block.uncompressed = 0
+	s.block.count = 0
+	s.block.hash.unpadded = 0
+	s.block.hash.uncompressed = 0
+	s.block.hash.sha256.Reset()
+	s.index.sequence = seqIndexCount
+	s.index.size = 0
+	s.index.count = 0
+	s.index.hash.unpadded = 0
+	s.index.hash.uncompressed = 0
+	s.index.hash.sha256.Reset()
+	s.temp.pos = 0
+	s.temp.buf = s.temp.bufArray[:streamHeaderSize]
+	s.chain = nil
+	s.bcjsUsed = 0
+	s.deltasUsed = 0
+}

vendor/github.com/therootcompany/xz/dec_util.go

@@ -0,0 +1,52 @@
+/*
+ * XZ decompressor utility functions
+ *
+ * Author: Michael Cross <https://github.com/xi2>
+ *
+ * This file has been put into the public domain.
+ * You can do whatever you want with this file.
+ */
+
+package xz
+
+func getLE32(buf []byte) uint32 {
+	return uint32(buf[0]) |
+		uint32(buf[1])<<8 |
+		uint32(buf[2])<<16 |
+		uint32(buf[3])<<24
+}
+
+func getBE32(buf []byte) uint32 {
+	return uint32(buf[0])<<24 |
+		uint32(buf[1])<<16 |
+		uint32(buf[2])<<8 |
+		uint32(buf[3])
+}
+
+func putLE32(val uint32, buf []byte) {
+	buf[0] = byte(val)
+	buf[1] = byte(val >> 8)
+	buf[2] = byte(val >> 16)
+	buf[3] = byte(val >> 24)
+	return
+}
+
+func putBE32(val uint32, buf []byte) {
+	buf[0] = byte(val >> 24)
+	buf[1] = byte(val >> 16)
+	buf[2] = byte(val >> 8)
+	buf[3] = byte(val)
+	return
+}
+
+func putLE64(val uint64, buf []byte) {
+	buf[0] = byte(val)
+	buf[1] = byte(val >> 8)
+	buf[2] = byte(val >> 16)
+	buf[3] = byte(val >> 24)
+	buf[4] = byte(val >> 32)
+	buf[5] = byte(val >> 40)
+	buf[6] = byte(val >> 48)
+	buf[7] = byte(val >> 56)
+	return
+}

vendor/github.com/therootcompany/xz/dec_xz.go

@@ -0,0 +1,124 @@
+/*
+ * XZ decompressor
+ *
+ * Authors: Lasse Collin <lasse.collin@tukaani.org>
+ *          Igor Pavlov <http://7-zip.org/>
+ *
+ * Translation to Go: Michael Cross <https://github.com/xi2>
+ *
+ * This file has been put into the public domain.
+ * You can do whatever you want with this file.
+ */
+
+package xz
+
+/* from linux/include/linux/xz.h **************************************/
+
+/**
+ * xzRet - Return codes
+ * @xzOK:                   Everything is OK so far. More input or more
+ *                          output space is required to continue.
+ * @xzStreamEnd:            Operation finished successfully.
+ * @xzUnSupportedCheck:     Integrity check type is not supported. Decoding
+ *                          is still possible by simply calling xzDecRun
+ *                          again.
+ * @xzMemlimitError:        A bigger LZMA2 dictionary would be needed than
+ *                          allowed by the dictMax argument given to
+ *                          xzDecInit.
+ * @xzFormatError:          File format was not recognized (wrong magic
+ *                          bytes).
+ * @xzOptionsError:         This implementation doesn't support the requested
+ *                          compression options. In the decoder this means
+ *                          that the header CRC32 matches, but the header
+ *                          itself specifies something that we don't support.
+ * @xzDataError:            Compressed data is corrupt.
+ * @xzBufError:             Cannot make any progress.
+ *
+ * xzBufError is returned when two consecutive calls to XZ code cannot
+ * consume any input and cannot produce any new output.  This happens
+ * when there is no new input available, or the output buffer is full
+ * while at least one output byte is still pending. Assuming your code
+ * is not buggy, you can get this error only when decoding a
+ * compressed stream that is truncated or otherwise corrupt.
+ */
+type xzRet int
+
+const (
+	xzOK xzRet = iota
+	xzStreamEnd
+	xzUnsupportedCheck
+	xzMemlimitError
+	xzFormatError
+	xzOptionsError
+	xzDataError
+	xzBufError
+)
+
+/**
+ * xzBuf - Passing input and output buffers to XZ code
+ * @in:         Input buffer.
+ * @inPos:      Current position in the input buffer. This must not exceed
+ *              input buffer size.
+ * @out:        Output buffer.
+ * @outPos:     Current position in the output buffer. This must not exceed
+ *              output buffer size.
+ *
+ * Only the contents of the output buffer from out[outPos] onward, and
+ * the variables inPos and outPos are modified by the XZ code.
+ */
+type xzBuf struct {
+	in     []byte
+	inPos  int
+	out    []byte
+	outPos int
+}
+
+/* All XZ filter IDs */
+type xzFilterID int64
+
+const (
+	idDelta       xzFilterID = 0x03
+	idBCJX86      xzFilterID = 0x04
+	idBCJPowerPC  xzFilterID = 0x05
+	idBCJIA64     xzFilterID = 0x06
+	idBCJARM      xzFilterID = 0x07
+	idBCJARMThumb xzFilterID = 0x08
+	idBCJSPARC    xzFilterID = 0x09
+	idLZMA2       xzFilterID = 0x21
+)
+
+// CheckID is the type of the data integrity check in an XZ stream
+// calculated from the uncompressed data.
+type CheckID int
+
+func (id CheckID) String() string {
+	switch id {
+	case CheckNone:
+		return "None"
+	case CheckCRC32:
+		return "CRC32"
+	case CheckCRC64:
+		return "CRC64"
+	case CheckSHA256:
+		return "SHA256"
+	default:
+		return "Unknown"
+	}
+}
+
+const (
+	CheckNone   CheckID = 0x00
+	CheckCRC32  CheckID = 0x01
+	CheckCRC64  CheckID = 0x04
+	CheckSHA256 CheckID = 0x0A
+	checkMax    CheckID = 0x0F
+	checkUnset  CheckID = -1
+)
+
+// An XZ stream contains a stream header which holds information about
+// the stream. That information is exposed as fields of the
+// Reader. Currently it contains only the stream's data integrity
+// check type.
+type Header struct {
+	CheckType CheckID // type of the stream's data integrity check
+}

vendor/github.com/therootcompany/xz/doc.go

@@ -0,0 +1,35 @@
+// Package xz implements XZ decompression natively in Go.
+//
+// Usage
+//
+// For ease of use, this package is designed to have a similar API to
+// compress/gzip. See the examples for further details.
+//
+// Implementation
+//
+// This package is a translation from C to Go of XZ Embedded
+// (http://tukaani.org/xz/embedded.html) with enhancements made so as
+// to implement all mandatory and optional parts of the XZ file format
+// specification v1.0.4. It supports all filters and block check
+// types, supports multiple streams, and performs index verification
+// using SHA-256 as recommended by the specification.
+//
+// Speed
+//
+// On the author's Intel Ivybridge i5, decompression speed is about
+// half that of the standard XZ Utils (tested with a recent linux
+// kernel tarball).
+//
+// Thanks
+//
+// Thanks are due to Lasse Collin and Igor Pavlov, the authors of XZ
+// Embedded, on whose code package xz is based. It would not exist
+// without their decision to allow others to modify and reuse their
+// code.
+//
+// Bug reports
+//
+// For bug reports relating to this package please contact the author
+// through https://github.com/xi2/xz/issues, and not the authors of XZ
+// Embedded.
+package xz

vendor/github.com/therootcompany/xz/reader.go

@@ -0,0 +1,256 @@
+/*
+ * Package xz Go Reader API
+ *
+ * Author: Michael Cross <https://github.com/xi2>
+ *
+ * This file has been put into the public domain.
+ * You can do whatever you want with this file.
+ */
+
+package xz
+
+import (
+	"errors"
+	"io"
+)
+
+// Package specific errors.
+var (
+	ErrUnsupportedCheck = errors.New("xz: integrity check type not supported")
+	ErrMemlimit         = errors.New("xz: LZMA2 dictionary size exceeds max")
+	ErrFormat           = errors.New("xz: file format not recognized")
+	ErrOptions          = errors.New("xz: compression options not supported")
+	ErrData             = errors.New("xz: data is corrupt")
+	ErrBuf              = errors.New("xz: data is truncated or corrupt")
+)
+
+// DefaultDictMax is the default maximum dictionary size in bytes used
+// by the decoder. This value is sufficient to decompress files
+// created with XZ Utils "xz -9".
+const DefaultDictMax = 1 << 26 // 64 MiB
+
+// inBufSize is the input buffer size used by the decoder.
+const inBufSize = 1 << 13 // 8 KiB
+
+// A Reader is an io.Reader that can be used to retrieve uncompressed
+// data from an XZ file.
+//
+// In general, an XZ file can be a concatenation of other XZ
+// files. Reads from the Reader return the concatenation of the
+// uncompressed data of each.
+type Reader struct {
+	Header
+	r           io.Reader       // the wrapped io.Reader
+	multistream bool            // true if reader is in multistream mode
+	rEOF        bool            // true after io.EOF received on r
+	dEOF        bool            // true after decoder has completed
+	padding     int             // bytes of stream padding read (or -1)
+	in          [inBufSize]byte // backing array for buf.in
+	buf         *xzBuf          // decoder input/output buffers
+	dec         *xzDec          // decoder state
+	err         error           // the result of the last decoder call
+}
+
+// NewReader creates a new Reader reading from r. The decompressor
+// will use an LZMA2 dictionary size up to dictMax bytes in
+// size. Passing a value of zero sets dictMax to DefaultDictMax.  If
+// an individual XZ stream requires a dictionary size greater than
+// dictMax in order to decompress, Read will return ErrMemlimit.
+//
+// If NewReader is passed a value of nil for r then a Reader is
+// created such that all read attempts will return io.EOF. This is
+// useful if you just want to allocate memory for a Reader which will
+// later be initialized with Reset.
+//
+// Due to internal buffering, the Reader may read more data than
+// necessary from r.
+func NewReader(r io.Reader, dictMax uint32) (*Reader, error) {
+	if dictMax == 0 {
+		dictMax = DefaultDictMax
+	}
+	z := &Reader{
+		r:           r,
+		multistream: true,
+		padding:     -1,
+		buf:         &xzBuf{},
+	}
+	if r == nil {
+		z.rEOF, z.dEOF = true, true
+	}
+	z.dec = xzDecInit(dictMax, &z.Header)
+	var err error
+	if r != nil {
+		_, err = z.Read(nil) // read stream header
+	}
+	return z, err
+}
+
+// decode is a wrapper around xzDecRun that additionally handles
+// stream padding. It treats the padding as a kind of stream that
+// decodes to nothing.
+//
+// When decoding padding, z.padding >= 0
+// When decoding a real stream, z.padding == -1
+func (z *Reader) decode() (ret xzRet) {
+	if z.padding >= 0 {
+		// read all padding in input buffer
+		for z.buf.inPos < len(z.buf.in) &&
+			z.buf.in[z.buf.inPos] == 0 {
+			z.buf.inPos++
+			z.padding++
+		}
+		switch {
+		case z.buf.inPos == len(z.buf.in) && z.rEOF:
+			// case: out of padding. no more input data available
+			if z.padding%4 != 0 {
+				ret = xzDataError
+			} else {
+				ret = xzStreamEnd
+			}
+		case z.buf.inPos == len(z.buf.in):
+			// case: read more padding next loop iteration
+			ret = xzOK
+		default:
+			// case: out of padding. more input data available
+			if z.padding%4 != 0 {
+				ret = xzDataError
+			} else {
+				xzDecReset(z.dec)
+				ret = xzStreamEnd
+			}
+		}
+	} else {
+		ret = xzDecRun(z.dec, z.buf)
+	}
+	return
+}
+
+func (z *Reader) Read(p []byte) (n int, err error) {
+	// restore err
+	err = z.err
+	// set decoder output buffer to p
+	z.buf.out = p
+	z.buf.outPos = 0
+	for {
+		// update n
+		n = z.buf.outPos
+		// if last call to decoder ended with an error, return that error
+		if err != nil {
+			break
+		}
+		// if decoder has finished, return with err == io.EOF
+		if z.dEOF {
+			err = io.EOF
+			break
+		}
+		// if p full, return with err == nil, unless we have not yet
+		// read the stream header with Read(nil)
+		if n == len(p) && z.CheckType != checkUnset {
+			break
+		}
+		// if needed, read more data from z.r
+		if z.buf.inPos == len(z.buf.in) && !z.rEOF {
+			rn, e := z.r.Read(z.in[:])
+			if e != nil && e != io.EOF {
+				// read error
+				err = e
+				break
+			}
+			if e == io.EOF {
+				z.rEOF = true
+			}
+			// set new input buffer in z.buf
+			z.buf.in = z.in[:rn]
+			z.buf.inPos = 0
+		}
+		// decode more data
+		ret := z.decode()
+		switch ret {
+		case xzOK:
+			// no action needed
+		case xzStreamEnd:
+			if z.padding >= 0 {
+				z.padding = -1
+				if !z.multistream || z.rEOF {
+					z.dEOF = true
+				}
+			} else {
+				z.padding = 0
+			}
+		case xzUnsupportedCheck:
+			err = ErrUnsupportedCheck
+		case xzMemlimitError:
+			err = ErrMemlimit
+		case xzFormatError:
+			err = ErrFormat
+		case xzOptionsError:
+			err = ErrOptions
+		case xzDataError:
+			err = ErrData
+		case xzBufError:
+			err = ErrBuf
+		}
+		// save err
+		z.err = err
+	}
+	return
+}
+
+// Multistream controls whether the reader is operating in multistream
+// mode.
+//
+// If enabled (the default), the Reader expects the input to be a
+// sequence of XZ streams, possibly interspersed with stream padding,
+// which it reads one after another. The effect is that the
+// concatenation of a sequence of XZ streams or XZ files is
+// treated as equivalent to the compressed result of the concatenation
+// of the sequence. This is standard behaviour for XZ readers.
+//
+// Calling Multistream(false) disables this behaviour; disabling the
+// behaviour can be useful when reading file formats that distinguish
+// individual XZ streams. In this mode, when the Reader reaches the
+// end of the stream, Read returns io.EOF. To start the next stream,
+// call z.Reset(nil) followed by z.Multistream(false). If there is no
+// next stream, z.Reset(nil) will return io.EOF.
+func (z *Reader) Multistream(ok bool) {
+	z.multistream = ok
+}
+
+// Reset, for non-nil values of io.Reader r, discards the Reader z's
+// state and makes it equivalent to the result of its original state
+// from NewReader, but reading from r instead. This permits reusing a
+// Reader rather than allocating a new one.
+//
+// If you wish to leave r unchanged use z.Reset(nil). This keeps r
+// unchanged and ensures internal buffering is preserved. If the
+// Reader was at the end of a stream it is then ready to read any
+// follow on streams. If there are no follow on streams z.Reset(nil)
+// returns io.EOF. If the Reader was not at the end of a stream then
+// z.Reset(nil) does nothing.
+func (z *Reader) Reset(r io.Reader) error {
+	switch {
+	case r == nil:
+		z.multistream = true
+		if !z.dEOF {
+			return nil
+		}
+		if z.rEOF {
+			return io.EOF
+		}
+		z.dEOF = false
+		_, err := z.Read(nil) // read stream header
+		return err
+	default:
+		z.r = r
+		z.multistream = true
+		z.rEOF = false
+		z.dEOF = false
+		z.padding = -1
+		z.buf.in = nil
+		z.buf.inPos = 0
+		xzDecReset(z.dec)
+		z.err = nil
+		_, err := z.Read(nil) // read stream header
+		return err
+	}
+}

vendor/modules.txt

@@ -83,6 +83,9 @@ github.com/nats-io/nuid
 github.com/novox/mesh-host/internal/declaration
 github.com/novox/mesh-host/rootsearch
 github.com/novox/mesh-host/validate
+# github.com/therootcompany/xz v1.0.1
+## explicit; go 1.15
+github.com/therootcompany/xz
 # golang.org/x/crypto v0.57.0
 ## explicit; go 1.26.0
 golang.org/x/crypto/acme