From 10d4712f821d4b73ee161249fe1b4d81424eb694 Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 12 Oct 2026 12:46:36 +0200 Subject: [PATCH 01/26] Name who a read-write access grants its path's group to, so modules write shared data through the group the operator gave it (hq ADR 0312, issue 537) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Each read-write access is composed with a grant: the module's containers that mount its path and the account its tools run as (the operator's, or the module's own under ADR 0259 §8); that runtime restarts on the access, so it holds a group the engine put its account in. A read access grants nothing. Sent only to an engine whose statement says GrantContract (4), since an older one refuses the field. Pins the node-engine at the branch that reads it. --- cmd/mesh-controller/plan.go | 15 ++ go.mod | 2 +- go.sum | 4 +- internal/catalogue/declaration.go | 22 +++ internal/catalogue/grants.go | 131 +++++++++++++++ internal/catalogue/grants_test.go | 154 ++++++++++++++++++ internal/link/protocol.go | 11 ++ .../internal/declaration/declaration.go | 72 ++++++++ vendor/modules.txt | 4 +- 9 files changed, 410 insertions(+), 5 deletions(-) create mode 100644 internal/catalogue/grants.go create mode 100644 internal/catalogue/grants_test.go diff --git a/cmd/mesh-controller/plan.go b/cmd/mesh-controller/plan.go index b4811f4e..958993dc 100644 --- a/cmd/mesh-controller/plan.go +++ b/cmd/mesh-controller/plan.go @@ -918,9 +918,14 @@ func renderingFor(ctx context.Context, open *stores, node string, if err != nil { return catalogue.Rendering{}, inventory.Node{}, err } + grantsGroups, err := engineGrantsGroups(ctx, inv, node) + if err != nil { + return catalogue.Rendering{}, inventory.Node{}, err + } return catalogue.Rendering{ ReadsHealth: readsHealth, JudgesRoot: judgesRoot, + GrantsGroups: grantsGroups, BusMembership: memberships[node], Settings: settings, Generators: gens, Grants: grants, Needed: needed, Foreseen: foreseen, Ports: ports, Certificate: certificate, Authority: authority, Mesh: private, Names: names, @@ -953,6 +958,16 @@ func engineJudgesRoot(ctx context.Context, inv *inventory.Inventory, node string return had && stated.Contract >= link.RootContract, nil } +// engineGrantsGroups says whether a machine's node-engine reads an access's `grant` (novox/hq ADR 0312), by its own +// newest statement, for the same reason as engineReadsHealth: an older engine parses strictly. +func engineGrantsGroups(ctx context.Context, inv *inventory.Inventory, node string) (bool, error) { + stated, had, err := inv.HealthOf(ctx, node) + if err != nil { + return false, err + } + return had && stated.Contract >= link.GrantContract, nil +} + // zonesInTheMesh is every zone a module in the mesh declares, where the mesh placed it (novox/hq ADR // 0199): the zone settled from that node's settings, the node's private address, the port the // answering listen is published on there. diff --git a/go.mod b/go.mod index cdf300d5..64c2a70c 100644 --- a/go.mod +++ b/go.mod @@ -35,4 +35,4 @@ require ( // committed. Every build (the build agent's `go build`, the Dockerfile) compiles from vendor/ and // fetches nothing; go refuses to build when vendor/ and this file disagree, so a pin moved without // `go mod vendor` fails loudly, at once, everywhere. -replace github.com/novox/mesh-host => git.novox.be/novox/mesh-host v0.0.0-20261011092230-20d5af9b2d5f +replace github.com/novox/mesh-host => git.novox.be/novox/mesh-host v0.0.0-20261012102338-f4efc3cad0a6 diff --git a/go.sum b/go.sum index e7436cbb..d2d2ed69 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,5 @@ -git.novox.be/novox/mesh-host v0.0.0-20261011092230-20d5af9b2d5f h1:8N5OW2mdTNIck2pe4EciTYX5NsrPsHrTLENGNIWYNTU= -git.novox.be/novox/mesh-host v0.0.0-20261011092230-20d5af9b2d5f/go.mod h1:tcTK4LMs1d6JpZUwy3hSHMFG/MIuDr/XFs7/nbeaW/c= +git.novox.be/novox/mesh-host v0.0.0-20261012102338-f4efc3cad0a6 h1:2AvHCfs6dKOAOBTUoxkebEb5ZxHjQbAeeIMdgRtap6A= +git.novox.be/novox/mesh-host v0.0.0-20261012102338-f4efc3cad0a6/go.mod h1:tcTK4LMs1d6JpZUwy3hSHMFG/MIuDr/XFs7/nbeaW/c= git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261010191001-33917f91ac3c h1:l5onJwoIeH8yE/PjVlEeoPyXBsHp2NQiWLllz2fwX7s= git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261010191001-33917f91ac3c/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= diff --git a/internal/catalogue/declaration.go b/internal/catalogue/declaration.go index a74ca6dc..facc148e 100644 --- a/internal/catalogue/declaration.go +++ b/internal/catalogue/declaration.go @@ -246,6 +246,11 @@ type Rendering struct { // statement's contract is link.RootContract or later). To an older, strict engine the field is not // sent, and the account it names is not judged — which the self-check says, as not judged. JudgesRoot bool + + // GrantsGroups says this machine's node-engine reads an access's `grant` (novox/hq ADR 0312: its statement's + // contract is link.GrantContract or later). To an older, strict engine the field is not sent, nor the + // runtime's restarts on it, and a read-write access is reached as before: by whoever owns the path. + GrantsGroups bool } // RootField is a user resource's field saying the account must never become root without a person @@ -543,6 +548,9 @@ func (r Resolution) compose(with Rendering, owner map[string]string, } var out []map[string]any + // The read-write accesses each module grants its tools' account, by module (novox/hq ADR 0312): the runtime + // that runs as that account is restarted on them. + grantedToTools := map[string][]string{} for _, m := range r.Modules { if with.Adopted && m.Filtering != nil { // Nothing of a module that loads a filter, on an adopted node: its table would drop @@ -1115,6 +1123,13 @@ func (r Resolution) compose(with Rendering, owner map[string]string, out = append(out, copied) } + // **A read-write access grants its path's group to what runs this module's code** (novox/hq ADR 0312): + // the containers composed above that mount it, and the account its tools run as. Only to an engine that + // reads the field. + if with.GrantsGroups { + grantedToTools[m.Module] = grantAccesses(out, owner, m.Module, r.toolsAccount(m, with)) + } + // **What only the mesh knows, where this module asked for it.** The graph is the control // plane's; making a name resolve is the module's software. Emitted as ordinary files under // this module's name, so they are applied, reported and removed exactly as anything else @@ -1141,6 +1156,12 @@ func (r Resolution) compose(with Rendering, owner map[string]string, } owner[fmt.Sprint(process["id"])] = RuntimeModule out = append(out, process) + // Restarted when an access moved its account's groups, so its process holds them (novox/hq ADR 0312 §6). + for _, m := range r.Modules { + if m.RunsAs == "" { + restartOnGrants(process, grantedToTools[m.Module]) + } + } // And a runtime of its own for each module of its own account, after it (novox/hq ADR 0259 §8). owns, err := r.ownRuntimes(with) if err != nil { @@ -1149,6 +1170,7 @@ func (r Resolution) compose(with Rendering, owner map[string]string, for _, p := range owns { id := fmt.Sprint(p["id"]) owner[id] = strings.TrimSuffix(id, "."+OwnRuntimeID()) + restartOnGrants(p, grantedToTools[owner[id]]) out = append(out, p) } // What each module's bundles are given is read as the account the runtime runs as — not what a diff --git a/internal/catalogue/grants.go b/internal/catalogue/grants.go new file mode 100644 index 00000000..ec20ef15 --- /dev/null +++ b/internal/catalogue/grants.go @@ -0,0 +1,131 @@ +package catalogue + +import ( + "fmt" + "path/filepath" + "sort" + "strings" +) + +// A read-write access grants the path's own group to what runs the module's code (novox/hq ADR 0312, issue 537). +// +// **The controller says who; the node-engine reads which group.** Which group owns the operator's path is a fact +// of the machine the controller cannot know, and the path is never changed (§2): so each read-write access +// reaches a machine naming the identities its module's code runs as, and the node-engine reads the path's group +// at apply and grants it to them. Two kinds of identity run a module's code here: +// +// - each of its containers that mounts the path — the source of one of its volumes is the path, or below it — +// which gains the group as a supplementary group of its process and is recreated with it; +// - the account its tools run as, where it has a tools bundle the machine's runtime loads: the operator's +// account the runtime runs as, or the module's own account for a module served by a runtime of its own +// (ADR 0259 §8). That runtime names the access under its `restart-on`, so it is restarted when the access +// moved the account's groups, and its new process holds the group with no new login (§6). +// +// A read access grants nothing (§3): it is sent as before. A read-write access whose module runs nothing here is +// sent an empty grant, so the node-engine still states it unhealthy while its group cannot write (§2). +// +// **Only to an engine that reads it.** An older node-engine parses strictly and refuses a whole declaration for a +// field it does not know; the field and the restarts it brings are composed only where the engine's statement +// says link.GrantContract or later (Rendering.GrantsGroups). + +// GrantField is an access's field naming who its path's group is granted to (novox/hq ADR 0312). +const GrantField = "grant" + +// toolsAccount is the account module m's tools run as on this machine, or empty where they run nowhere as one: +// no runtime here, no bundle it loads, a module left out of an adopted machine's declaration, or a runtime +// running as root on a machine with no account. +func (r Resolution) toolsAccount(m Manifest, with Rendering) string { + if !r.runtimeHere() || (with.Adopted && m.Filtering != nil) { + return "" + } + loads := false + for _, b := range m.Bundles { + if len(b.Loads) > 0 { + loads = true + } + } + if !loads { + return "" + } + if m.RunsAs != "" { + return m.RunsAs + } + return r.Account +} + +// grantAccesses writes the grant of every read-write access of module, among the resources composed so far: the +// module's containers that mount the access's path, and account when it is not empty. The ids of the accesses +// that grant to account, for the runtime that runs as it to be restarted on. +func grantAccesses(out []map[string]any, owner map[string]string, module, account string) []string { + var containers []map[string]any + for _, res := range out { + if res["type"] == "container" && owner[fmt.Sprint(res["id"])] == module { + containers = append(containers, res) + } + } + var toAccount []string + for _, res := range out { + if res["type"] != "access" || owner[fmt.Sprint(res["id"])] != module || res["mode"] != AccessReadWrite { + continue + } + path, _ := res["path"].(string) + grant := map[string]any{} + var mounting []any + for _, c := range containers { + if mounts(c, path) { + mounting = append(mounting, fmt.Sprint(c["id"])) + } + } + if len(mounting) > 0 { + grant["containers"] = mounting + } + if account != "" { + grant["account"] = account + toAccount = append(toAccount, fmt.Sprint(res["id"])) + } + res[GrantField] = grant + } + sort.Strings(toAccount) + return toAccount +} + +// mounts says whether a container mounts path, or something below it, from the machine. +func mounts(container map[string]any, path string) bool { + volumes, _ := container["volumes"].([]any) + path = filepath.Clean(path) + for _, v := range volumes { + s, ok := v.(string) + if !ok { + continue + } + source, _, _ := strings.Cut(s, ":") + if !strings.HasPrefix(source, "/") { + continue // a named volume: the runtime's, not the operator's path + } + source = filepath.Clean(source) + if source == path || strings.HasPrefix(source, path+"/") { + return true + } + } + return false +} + +// restartOnGrants adds to a runtime process's restart-on the accesses that grant to the account it runs as, so it +// is restarted to hold a group the account was put in or taken out of (novox/hq ADR 0312 §6). +func restartOnGrants(process map[string]any, ids []string) { + if process == nil || len(ids) == 0 { + return + } + restarts, _ := process["restart-on"].([]any) + seen := map[string]bool{} + for _, id := range restarts { + seen[fmt.Sprint(id)] = true + } + for _, id := range ids { + if !seen[id] { + restarts = append(restarts, id) + seen[id] = true + } + } + process["restart-on"] = restarts +} diff --git a/internal/catalogue/grants_test.go b/internal/catalogue/grants_test.go new file mode 100644 index 00000000..39d833f0 --- /dev/null +++ b/internal/catalogue/grants_test.go @@ -0,0 +1,154 @@ +package catalogue + +import ( + "encoding/json" + "fmt" + "testing" + + "github.com/novox/mesh-host/validate" +) + +// Defends novox/hq ADR 0312 and issue 537 on the controller's side: each read-write access reaches the machine +// naming who its path's group is granted to — the module's containers that mount it, and the account its tools +// run as — and the tool runner is restarted on it; a read access grants nothing; and none of it is sent to a +// node-engine older than the field, which would refuse the whole declaration. + +const anImage = "x@sha256:1111111111111111111111111111111111111111111111111111111111111111" + +// anImporter files into one library it writes and reads another: a container mounting both, a step mounting +// neither, and tools served by the machine's runtime. +func anImporter(t *testing.T) Manifest { + t.Helper() + m := aToolsModule(t, "importer", "tools/index.js") + m.Accesses = []Access{ + {ID: "music", Path: "/storage/media/music", Mode: AccessReadWrite}, + {ID: "movies", Path: "/storage/media/movies", Mode: AccessRead}, + } + m.Resources = []map[string]any{ + {"id": "app", "type": "container", "name": "importer", "image": anImage, + "volumes": []any{"${access:music}/incoming:/music", "${access:movies}:/movies:ro"}}, + {"id": "sidecar", "type": "container", "name": "importer-sidecar", "image": anImage}, + } + return m +} + +func composeImporter(t *testing.T, r Resolution, with Rendering) map[string]map[string]any { + t.Helper() + with.Needed = map[string]map[string]string{RuntimeModule: {"broker": "sealed-credential"}} + with.ArtifactStore = "anchor.internal:5101" + out, err := r.Declaration(with) + if err != nil { + t.Fatal(err) + } + // What the node-engine this controller pins would make of it: nothing refused. + body, err := json.Marshal(map[string]any{"declaration": validate.Version, "resources": out}) + if err != nil { + t.Fatal(err) + } + if problems := validate.Declaration(body); len(problems) > 0 { + t.Fatalf("composed into something the node-engine refuses: %v", problems) + } + return byID(out) +} + +func TestAReadWriteAccessNamesTheContainersThatMountItAndTheToolsAccount(t *testing.T) { + r := Resolution{Node: "ace", Account: "jochen", Modules: []Manifest{anImporter(t), theRuntime(t)}} + got := composeImporter(t, r, Rendering{GrantsGroups: true}) + + music := got["importer."+AccessID("/storage/media/music")] + if music == nil { + t.Fatalf("no access resource for the library: %v", keysOf(got)) + } + grant, _ := music[GrantField].(map[string]any) + if fmt.Sprint(grant["containers"]) != "[importer.app]" || grant["account"] != "jochen" { + t.Errorf("the read-write access grants %v; want the container that mounts it and the tools' account", grant) + } + // A read access grants nothing (§3). + if movies := got["importer."+AccessID("/storage/media/movies")]; movies[GrantField] != nil { + t.Errorf("a read access names a grant: %v", movies[GrantField]) + } + // The tool runner is restarted when the access moved its account's groups (§6). + runtime := got[RuntimeModule+"."+RuntimeProcessID()] + found := false + for _, id := range runtime["restart-on"].([]any) { + if id == "importer."+AccessID("/storage/media/music") { + found = true + } + } + if !found { + t.Errorf("the tool runner is not restarted on the access that grants its account: %v", runtime["restart-on"]) + } +} + +func TestAModuleOfItsOwnAccountIsGrantedAsThatAccount(t *testing.T) { + m := anImporter(t) + m.RunsAs = "importer" + m.OwnSecrets = OwnSecrets{"broker": {Path: "/var/lib/mesh/importer/broker"}} + runtime := theRuntime(t) + runtime.Bundles[0].Binary = "node-tools" + r := Resolution{Node: "ace", Account: "jochen", Modules: []Manifest{m, runtime}} + with := Rendering{GrantsGroups: true} + with.Needed = map[string]map[string]string{RuntimeModule: {"broker": "sealed"}, "importer": {"broker": "sealed"}} + with.ArtifactStore = "anchor.internal:5101" + out, err := r.Declaration(with) + if err != nil { + t.Fatal(err) + } + got := byID(out) + grant, _ := got["importer."+AccessID("/storage/media/music")][GrantField].(map[string]any) + if grant["account"] != "importer" { + t.Errorf("the access grants %v; want the module's own account", grant) + } + own := got["importer."+OwnRuntimeID()] + if fmt.Sprint(own["restart-on"]) == "" || !restartsOnIt(own["restart-on"], "importer."+AccessID("/storage/media/music")) { + t.Errorf("the module's own runtime is not restarted on its access: %v", own["restart-on"]) + } + if restartsOnIt(got[RuntimeModule+"."+RuntimeProcessID()]["restart-on"], "importer."+AccessID("/storage/media/music")) { + t.Error("the machine's runtime is restarted on an access of a module it does not serve") + } +} + +func TestAModuleThatRunsNothingHereSendsAnEmptyGrant(t *testing.T) { + m := mod("spool", nil, nil, nil) + m.Accesses = []Access{{Path: "/storage/downloads", Mode: AccessReadWrite}} + out, err := Resolution{Node: "ace", Account: "jochen", Modules: []Manifest{m}}.Declaration(Rendering{GrantsGroups: true}) + if err != nil { + t.Fatal(err) + } + grant, has := byID(out)["spool."+AccessID("/storage/downloads")][GrantField].(map[string]any) + if !has || len(grant) != 0 { + t.Errorf("a read-write access of a module running nothing here is sent %v (sent %v); want an empty grant, "+ + "so its group is still judged", grant, has) + } +} + +func TestNoGrantIsSentToAnEngineOlderThanTheField(t *testing.T) { + r := Resolution{Node: "ace", Account: "jochen", Modules: []Manifest{anImporter(t), theRuntime(t)}} + got := composeImporter(t, r, Rendering{}) + for id, res := range got { + if _, has := res[GrantField]; has { + t.Errorf("%s names a grant to an engine that does not read it", id) + } + } + if restartsOnIt(got[RuntimeModule+"."+RuntimeProcessID()]["restart-on"], "importer."+AccessID("/storage/media/music")) { + t.Error("the tool runner is restarted on an access an older engine grants nothing for") + } +} + +func restartsOnIt(list any, want string) bool { + items, _ := list.([]any) + for _, i := range items { + if fmt.Sprint(i) == want { + return true + } + } + return false +} + +func keysOf(m map[string]map[string]any) []string { + var out []string + for k := range m { + out = append(out, k) + } + return out +} diff --git a/internal/link/protocol.go b/internal/link/protocol.go index 63349ece..5b431422 100644 --- a/internal/link/protocol.go +++ b/internal/link/protocol.go @@ -436,6 +436,12 @@ const ReadinessContract = 2 // parses strictly and would refuse the whole declaration for it. const RootContract = 3 +// GrantContract is the statement of an engine that also reads an access's `grant` (novox/hq ADR 0312): it grants a +// read-write access's path group to the containers and the account the grant names, takes it back, and states the +// access unhealthy while the group cannot write. Only to such an engine is the field sent: an older one parses +// strictly and would refuse the whole declaration for it. +const GrantContract = 4 + // ReasonRoot starts the reason of an account verdict that found a way to root (ADR 0266); the node-engine's // own words (mesh-host internal/accounts ReasonRoot). const ReasonRoot = "can become root without a person" @@ -487,6 +493,11 @@ const KindAccount = "account" // ReasonUsedAsFound. const KindDirectory = "directory" +// KindAccess is a read-write access that grants its path's group (novox/hq ADR 0312): healthy while the group may +// write the path, unhealthy while it cannot — the operator's to decide, since the mesh changes nothing on the path +// — or while the path is owned by a group that is never granted. The node-engine's own (mesh-host internal/link). +const KindAccess = "access" + // ReasonUsedAsFound starts the reason of a directory used as found. const ReasonUsedAsFound = "used as found:" diff --git a/vendor/github.com/novox/mesh-host/internal/declaration/declaration.go b/vendor/github.com/novox/mesh-host/internal/declaration/declaration.go index 50db892e..e4318f9c 100644 --- a/vendor/github.com/novox/mesh-host/internal/declaration/declaration.go +++ b/vendor/github.com/novox/mesh-host/internal/declaration/declaration.go @@ -464,6 +464,35 @@ type Access struct { Path string `json:"path"` // Mode is how this module reaches the path: read or read-write. Absent narrows to read. Mode string `json:"mode,omitempty"` + + // Grant is who the path's own group is granted to, for a read-write access (novox/hq ADR 0312 §1): the + // identities the module's code runs as. The controller says who; the node-engine reads which group owns + // the path when it applies, because only the machine knows, and never changes the path (§2). Absent is a + // controller older than the grant, or an access that grants nothing — and a read access always is one (§3). + Grant *AccessGrant `json:"grant,omitempty"` +} + +// AccessGrant is the identities a read-write access grants its path's group to (novox/hq ADR 0312 §1). +type AccessGrant struct { + // Containers are the module's containers that mount the path, by id: each is created with the group as a + // supplementary group of its process, and recreated when the group moves (§6). + Containers []string `json:"containers,omitempty"` + // Account is the account that runs the module's tools: put in the group as an account group, as ADR 0252 + // §1 puts one, and taken out again as §3 takes back (§5). Empty for a module whose code runs nowhere as + // an account. + Account string `json:"account,omitempty"` +} + +// UnmarshalJSON reads a grant as strictly as a resource is read: a field the engine does not know is a +// thing the controller believes it asked for, and is refused rather than ignored. +func (g *AccessGrant) UnmarshalJSON(raw []byte) error { + type plain AccessGrant + var p plain + if err := strictDecode(raw, &p); err != nil { + return fmt.Errorf("a grant: %w", err) + } + *g = AccessGrant(p) + return nil } func (a *Access) Identity() string { return a.ID } @@ -482,6 +511,22 @@ func (a *Access) validate(where string, _ bool) []string { problems = append(problems, fmt.Sprintf( "%s: an access is %q or %q, not %q", where, AccessRead, AccessReadWrite, a.Mode)) } + if a.Grant != nil && a.Mode != AccessReadWrite { + // A read access reads as any account may, and gains no group (novox/hq ADR 0312 §3). + problems = append(problems, where+": a read access grants nothing, and this one names a grant; "+ + "only a read-write access grants its path's group (novox/hq ADR 0312 §3)") + } + if a.Grant != nil { + for _, id := range a.Grant.Containers { + if id == "" { + problems = append(problems, where+": a grant names a container with no id") + } + } + if a.Grant.Account != "" && strings.ContainsAny(a.Grant.Account, " :/\t\n") { + problems = append(problems, where+": a grant's account is an account's name, and "+ + a.Grant.Account+" is not one") + } + } return problems } @@ -1668,6 +1713,7 @@ func parse(raw []byte, allowActions bool) (*Declaration, error) { d.Resources = append(d.Resources, resource) } problems = append(problems, checkWhileStopped(d.Resources)...) + problems = append(problems, checkGrants(d.Resources)...) problems = append(problems, checkAdoption(env.Adoption, d.Resources, allowActions)...) problems = append(problems, checkReplaces(d.Resources)...) if env.Adoption == nil { @@ -1732,6 +1778,32 @@ func checkWhileStopped(resources []Resource) []string { return problems } +// checkGrants judges each access's grant against the whole declaration (novox/hq ADR 0312): a container it +// names is a container of this declaration, as a maintenance window's are. A grant to a container the machine +// is not told about would be a group given to nothing, said as granted. +func checkGrants(resources []Resource) []string { + containers := map[string]bool{} + for _, r := range resources { + if r.Kind() == TypeContainer { + containers[r.Identity()] = true + } + } + var problems []string + for _, r := range resources { + a, ok := r.(*Access) + if !ok || a.Grant == nil { + continue + } + for _, id := range a.Grant.Containers { + if id != "" && !containers[id] { + problems = append(problems, fmt.Sprintf("resource %q: its grant names %q, and this declaration "+ + "has no container by that id", a.ID, id)) + } + } + } + return problems +} + func strictDecode(raw []byte, into any) error { // DisallowUnknownFields is the whole point rather than strictness for its own sake: a // field the host does not know is a thing the control plane believes it asked for. diff --git a/vendor/modules.txt b/vendor/modules.txt index bb6a448e..56f70d56 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -78,7 +78,7 @@ github.com/nats-io/nkeys # github.com/nats-io/nuid v1.0.1 ## explicit github.com/nats-io/nuid -# github.com/novox/mesh-host v0.0.0 => git.novox.be/novox/mesh-host v0.0.0-20261011092230-20d5af9b2d5f +# github.com/novox/mesh-host v0.0.0 => git.novox.be/novox/mesh-host v0.0.0-20261012102338-f4efc3cad0a6 ## explicit; go 1.26.0 github.com/novox/mesh-host/internal/declaration github.com/novox/mesh-host/rootsearch @@ -135,4 +135,4 @@ golang.org/x/text/width # golang.org/x/time v0.15.0 ## explicit; go 1.25.0 golang.org/x/time/rate -# github.com/novox/mesh-host => git.novox.be/novox/mesh-host v0.0.0-20261011092230-20d5af9b2d5f +# github.com/novox/mesh-host => git.novox.be/novox/mesh-host v0.0.0-20261012102338-f4efc3cad0a6 From 30599c185855b4f8d4cf486dd0287347299162fa Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 12 Oct 2026 13:09:05 +0200 Subject: [PATCH 02/26] Pin the node-engine at its reviewed grant, and hold the mount match to the path itself (hq ADR 0312, issue 537 review) A path beginning with the same letters is another path, and a read-only mount of a read-write access is still granted: the grant follows what mounts it. --- go.mod | 2 +- go.sum | 4 ++-- internal/catalogue/grants.go | 5 ++++- internal/catalogue/grants_test.go | 20 +++++++++++++++++++ .../internal/declaration/declaration.go | 12 ++++++++--- vendor/modules.txt | 4 ++-- 6 files changed, 38 insertions(+), 9 deletions(-) diff --git a/go.mod b/go.mod index 64c2a70c..b2fd83ca 100644 --- a/go.mod +++ b/go.mod @@ -35,4 +35,4 @@ require ( // committed. Every build (the build agent's `go build`, the Dockerfile) compiles from vendor/ and // fetches nothing; go refuses to build when vendor/ and this file disagree, so a pin moved without // `go mod vendor` fails loudly, at once, everywhere. -replace github.com/novox/mesh-host => git.novox.be/novox/mesh-host v0.0.0-20261012102338-f4efc3cad0a6 +replace github.com/novox/mesh-host => git.novox.be/novox/mesh-host v0.0.0-20261012105342-c433e76de229 diff --git a/go.sum b/go.sum index d2d2ed69..9e8b699c 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,5 @@ -git.novox.be/novox/mesh-host v0.0.0-20261012102338-f4efc3cad0a6 h1:2AvHCfs6dKOAOBTUoxkebEb5ZxHjQbAeeIMdgRtap6A= -git.novox.be/novox/mesh-host v0.0.0-20261012102338-f4efc3cad0a6/go.mod h1:tcTK4LMs1d6JpZUwy3hSHMFG/MIuDr/XFs7/nbeaW/c= +git.novox.be/novox/mesh-host v0.0.0-20261012105342-c433e76de229 h1:xakx5tpeQBQhzLMd5Besg3h5OY8l752oExn+fs1xocg= +git.novox.be/novox/mesh-host v0.0.0-20261012105342-c433e76de229/go.mod h1:tcTK4LMs1d6JpZUwy3hSHMFG/MIuDr/XFs7/nbeaW/c= git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261010191001-33917f91ac3c h1:l5onJwoIeH8yE/PjVlEeoPyXBsHp2NQiWLllz2fwX7s= git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261010191001-33917f91ac3c/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= diff --git a/internal/catalogue/grants.go b/internal/catalogue/grants.go index ec20ef15..2574bdb7 100644 --- a/internal/catalogue/grants.go +++ b/internal/catalogue/grants.go @@ -89,7 +89,10 @@ func grantAccesses(out []map[string]any, owner map[string]string, module, accoun return toAccount } -// mounts says whether a container mounts path, or something below it, from the machine. +// mounts says whether a container mounts path, or something below it, from the machine. A path that merely +// begins with the same letters — /storage/media2 beside /storage/media — is another path. A read-only mount of a +// read-write access still counts: the module declared the access read-write, the engine grants the group to what +// mounts it, and a read-only mount only means that container chose not to write through it. func mounts(container map[string]any, path string) bool { volumes, _ := container["volumes"].([]any) path = filepath.Clean(path) diff --git a/internal/catalogue/grants_test.go b/internal/catalogue/grants_test.go index 39d833f0..382b8e68 100644 --- a/internal/catalogue/grants_test.go +++ b/internal/catalogue/grants_test.go @@ -152,3 +152,23 @@ func keysOf(m map[string]map[string]any) []string { } return out } + +// Only a container that mounts the access's path, or below it, is named: a path that begins with the same letters is +// another path. A read-only mount of a read-write access is still named — the access is read-write, and the grant +// follows what mounts it, not how. +func TestOnlyAContainerMountingThePathIsNamedReadOnlyIncluded(t *testing.T) { + m := mod("library", nil, nil, nil) + m.Accesses = []Access{{ID: "media", Path: "/storage/media", Mode: AccessReadWrite}} + m.Resources = []map[string]any{ + {"id": "beside", "type": "container", "name": "beside", "image": anImage, "volumes": []any{"/storage/media2:/x"}}, + {"id": "reader", "type": "container", "name": "reader", "image": anImage, "volumes": []any{"${access:media}:/media:ro"}}, + } + out, err := Resolution{Node: "ace", Modules: []Manifest{m}}.Declaration(Rendering{GrantsGroups: true}) + if err != nil { + t.Fatal(err) + } + grant, _ := byID(out)["library."+AccessID("/storage/media")][GrantField].(map[string]any) + if fmt.Sprint(grant["containers"]) != "[library.reader]" { + t.Errorf("the access grants %v; want the read-only mount and not /storage/media2", grant) + } +} diff --git a/vendor/github.com/novox/mesh-host/internal/declaration/declaration.go b/vendor/github.com/novox/mesh-host/internal/declaration/declaration.go index e4318f9c..e2f6d294 100644 --- a/vendor/github.com/novox/mesh-host/internal/declaration/declaration.go +++ b/vendor/github.com/novox/mesh-host/internal/declaration/declaration.go @@ -522,9 +522,11 @@ func (a *Access) validate(where string, _ bool) []string { problems = append(problems, where+": a grant names a container with no id") } } - if a.Grant.Account != "" && strings.ContainsAny(a.Grant.Account, " :/\t\n") { - problems = append(problems, where+": a grant's account is an account's name, and "+ - a.Grant.Account+" is not one") + if a.Grant.Account != "" && !accountName.MatchString(a.Grant.Account) { + // Handed to id, usermod and gpasswd as an argument: a name starting with a dash would be read as an + // option by each of them. + problems = append(problems, fmt.Sprintf("%s: a grant's account is an account's name — letters, digits, "+ + "dots, underscores and dashes, not starting with a dash — and %q is not one", where, a.Grant.Account)) } } return problems @@ -1778,6 +1780,10 @@ func checkWhileStopped(resources []Resource) []string { return problems } +// accountName is what a grant's account may be: the portable account name the user tools take, optionally ending +// in `$`, never starting with a dash. +var accountName = regexp.MustCompile(`^[A-Za-z0-9_.][A-Za-z0-9_.-]*\$?$`) + // checkGrants judges each access's grant against the whole declaration (novox/hq ADR 0312): a container it // names is a container of this declaration, as a maintenance window's are. A grant to a container the machine // is not told about would be a group given to nothing, said as granted. diff --git a/vendor/modules.txt b/vendor/modules.txt index 56f70d56..775fd133 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -78,7 +78,7 @@ github.com/nats-io/nkeys # github.com/nats-io/nuid v1.0.1 ## explicit github.com/nats-io/nuid -# github.com/novox/mesh-host v0.0.0 => git.novox.be/novox/mesh-host v0.0.0-20261012102338-f4efc3cad0a6 +# github.com/novox/mesh-host v0.0.0 => git.novox.be/novox/mesh-host v0.0.0-20261012105342-c433e76de229 ## explicit; go 1.26.0 github.com/novox/mesh-host/internal/declaration github.com/novox/mesh-host/rootsearch @@ -135,4 +135,4 @@ golang.org/x/text/width # golang.org/x/time v0.15.0 ## explicit; go 1.25.0 golang.org/x/time/rate -# github.com/novox/mesh-host => git.novox.be/novox/mesh-host v0.0.0-20261012102338-f4efc3cad0a6 +# github.com/novox/mesh-host => git.novox.be/novox/mesh-host v0.0.0-20261012105342-c433e76de229 From 74bd90df7ef4f39b0e5dbae5fcc1faa83e731c87 Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 12 Oct 2026 13:39:12 +0200 Subject: [PATCH 03/26] Pin the node-engine at its merge of the access grant, and capture it beside (hq ADR 0312, issue 537) --- go.mod | 2 +- go.sum | 4 ++-- testdata/beside/CAPTURED | 2 +- vendor/modules.txt | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index b2fd83ca..527faf8e 100644 --- a/go.mod +++ b/go.mod @@ -35,4 +35,4 @@ require ( // committed. Every build (the build agent's `go build`, the Dockerfile) compiles from vendor/ and // fetches nothing; go refuses to build when vendor/ and this file disagree, so a pin moved without // `go mod vendor` fails loudly, at once, everywhere. -replace github.com/novox/mesh-host => git.novox.be/novox/mesh-host v0.0.0-20261012105342-c433e76de229 +replace github.com/novox/mesh-host => git.novox.be/novox/mesh-host v0.0.0-20261012112256-3f7f797168d3 diff --git a/go.sum b/go.sum index 9e8b699c..dfc79fdf 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,5 @@ -git.novox.be/novox/mesh-host v0.0.0-20261012105342-c433e76de229 h1:xakx5tpeQBQhzLMd5Besg3h5OY8l752oExn+fs1xocg= -git.novox.be/novox/mesh-host v0.0.0-20261012105342-c433e76de229/go.mod h1:tcTK4LMs1d6JpZUwy3hSHMFG/MIuDr/XFs7/nbeaW/c= +git.novox.be/novox/mesh-host v0.0.0-20261012112256-3f7f797168d3 h1:VpH8CesuBOo7k8GcXZq1eRUfbvUoktfmcUynI5N3IFQ= +git.novox.be/novox/mesh-host v0.0.0-20261012112256-3f7f797168d3/go.mod h1:tcTK4LMs1d6JpZUwy3hSHMFG/MIuDr/XFs7/nbeaW/c= git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261010191001-33917f91ac3c h1:l5onJwoIeH8yE/PjVlEeoPyXBsHp2NQiWLllz2fwX7s= git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261010191001-33917f91ac3c/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= diff --git a/testdata/beside/CAPTURED b/testdata/beside/CAPTURED index a37baf3b..c110ebf7 100644 --- a/testdata/beside/CAPTURED +++ b/testdata/beside/CAPTURED @@ -8,7 +8,7 @@ private addresses. That is acceptable while mesh-controller is private; if it ev a second place to clean besides the catalogue itself. mesh-catalog b9de001833b1b61b297182b8e3bcdae1cbdeace9 modules/*/module.json, modules/nats/Dockerfile -mesh-host a76baa4b47f07a8b2bab065603414530bd02b53a examples/foundation-first-node-nats.lock +mesh-host 3f7f797168d33ec22241bdd6ef2c64de86eea914 examples/foundation-first-node-nats.lock mesh-sdk 33917f91ac3cc213b879b32f3c70e2aa70395f98 conformance/events/module-event.json To move them, from this repository's root, with the two repositories checked out beside it: diff --git a/vendor/modules.txt b/vendor/modules.txt index 775fd133..470b8478 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -78,7 +78,7 @@ github.com/nats-io/nkeys # github.com/nats-io/nuid v1.0.1 ## explicit github.com/nats-io/nuid -# github.com/novox/mesh-host v0.0.0 => git.novox.be/novox/mesh-host v0.0.0-20261012105342-c433e76de229 +# github.com/novox/mesh-host v0.0.0 => git.novox.be/novox/mesh-host v0.0.0-20261012112256-3f7f797168d3 ## explicit; go 1.26.0 github.com/novox/mesh-host/internal/declaration github.com/novox/mesh-host/rootsearch @@ -135,4 +135,4 @@ golang.org/x/text/width # golang.org/x/time v0.15.0 ## explicit; go 1.25.0 golang.org/x/time/rate -# github.com/novox/mesh-host => git.novox.be/novox/mesh-host v0.0.0-20261012105342-c433e76de229 +# github.com/novox/mesh-host => git.novox.be/novox/mesh-host v0.0.0-20261012112256-3f7f797168d3 From 3e10ad1535b7570e3505a8c2ca65fd3230788e9f Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 12 Oct 2026 14:49:33 +0200 Subject: [PATCH 04/26] Raise a machine that refuses or only partly applies its declarations past one apply interval, naming the package manager and its words, so the mesh says it rather than only status (issue 233) --- .gitignore | 2 + cmd/mesh-controller/apply_failing.go | 118 ++++++++++++++++++++++ cmd/mesh-controller/apply_failing_test.go | 83 +++++++++++++++ cmd/mesh-controller/plain_words.go | 7 ++ cmd/mesh-controller/signals.go | 9 ++ cmd/mesh-controller/signals_test.go | 8 ++ cmd/mesh-controller/watchdogs.go | 5 + 7 files changed, 232 insertions(+) create mode 100644 cmd/mesh-controller/apply_failing.go create mode 100644 cmd/mesh-controller/apply_failing_test.go diff --git a/.gitignore b/.gitignore index 7f67f6c5..ff196481 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,5 @@ /mesh-controller /postgres-provisioner /redis-provisioner +# And where `go build` in cmd/mesh-controller leaves it (novox/hq issue 233). +/cmd/mesh-controller/mesh-controller diff --git a/cmd/mesh-controller/apply_failing.go b/cmd/mesh-controller/apply_failing.go new file mode 100644 index 00000000..5e9230e9 --- /dev/null +++ b/cmd/mesh-controller/apply_failing.go @@ -0,0 +1,118 @@ +package main + +import ( + "fmt" + "sort" + "strings" + "time" + + "github.com/novox/mesh-controller/internal/conditions" + "github.com/novox/mesh-controller/internal/inventory" +) + +// kindApplyFailing is S21's kind: a machine whose declarations have been refused, or applied only in part, +// for longer than one apply interval (novox/hq issue 233). +const kindApplyFailing = "apply-failing" + +// applyEvery is how often a node-engine applies what it was last told when nothing new arrives: its +// ReconcileEvery. A failure still there after one more of them is not one the next apply clears. +const applyEvery = 5 * time.Minute + +// packageManagerKey is how a node-engine names a package manager that does not answer among what failed +// (mesh-host's probe.go): "the package manager (pacman)". +const packageManagerKey = "the package manager (" + +// watchApplyFailing is S21: **a machine that does not apply what it was sent is said by the mesh, not only +// to someone who asks for `status`** (novox/hq issue 233, issue 187's family). A workstation refused every +// declaration for an hour and a half because its package manager did not answer, and the mesh's only word +// on it was a line in `status`. Raised once its reports have said the same failure for longer than one +// apply interval, over at least two reports — a laptop shut for a week has had one attempt, not a week of +// them — naming the refusal's or the failure's words; cleared by a clean apply. Not while the machine is +// silent or asleep: S1 says silent, and asleep is not lost. +func watchApplyFailing(f *signalFacts) []conditions.Observation { + quiet := map[string]bool{} + for _, m := range f.machines { + if m.asleep() || (!m.lastHeard.IsZero() && f.now.Sub(m.lastHeard) > heartbeatBound(m.every)) { + quiet[m.name] = true + } + } + var out []conditions.Observation + for _, d := range f.doing { + if !d.Wrong() || d.Since == nil || d.Times < 2 || f.now.Sub(*d.Since) <= applyEvery || quiet[d.Node] { + continue + } + out = append(out, applyFailing(d, f.now)) + } + return out +} + +// applyFailing is one machine's S21 condition. +func applyFailing(d inventory.Doing, now time.Time) conditions.Observation { + m := d.Node + since := d.Since.UTC().Format("2006-01-02 15:04 MST") + o := conditions.Observation{Scope: conditions.ScopeMachine, ID: m, Kind: kindApplyFailing, Machine: m, + Severity: conditions.Warning, Said: doingWords(d), + Resolved: m + " applies what it is sent again"} + how := "refused its declaration whole" + if d.Outcome != inventory.OutcomeRefused { + how = "applied its declaration only in part" + } + o.Summary = fmt.Sprintf("%s has %s on %d reports in a row since %s (bound %s): %s", m, how, d.Times, since, + ago(applyEvery), failedNames(d)) + o.Headline = m + " is not applying its update" + o.Explanation = fmt.Sprintf("%s has %s for %s, and trying again has not helped. The details say what fails.", + m, plainHow(d), humanDuration(now.Sub(*d.Since))) + if tool := unansweringTool(d); tool != "" { + o.Summary = fmt.Sprintf("%s: %s does not answer, so no package reaches it; everything else it was sent "+ + "is applied, on %d reports in a row since %s (bound %s)", m, tool, d.Times, since, ago(applyEvery)) + o.Headline = fmt.Sprintf("%s cannot install packages", m) + o.Explanation = fmt.Sprintf("The package manager on %s (%s) has not answered for %s, so no package reaches "+ + "it. Everything else is still applied, so an update that repairs it can land; if none does, it needs a "+ + "look on the machine.", m, tool, humanDuration(now.Sub(*d.Since))) + } + return o +} + +// plainHow is how a machine failed, as the operator reads it. +func plainHow(d inventory.Doing) string { + if d.Outcome == inventory.OutcomeRefused { + return "refused every update" + } + return "applied its updates only in part" +} + +// unansweringTool is the package manager a machine said does not answer, or "". +func unansweringTool(d inventory.Doing) string { + for _, r := range d.Failed { + if rest, ok := strings.CutPrefix(r.ID, packageManagerKey); ok { + return strings.TrimSuffix(rest, ")") + } + } + return "" +} + +// failedNames is what failed, by name: the refusal, or the failed resources' ids. +func failedNames(d inventory.Doing) string { + if d.Outcome == inventory.OutcomeRefused { + return "refused" + } + ids := make([]string, 0, len(d.Failed)) + for _, r := range d.Failed { + ids = append(ids, r.ID) + } + sort.Strings(ids) + return "failed " + strings.Join(ids, ", ") +} + +// doingWords is the machine's own words on what failed: the evidence, kept inside the mesh. +func doingWords(d inventory.Doing) string { + if d.Outcome == inventory.OutcomeRefused { + return "refused: " + oneLine(d.Refused) + } + var parts []string + for _, r := range d.Failed { + parts = append(parts, r.ID+": "+oneLine(r.Error)) + } + sort.Strings(parts) + return strings.Join(parts, "; ") +} diff --git a/cmd/mesh-controller/apply_failing_test.go b/cmd/mesh-controller/apply_failing_test.go new file mode 100644 index 00000000..36248b3f --- /dev/null +++ b/cmd/mesh-controller/apply_failing_test.go @@ -0,0 +1,83 @@ +package main + +import ( + "strings" + "testing" + "time" + + "github.com/novox/mesh-controller/internal/inventory" +) + +// unansweringPackageManager is anchor's last report as the workstation's read on 2026-10-04, once its +// node-engine applies the rest (novox/hq issue 233): the package failed and the package manager did not answer, +// the same failure on times reports since for before now. +func unansweringPackageManager(now time.Time, forHowLong time.Duration, times int) inventory.Doing { + since := now.Add(-forHowLong) + return inventory.Doing{Node: "anchor", Outcome: inventory.OutcomeFailed, At: now.Add(-time.Second), + Since: &since, Times: times, Applied: 41, Failed: []inventory.FailedResource{ + {ID: "apply", Error: `applying "pacman.base": nothing can be said about "base": this is the arch host and ` + + "pacman does not answer here. Either this machine is not Arch, or its package database is broken: " + + "pacman exited 1: error: config file /etc/pacman.conf could not be read: No such file or directory"}, + {ID: "the package manager (pacman)", Error: "pacman does not answer here, so every package in this " + + "declaration fails and the rest of it is applied (novox/hq issue 233): pacman exited 1: error: config " + + "file /etc/pacman.conf could not be read: No such file or directory"}}} +} + +// **A machine whose package manager does not answer is raised by the mesh itself, naming the tool and the +// probe's words** (novox/hq issue 233): the words in the evidence, the path kept out of the summary. +func TestAPackageManagerThatDoesNotAnswerIsRaisedNamingItsWords(t *testing.T) { + now := time.Date(2026, 10, 4, 14, 0, 0, 0, time.UTC) + f := calm(now) + f.doing = []inventory.Doing{unansweringPackageManager(now, 90*time.Minute, 18)} + got := watchApplyFailing(f) + if len(got) != 1 { + t.Fatalf("raised %d conditions, want one: %+v", len(got), got) + } + o := got[0] + if o.Key() != "machine.anchor.apply-failing" || o.Machine != "anchor" { + t.Errorf("the condition is not the machine's: %s", o.Key()) + } + if !strings.Contains(o.Summary, "pacman does not answer") || !strings.Contains(o.Summary, "18 reports") { + t.Errorf("the summary does not name the tool and how long: %s", o.Summary) + } + if strings.Contains(o.Summary, "/etc/") { + t.Errorf("the summary carries a path, which belongs in the evidence: %s", o.Summary) + } + if !strings.Contains(o.Said, "could not be read") { + t.Errorf("the evidence does not carry the probe's words: %s", o.Said) + } + if !strings.Contains(o.Headline, "cannot install packages") || o.Explanation == "" || o.Resolved == "" { + t.Errorf("the plain words do not say it: %+v", o) + } +} + +// **A whole refusal is raised too**, with the node-engine's words: what the workstation's report said before the +// node-engine applied the rest. +func TestARefusalThatGoesOnIsRaised(t *testing.T) { + now := time.Date(2026, 10, 4, 14, 0, 0, 0, time.UTC) + since := now.Add(-time.Hour) + f := calm(now) + f.doing = []inventory.Doing{{Node: "anchor", Outcome: inventory.OutcomeRefused, Since: &since, Times: 12, + Refused: "this is the arch host and pacman does not answer here", At: now.Add(-time.Minute)}} + got := watchApplyFailing(f) + if len(got) != 1 || !strings.Contains(got[0].Summary, "refused its declaration whole") || + !strings.Contains(got[0].Said, "pacman does not answer") { + t.Fatalf("a refusal that went on for an hour was not raised with its words: %+v", got) + } +} + +// **One attempt is not a fault that goes on, and a silent or sleeping machine is S1's**: a laptop shut for a +// week comes back with one report of an old failure; a machine not heard from says silent, not this. +func TestOneReportOrAQuietMachineIsNotRaised(t *testing.T) { + now := time.Date(2026, 10, 4, 14, 0, 0, 0, time.UTC) + f := calm(now) + f.doing = []inventory.Doing{unansweringPackageManager(now, 7*24*time.Hour, 1)} + if got := watchApplyFailing(f); len(got) != 0 { + t.Errorf("one report was raised as a failure that goes on: %+v", got) + } + f.doing = []inventory.Doing{unansweringPackageManager(now, time.Hour, 12)} + f.machines[0].lastHeard = now.Add(-time.Hour) + if got := watchApplyFailing(f); len(got) != 0 { + t.Errorf("a silent machine was raised for its last report: %+v", got) + } +} diff --git a/cmd/mesh-controller/plain_words.go b/cmd/mesh-controller/plain_words.go index 35846e47..7cd4f3b6 100644 --- a/cmd/mesh-controller/plain_words.go +++ b/cmd/mesh-controller/plain_words.go @@ -81,6 +81,13 @@ var plainWordings = map[string]func(conditions.Observation) words{ "It usually catches up by itself.", m, m), Resolved: m + " applied the change"} }), + kindApplyFailing: worded(func(o conditions.Observation) words { + m := machineOr(o, "a machine") + return words{Headline: m + " is not applying its update", + Explanation: fmt.Sprintf("%s has refused or only partly applied what the mesh sent it, again and again. "+ + "The details say what fails.", conditions.Capital(m)), + Resolved: m + " applies what it is sent again"} + }), "tools-silent": worded(func(o conditions.Observation) words { m := machineOr(o, "a machine") return words{Headline: "Tools on " + m + " are not answering", diff --git a/cmd/mesh-controller/signals.go b/cmd/mesh-controller/signals.go index 34a6781c..c855e2b7 100644 --- a/cmd/mesh-controller/signals.go +++ b/cmd/mesh-controller/signals.go @@ -254,6 +254,15 @@ var signalsTable = []signalRow{ newest: func(f *signalFacts) time.Time { return newestOf(f.bus.waits, func(w busWaitFacts) time.Time { return w.since }) }}, + {Row: "S21", Signal: "a clean apply", Emitter: "node-engine", Trigger: "each report of a declaration refused or applied only in part (novox/hq issue 233)", + Bound: "one apply interval (5 min) past the first report of the same failure, over at least two reports; not " + + "while the machine is silent or asleep; naming the refusal, or the package manager that does not answer " + + "with its words; cleared by a clean apply", + Kind: kindApplyFailing, Severity: conditions.Warning, Phase: 2, + needs: func(f *signalFacts) error { return f.doingErr }, watch: watchApplyFailing, + newest: func(f *signalFacts) time.Time { + return newestOf(f.doing, func(d inventory.Doing) time.Time { return d.At }) + }}, } // watchFacts is S14: the snapshot a merge check is fed is older than its bound, or none was kept since diff --git a/cmd/mesh-controller/signals_test.go b/cmd/mesh-controller/signals_test.go index 9abf6ce8..64c4becc 100644 --- a/cmd/mesh-controller/signals_test.go +++ b/cmd/mesh-controller/signals_test.go @@ -186,6 +186,14 @@ var suppressions = map[string]suppression{ }, past: func(f *signalFacts) { f.refusedSends = []inventory.Send{refusedSend(f.now.Add(-time.Second))} }, }, + // A machine whose package manager does not answer (novox/hq issue 233): said once the same failure has + // outlasted one apply interval over two reports. Inside: a second report, only five minutes after the first. + "S21": { + inside: func(f *signalFacts) { f.doing = []inventory.Doing{unansweringPackageManager(f.now, 5*time.Minute, 2)} }, + past: func(f *signalFacts) { + f.doing = []inventory.Doing{unansweringPackageManager(f.now, 5*time.Minute+time.Second, 2)} + }, + }, // Twice by hand within a fortnight is a healer wanted; once, or the first of two a day too old, is not. "S15": { inside: func(f *signalFacts) { diff --git a/cmd/mesh-controller/watchdogs.go b/cmd/mesh-controller/watchdogs.go index 58692163..fc3fd58c 100644 --- a/cmd/mesh-controller/watchdogs.go +++ b/cmd/mesh-controller/watchdogs.go @@ -59,6 +59,10 @@ type signalFacts struct { // from, each naming its sender (S20, novox/hq issue 234). refusedSends []inventory.Send refusedSendsErr error + // doing is every machine whose last report was not a clean apply, with since when it has said the same + // failure (S21, novox/hq issue 233). + doing []inventory.Doing + doingErr error loop loopFacts loopErr error @@ -369,6 +373,7 @@ func (w *watchdogs) gather(ctx context.Context) *signalFacts { } f.handActs, f.handActsErr = w.gatherHandActs(ctx, now) f.refusedSends, f.refusedSendsErr = inv.RefusedSendsSince(ctx, now.Add(-generationRefusalsSaid)) + f.doing, f.doingErr = inv.NotDoingWhatTheyWereTold(ctx) f.facts.taken, _, f.facts.began, f.facts.err = exportedFacts.last() return f } From 10f76cafc4852f55962b8e456b5faafeb48d8f65 Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 12 Oct 2026 15:10:52 +0200 Subject: [PATCH 05/26] Read an xz release in Go, so the build seat needs no xz tool (hq issue 345) Betterbird's pinned .tar.xz failed on the build seat with "this build seat has no xz to read it": the seat's image is the catalogue's build-agent recipe, which installs git and docker-cli alone, and xz had been added only to cmd/mesh-builder/Dockerfile, which the seat does not build from. The stream is now read with github.com/ulikunitz/xz (vendored), its dictionary capped at xz's own largest preset, and what it unpacks to bounded as before; the xz line in the unused Dockerfile is taken back out. --- cmd/mesh-builder/Dockerfile | 5 +- go.mod | 1 + go.sum | 2 + internal/builder/testdata/release.tar.xz | Bin 0 -> 252 bytes internal/builder/upstream_file.go | 55 +- internal/builder/upstream_file_test.go | 22 + vendor/github.com/ulikunitz/xz/.gitignore | 28 + vendor/github.com/ulikunitz/xz/LICENSE | 26 + vendor/github.com/ulikunitz/xz/README.md | 88 +++ vendor/github.com/ulikunitz/xz/SECURITY.md | 19 + vendor/github.com/ulikunitz/xz/TODO.md | 395 ++++++++++ vendor/github.com/ulikunitz/xz/bits.go | 79 ++ vendor/github.com/ulikunitz/xz/crc.go | 54 ++ vendor/github.com/ulikunitz/xz/format.go | 721 ++++++++++++++++++ .../github.com/ulikunitz/xz/fox-check-none.xz | Bin 0 -> 96 bytes vendor/github.com/ulikunitz/xz/fox.xz | Bin 0 -> 104 bytes .../ulikunitz/xz/internal/hash/cyclic_poly.go | 181 +++++ .../ulikunitz/xz/internal/hash/doc.go | 14 + .../ulikunitz/xz/internal/hash/rabin_karp.go | 66 ++ .../ulikunitz/xz/internal/hash/roller.go | 29 + .../ulikunitz/xz/internal/xlog/xlog.go | 456 +++++++++++ .../github.com/ulikunitz/xz/lzma/bintree.go | 522 +++++++++++++ vendor/github.com/ulikunitz/xz/lzma/bitops.go | 47 ++ .../github.com/ulikunitz/xz/lzma/breader.go | 40 + vendor/github.com/ulikunitz/xz/lzma/buffer.go | 171 +++++ .../ulikunitz/xz/lzma/bytewriter.go | 37 + .../github.com/ulikunitz/xz/lzma/decoder.go | 277 +++++++ .../ulikunitz/xz/lzma/decoderdict.go | 128 ++++ .../ulikunitz/xz/lzma/directcodec.go | 38 + .../github.com/ulikunitz/xz/lzma/distcodec.go | 140 ++++ .../github.com/ulikunitz/xz/lzma/encoder.go | 268 +++++++ .../ulikunitz/xz/lzma/encoderdict.go | 149 ++++ vendor/github.com/ulikunitz/xz/lzma/fox.lzma | Bin 0 -> 67 bytes .../github.com/ulikunitz/xz/lzma/hashtable.go | 309 ++++++++ vendor/github.com/ulikunitz/xz/lzma/header.go | 170 +++++ .../github.com/ulikunitz/xz/lzma/header2.go | 398 ++++++++++ .../ulikunitz/xz/lzma/lengthcodec.go | 115 +++ .../ulikunitz/xz/lzma/literalcodec.go | 125 +++ .../ulikunitz/xz/lzma/matchalgorithm.go | 52 ++ .../github.com/ulikunitz/xz/lzma/operation.go | 55 ++ vendor/github.com/ulikunitz/xz/lzma/prob.go | 53 ++ .../ulikunitz/xz/lzma/properties.go | 69 ++ .../ulikunitz/xz/lzma/rangecodec.go | 222 ++++++ vendor/github.com/ulikunitz/xz/lzma/reader.go | 193 +++++ .../github.com/ulikunitz/xz/lzma/reader2.go | 231 ++++++ vendor/github.com/ulikunitz/xz/lzma/state.go | 145 ++++ .../ulikunitz/xz/lzma/treecodecs.go | 133 ++++ vendor/github.com/ulikunitz/xz/lzma/writer.go | 209 +++++ .../github.com/ulikunitz/xz/lzma/writer2.go | 305 ++++++++ vendor/github.com/ulikunitz/xz/lzmafilter.go | 117 +++ vendor/github.com/ulikunitz/xz/make-docs | 5 + vendor/github.com/ulikunitz/xz/none-check.go | 23 + vendor/github.com/ulikunitz/xz/reader.go | 359 +++++++++ vendor/github.com/ulikunitz/xz/writer.go | 399 ++++++++++ vendor/modules.txt | 6 + 55 files changed, 7707 insertions(+), 44 deletions(-) create mode 100644 internal/builder/testdata/release.tar.xz create mode 100644 vendor/github.com/ulikunitz/xz/.gitignore create mode 100644 vendor/github.com/ulikunitz/xz/LICENSE create mode 100644 vendor/github.com/ulikunitz/xz/README.md create mode 100644 vendor/github.com/ulikunitz/xz/SECURITY.md create mode 100644 vendor/github.com/ulikunitz/xz/TODO.md create mode 100644 vendor/github.com/ulikunitz/xz/bits.go create mode 100644 vendor/github.com/ulikunitz/xz/crc.go create mode 100644 vendor/github.com/ulikunitz/xz/format.go create mode 100644 vendor/github.com/ulikunitz/xz/fox-check-none.xz create mode 100644 vendor/github.com/ulikunitz/xz/fox.xz create mode 100644 vendor/github.com/ulikunitz/xz/internal/hash/cyclic_poly.go create mode 100644 vendor/github.com/ulikunitz/xz/internal/hash/doc.go create mode 100644 vendor/github.com/ulikunitz/xz/internal/hash/rabin_karp.go create mode 100644 vendor/github.com/ulikunitz/xz/internal/hash/roller.go create mode 100644 vendor/github.com/ulikunitz/xz/internal/xlog/xlog.go create mode 100644 vendor/github.com/ulikunitz/xz/lzma/bintree.go create mode 100644 vendor/github.com/ulikunitz/xz/lzma/bitops.go create mode 100644 vendor/github.com/ulikunitz/xz/lzma/breader.go create mode 100644 vendor/github.com/ulikunitz/xz/lzma/buffer.go create mode 100644 vendor/github.com/ulikunitz/xz/lzma/bytewriter.go create mode 100644 vendor/github.com/ulikunitz/xz/lzma/decoder.go create mode 100644 vendor/github.com/ulikunitz/xz/lzma/decoderdict.go create mode 100644 vendor/github.com/ulikunitz/xz/lzma/directcodec.go create mode 100644 vendor/github.com/ulikunitz/xz/lzma/distcodec.go create mode 100644 vendor/github.com/ulikunitz/xz/lzma/encoder.go create mode 100644 vendor/github.com/ulikunitz/xz/lzma/encoderdict.go create mode 100644 vendor/github.com/ulikunitz/xz/lzma/fox.lzma create mode 100644 vendor/github.com/ulikunitz/xz/lzma/hashtable.go create mode 100644 vendor/github.com/ulikunitz/xz/lzma/header.go create mode 100644 vendor/github.com/ulikunitz/xz/lzma/header2.go create mode 100644 vendor/github.com/ulikunitz/xz/lzma/lengthcodec.go create mode 100644 vendor/github.com/ulikunitz/xz/lzma/literalcodec.go create mode 100644 vendor/github.com/ulikunitz/xz/lzma/matchalgorithm.go create mode 100644 vendor/github.com/ulikunitz/xz/lzma/operation.go create mode 100644 vendor/github.com/ulikunitz/xz/lzma/prob.go create mode 100644 vendor/github.com/ulikunitz/xz/lzma/properties.go create mode 100644 vendor/github.com/ulikunitz/xz/lzma/rangecodec.go create mode 100644 vendor/github.com/ulikunitz/xz/lzma/reader.go create mode 100644 vendor/github.com/ulikunitz/xz/lzma/reader2.go create mode 100644 vendor/github.com/ulikunitz/xz/lzma/state.go create mode 100644 vendor/github.com/ulikunitz/xz/lzma/treecodecs.go create mode 100644 vendor/github.com/ulikunitz/xz/lzma/writer.go create mode 100644 vendor/github.com/ulikunitz/xz/lzma/writer2.go create mode 100644 vendor/github.com/ulikunitz/xz/lzmafilter.go create mode 100644 vendor/github.com/ulikunitz/xz/make-docs create mode 100644 vendor/github.com/ulikunitz/xz/none-check.go create mode 100644 vendor/github.com/ulikunitz/xz/reader.go create mode 100644 vendor/github.com/ulikunitz/xz/writer.go diff --git a/cmd/mesh-builder/Dockerfile b/cmd/mesh-builder/Dockerfile index 57e93564..3951536d 100644 --- a/cmd/mesh-builder/Dockerfile +++ b/cmd/mesh-builder/Dockerfile @@ -16,8 +16,7 @@ RUN CGO_ENABLED=0 go build -trimpath -ldflags '-s -w' -o /mesh-builder ./cmd/mes FROM ${ALPINE_BASE} # git to clone what it is asked to build, and the docker client to build and push it. The daemon # is the machine's, reached through its socket — a build machine shares the runtime it was given -# rather than running one inside itself. xz to read an upstream release an archive is pinned to -# (novox/hq ADR 0316): Go's standard library reads gzip and bzip2 and not xz. -RUN apk add --no-cache git docker-cli xz +# rather than running one inside itself. +RUN apk add --no-cache git docker-cli COPY --from=build /mesh-builder /usr/local/bin/mesh-builder ENTRYPOINT ["/usr/local/bin/mesh-builder"] diff --git a/go.mod b/go.mod index cdf300d5..e683fded 100644 --- a/go.mod +++ b/go.mod @@ -8,6 +8,7 @@ require ( github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 github.com/novox/mesh-host v0.0.0 + github.com/ulikunitz/xz v0.5.17 golang.org/x/crypto v0.57.0 golang.org/x/net v0.58.0 golang.org/x/sys v0.48.0 diff --git a/go.sum b/go.sum index e7436cbb..cace9cc9 100644 --- a/go.sum +++ b/go.sum @@ -38,6 +38,8 @@ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UV github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/ulikunitz/xz v0.5.17 h1:flR0y/x1hgM8EGV1AW3Xll6T413G0glV8UfBwR617V4= +github.com/ulikunitz/xz v0.5.17/go.mod h1:H9Rt/W6/Qj27PGauhQc6nfCDy7vHpzsOThBSaYDoEhw= golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M= golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA= golang.org/x/net v0.58.0 h1:ynWG7rqYi4ccpTEuPZ2QGWHktVEM9DMCj9yzDE0Q7To= diff --git a/internal/builder/testdata/release.tar.xz b/internal/builder/testdata/release.tar.xz new file mode 100644 index 0000000000000000000000000000000000000000..76626e1cb5c18315893732b3031e1eb1f4356b39 GIT binary patch literal 252 zcmVvo|O9Af$ zar;<9A#aaLT}W0))(^hP2Jpp!O(!{C8p8FTO=o9O?sEU25U7`~kM0BQh|CD&A0#@G z2d9i3S*LOX22{xwG9`j$_4%h<+@I_!z`+u$!t{!N8s5)qdvN2=0}Gm&K*Ta`)M4N@ zByD1hcqexSz^lG{%WKRr2. + +In both cases expect a response in at least 7 days. + +## Security Advisories + +All security advisories for this project are published under +[github.com/ulikunitz/xz/security/advisories](https://github.com/ulikunitz/xz/security/advisories?state=published). diff --git a/vendor/github.com/ulikunitz/xz/TODO.md b/vendor/github.com/ulikunitz/xz/TODO.md new file mode 100644 index 00000000..5496e79a --- /dev/null +++ b/vendor/github.com/ulikunitz/xz/TODO.md @@ -0,0 +1,395 @@ +# TODO list + +## Release v0.6 + +1. Review encoder and check for lzma improvements under xz. +2. Fix binary tree matcher. +3. Compare compression ratio with xz tool using comparable parameters and optimize parameters +4. rename operation action and make it a simple type of size 8 +5. make maxMatches, wordSize parameters +6. stop searching after a certain length is found (parameter sweetLen) + +## Release v0.7 + +1. Optimize code +2. Do statistical analysis to get linear presets. +3. Test sync.Pool compatability for xz and lzma Writer and Reader +4. Fuzz optimized code. + +## Release v0.8 + +1. Support parallel go routines for writing and reading xz files. +2. Support a ReaderAt interface for xz files with small block sizes. +3. Improve compatibility between gxz and xz +4. Provide manual page for gxz + +## Release v0.9 + +1. Improve documentation +2. Fuzz again + +## Release v1.0 + +1. Full functioning gxz +2. Add godoc URL to README.md (godoc.org) +3. Resolve all issues. +4. Define release candidates. +5. Public announcement. + +## Package lzma + +### v0.6 + +* Rewrite Encoder into a simple greedy one-op-at-a-time encoder including + * simple scan at the dictionary head for the same byte + * use the killer byte (requiring matches to get longer, the first test should be the byte that would make the match longer) + +## Optimizations + +* There may be a lot of false sharing in lzma. State; check whether this can be improved by reorganizing the internal structure of it. + +* Check whether batching encoding and decoding improves speed. + +### DAG optimizations + +* Use full buffer to create minimal bit-length above range encoder. +* Might be too slow (see v0.4) + +### Different match finders + +* hashes with 2, 3 characters additional to 4 characters +* binary trees with 2-7 characters (uint64 as key, use uint32 as + + pointers into a an array) + +* rb-trees with 2-7 characters (uint64 as key, use uint32 as pointers + + into an array with bit-steeling for the colors) + +## Release Procedure + +* execute goch -l for all packages; probably with lower param like 0.5. +* check orthography with gospell +* Write release notes in doc/relnotes. +* Update README.md +* xb copyright . in xz directory to ensure all new files have Copyright header +* `VERSION= go generate github.com/ulikunitz/xz/...` to update version files +* Execute [test for Linux/amd64, Linux/x86 and Windows/amd64](.github/workflows/go-test-platforms.yml). +* Update TODO.md - write short log entry +* `git checkout master && git merge dev` +* `git tag -a ` +* `git push` + +## Log + +## 2026-09-19 + +Includes a fix for lzma.breader handling 0, nil Reader returns correctly. + +## 2026-07-20 + +New release v0.5.16 to support other operating systems like Illumos using a +simple fallback function for IsTerminal to avoid adding a dependency. + +## 2025-08-28 + +Release v0.5.14 addresses the security vulnerability CVE-2025-58058. If you put +bytes in from of a LZMA stream, the header might not be read correctly and +memory for the dictionary buffer allocated. I have implemented mitigations for +the problem. + +### 2025-08-20 + +Release v0.5.13 addressed issue #61 regarding handling of multiple WriteClosers +together. So I added a new package xio with a WriteCloserStack to address the +issue. + +### 2024-04-03 + +Release v0.5.12 updates README.md and SECURITY.md to address the supply chain +attack on the original xz implementation. + +### 2022-12-12 + +Matt Dantay (@bodgit) reported an issue with the LZMA reader. The implementation +returned an error if the dictionary size was less than 4096 byte, but the +recommendation stated the actual used window size should be set to 4096 byte in +that case. It actually was the pull request +[#52](https://github.com/ulikunitz/xz/pull/52). The new patch v0.5.11 will fix +it. + +### 2021-02-02 + +Mituo Heijo has fuzzed xz and found a bug in the function readIndexBody. The +function allocated a slice of records immediately after reading the value +without further checks. Since the number has been too large the make function +did panic. The fix is to check the number against the expected number of records +before allocating the records. + +### 2020-12-17 + +Release v0.5.9 fixes warnings, a typo and adds SECURITY.md. + +One fix is interesting. + +```go +const ( + a byte = 0x1 + b = 0x2 +) +``` + +The constants a and b don't have the same type. Correct is + +```go +const ( + a byte = 0x1 + b byte = 0x2 +) +``` + +### 2020-08-19 + +Release v0.5.8 fixes issue +[issue #35](https://github.com/ulikunitz/xz/issues/35). + +### 2020-02-24 + +Release v0.5.7 supports the check-ID None and fixes +[issue #27](https://github.com/ulikunitz/xz/issues/27). + +### 2019-02-20 + +Release v0.5.6 supports the go.mod file. + +### 2018-10-28 + +Release v0.5.5 fixes issues #19 observing ErrLimit outputs. + +### 2017-06-05 + +Release v0.5.4 fixes issues #15 of another problem with the padding size +check for the xz block header. I removed the check completely. + +### 2017-02-15 + +Release v0.5.3 fixes issue #12 regarding the decompression of an empty +XZ stream. Many thanks to Tomasz Kłak, who reported the issue. + +### 2016-12-02 + +Release v0.5.2 became necessary to allow the decoding of xz files with +4-byte padding in the block header. Many thanks to Greg, who reported +the issue. + +### 2016-07-23 + +Release v0.5.1 became necessary to fix problems with 32-bit platforms. +Many thanks to Bruno Brigas, who reported the issue. + +### 2016-07-04 + +Release v0.5 provides improvements to the compressor and provides support for +the decompression of xz files with multiple xz streams. + +### 2016-01-31 + +Another compression rate increase by checking the byte at length of the +best match first, before checking the whole prefix. This makes the +compressor even faster. We have now a large time budget to beat the +compression ratio of the xz tool. For enwik8 we have now over 40 seconds +to reduce the compressed file size for another 7 MiB. + +### 2016-01-30 + +I simplified the encoder. Speed and compression rate increased +dramatically. A high compression rate affects also the decompression +speed. The approach with the buffer and optimizing for operation +compression rate has not been successful. Going for the maximum length +appears to be the best approach. + +### 2016-01-28 + +The release v0.4 is ready. It provides a working xz implementation, +which is rather slow, but works and is interoperable with the xz tool. +It is an important milestone. + +### 2016-01-10 + +I have the first working implementation of an xz reader and writer. I'm +happy about reaching this milestone. + +### 2015-12-02 + +I'm now ready to implement xz because, I have a working LZMA2 +implementation. I decided today that v0.4 will use the slow encoder +using the operations buffer to be able to go back, if I intend to do so. + +### 2015-10-21 + +I have restarted the work on the library. While trying to implement +LZMA2, I discovered that I need to resimplify the encoder and decoder +functions. The option approach is too complicated. Using a limited byte +writer and not caring for written bytes at all and not to try to handle +uncompressed data simplifies the LZMA encoder and decoder much. +Processing uncompressed data and handling limits is a feature of the +LZMA2 format not of LZMA. + +I learned an interesting method from the LZO format. If the last copy is +too far away they are moving the head one 2 bytes and not 1 byte to +reduce processing times. + +### 2015-08-26 + +I have now reimplemented the lzma package. The code is reasonably fast, +but can still be optimized. The next step is to implement LZMA2 and then +xz. + +### 2015-07-05 + +Created release v0.3. The version is the foundation for a full xz +implementation that is the target of v0.4. + +### 2015-06-11 + +The gflag package has been developed because I couldn't use flag and +pflag for a fully compatible support of gzip's and lzma's options. It +seems to work now quite nicely. + +### 2015-06-05 + +The overflow issue was interesting to research, however Henry S. Warren +Jr. Hacker's Delight book was very helpful as usual and had the issue +explained perfectly. Fefe's information on his website was based on the +C FAQ and quite bad, because it didn't address the issue of -MININT == +MININT. + +### 2015-06-04 + +It has been a productive day. I improved the interface of lzma. Reader +and lzma. Writer and fixed the error handling. + +### 2015-06-01 + +By computing the bit length of the LZMA operations I was able to +improve the greedy algorithm implementation. By using an 8 MByte buffer +the compression rate was not as good as for xz but already better then +gzip default. + +Compression is currently slow, but this is something we will be able to +improve over time. + +### 2015-05-26 + +Checked the license of ogier/pflag. The binary lzmago binary should +include the license terms for the pflag library. + +I added the endorsement clause as used by Google for the Go sources the +LICENSE file. + +### 2015-05-22 + +The package lzb contains now the basic implementation for creating or +reading LZMA byte streams. It allows the support for the implementation +of the DAG-shortest-path algorithm for the compression function. + +### 2015-04-23 + +Completed yesterday the lzbase classes. I'm a little bit concerned that +using the components may require too much code, but on the other hand +there is a lot of flexibility. + +### 2015-04-22 + +Implemented Reader and Writer during the Bayern game against Porto. The +second half gave me enough time. + +### 2015-04-21 + +While showering today morning I discovered that the design for OpEncoder +and OpDecoder doesn't work, because encoding/decoding might depend on +the current status of the dictionary. This is not exactly the right way +to start the day. + +Therefore we need to keep the Reader and Writer design. This time around +we simplify it by ignoring size limits. These can be added by wrappers +around the Reader and Writer interfaces. The Parameters type isn't +needed anymore. + +However I will implement a ReaderState and WriterState type to use +static typing to ensure the right State object is combined with the +right lzbase. Reader and lzbase. Writer. + +As a start I have implemented ReaderState and WriterState to ensure +that the state for reading is only used by readers and WriterState only +used by Writers. + +### 2015-04-20 + +Today I implemented the OpDecoder and tested OpEncoder and OpDecoder. + +### 2015-04-08 + +Came up with a new simplified design for lzbase. I implemented already +the type State that replaces OpCodec. + +### 2015-04-06 + +The new lzma package is now fully usable and lzmago is using it now. The +old lzma package has been completely removed. + +### 2015-04-05 + +Implemented lzma. Reader and tested it. + +### 2015-04-04 + +Implemented baseReader by adapting code form lzma. Reader. + +### 2015-04-03 + +The opCodec has been copied yesterday to lzma2. opCodec has a high +number of dependencies on other files in lzma2. Therefore I had to copy +almost all files from lzma. + +### 2015-03-31 + +Removed only a TODO item. + +However in Francesco Campoy's presentation "Go for Javaneros +(Javaïstes?)" is the the idea that using an embedded field E, all the +methods of E will be defined on T. If E is an interface T satisfies E. + + + +I have never used this, but it seems to be a cool idea. + +### 2015-03-30 + +Finished the type writerDict and wrote a simple test. + +### 2015-03-25 + +I started to implement the writerDict. + +### 2015-03-24 + +After thinking long about the LZMA2 code and several false starts, I +have now a plan to create a self-sufficient lzma2 package that supports +the classic LZMA format as well as LZMA2. The core idea is to support a +baseReader and baseWriter type that support the basic LZMA stream +without any headers. Both types must support the reuse of dictionaries +and the opCodec. + +### 2015-01-10 + +1. Implemented simple lzmago tool +2. Tested tool against large 4.4G file + * compression worked correctly; tested decompression with lzma + * decompression hits a full buffer condition +3. Fixed a bug in the compressor and wrote a test for it +4. Executed full cycle for 4.4 GB file; performance can be improved ;-) + +### 2015-01-11 + +* Release v0.2 because of the working LZMA encoder and decoder diff --git a/vendor/github.com/ulikunitz/xz/bits.go b/vendor/github.com/ulikunitz/xz/bits.go new file mode 100644 index 00000000..b30f1ec9 --- /dev/null +++ b/vendor/github.com/ulikunitz/xz/bits.go @@ -0,0 +1,79 @@ +// Copyright 2014-2022 Ulrich Kunitz. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package xz + +import ( + "errors" + "io" +) + +// putUint32LE puts the little-endian representation of x into the first +// four bytes of p. +func putUint32LE(p []byte, x uint32) { + p[0] = byte(x) + p[1] = byte(x >> 8) + p[2] = byte(x >> 16) + p[3] = byte(x >> 24) +} + +// putUint64LE puts the little-endian representation of x into the first +// eight bytes of p. +func putUint64LE(p []byte, x uint64) { + p[0] = byte(x) + p[1] = byte(x >> 8) + p[2] = byte(x >> 16) + p[3] = byte(x >> 24) + p[4] = byte(x >> 32) + p[5] = byte(x >> 40) + p[6] = byte(x >> 48) + p[7] = byte(x >> 56) +} + +// uint32LE converts a little endian representation to an uint32 value. +func uint32LE(p []byte) uint32 { + return uint32(p[0]) | uint32(p[1])<<8 | uint32(p[2])<<16 | + uint32(p[3])<<24 +} + +// putUvarint puts a uvarint representation of x into the byte slice. +func putUvarint(p []byte, x uint64) int { + i := 0 + for x >= 0x80 { + p[i] = byte(x) | 0x80 + x >>= 7 + i++ + } + p[i] = byte(x) + return i + 1 +} + +// errOverflow indicates an overflow of the 64-bit unsigned integer. +var errOverflowU64 = errors.New("xz: uvarint overflows 64-bit unsigned integer") + +// readUvarint reads a uvarint from the given byte reader. +func readUvarint(r io.ByteReader) (x uint64, n int, err error) { + const maxUvarintLen = 10 + + var s uint + i := 0 + for { + b, err := r.ReadByte() + if err != nil { + return x, i, err + } + i++ + if i > maxUvarintLen { + return x, i, errOverflowU64 + } + if b < 0x80 { + if i == maxUvarintLen && b > 1 { + return x, i, errOverflowU64 + } + return x | uint64(b)< 0 { + k = 4 - k + } + return k +} + +/*** Header ***/ + +// headerMagic stores the magic bytes for the header +var headerMagic = []byte{0xfd, '7', 'z', 'X', 'Z', 0x00} + +// HeaderLen provides the length of the xz file header. +const HeaderLen = 12 + +// Constants for the checksum methods supported by xz. +const ( + None byte = 0x0 + CRC32 byte = 0x1 + CRC64 byte = 0x4 + SHA256 byte = 0xa +) + +// errInvalidFlags indicates that flags are invalid. +var errInvalidFlags = errors.New("xz: invalid flags") + +// verifyFlags returns the error errInvalidFlags if the value is +// invalid. +func verifyFlags(flags byte) error { + switch flags { + case None, CRC32, CRC64, SHA256: + return nil + default: + return errInvalidFlags + } +} + +// flagstrings maps flag values to strings. +var flagstrings = map[byte]string{ + None: "None", + CRC32: "CRC-32", + CRC64: "CRC-64", + SHA256: "SHA-256", +} + +// flagString returns the string representation for the given flags. +func flagString(flags byte) string { + s, ok := flagstrings[flags] + if !ok { + return "invalid" + } + return s +} + +// newHashFunc returns a function that creates hash instances for the +// hash method encoded in flags. +func newHashFunc(flags byte) (newHash func() hash.Hash, err error) { + switch flags { + case None: + newHash = newNoneHash + case CRC32: + newHash = newCRC32 + case CRC64: + newHash = newCRC64 + case SHA256: + newHash = sha256.New + default: + err = errInvalidFlags + } + return +} + +// header provides the actual content of the xz file header: the flags. +type header struct { + flags byte +} + +// Errors returned by readHeader. +var errHeaderMagic = errors.New("xz: invalid header magic bytes") + +// ValidHeader checks whether data is a correct xz file header. The +// length of data must be HeaderLen. +func ValidHeader(data []byte) bool { + var h header + err := h.UnmarshalBinary(data) + return err == nil +} + +// String returns a string representation of the flags. +func (h header) String() string { + return flagString(h.flags) +} + +// UnmarshalBinary reads header from the provided data slice. +func (h *header) UnmarshalBinary(data []byte) error { + // header length + if len(data) != HeaderLen { + return errors.New("xz: wrong file header length") + } + + // magic header + if !bytes.Equal(headerMagic, data[:6]) { + return errHeaderMagic + } + + // checksum + crc := crc32.NewIEEE() + crc.Write(data[6:8]) + if uint32LE(data[8:]) != crc.Sum32() { + return errors.New("xz: invalid checksum for file header") + } + + // stream flags + if data[6] != 0 { + return errInvalidFlags + } + flags := data[7] + if err := verifyFlags(flags); err != nil { + return err + } + + h.flags = flags + return nil +} + +// MarshalBinary generates the xz file header. +func (h *header) MarshalBinary() (data []byte, err error) { + if err = verifyFlags(h.flags); err != nil { + return nil, err + } + + data = make([]byte, 12) + copy(data, headerMagic) + data[7] = h.flags + + crc := crc32.NewIEEE() + crc.Write(data[6:8]) + putUint32LE(data[8:], crc.Sum32()) + + return data, nil +} + +/*** Footer ***/ + +// footerLen defines the length of the footer. +const footerLen = 12 + +// footerMagic contains the footer magic bytes. +var footerMagic = []byte{'Y', 'Z'} + +// footer represents the content of the xz file footer. +type footer struct { + indexSize int64 + flags byte +} + +// String prints a string representation of the footer structure. +func (f footer) String() string { + return fmt.Sprintf("%s index size %d", flagString(f.flags), f.indexSize) +} + +// Minimum and maximum for the size of the index (backward size). +const ( + minIndexSize = 4 + maxIndexSize = (1 << 32) * 4 +) + +// MarshalBinary converts footer values into an xz file footer. Note +// that the footer value is checked for correctness. +func (f *footer) MarshalBinary() (data []byte, err error) { + if err = verifyFlags(f.flags); err != nil { + return nil, err + } + if !(minIndexSize <= f.indexSize && f.indexSize <= maxIndexSize) { + return nil, errors.New("xz: index size out of range") + } + if f.indexSize%4 != 0 { + return nil, errors.New( + "xz: index size not aligned to four bytes") + } + + data = make([]byte, footerLen) + + // backward size (index size) + s := (f.indexSize / 4) - 1 + putUint32LE(data[4:], uint32(s)) + // flags + data[9] = f.flags + // footer magic + copy(data[10:], footerMagic) + + // CRC-32 + crc := crc32.NewIEEE() + crc.Write(data[4:10]) + putUint32LE(data, crc.Sum32()) + + return data, nil +} + +// UnmarshalBinary sets the footer value by unmarshalling an xz file +// footer. +func (f *footer) UnmarshalBinary(data []byte) error { + if len(data) != footerLen { + return errors.New("xz: wrong footer length") + } + + // magic bytes + if !bytes.Equal(data[10:], footerMagic) { + return errors.New("xz: footer magic invalid") + } + + // CRC-32 + crc := crc32.NewIEEE() + crc.Write(data[4:10]) + if uint32LE(data) != crc.Sum32() { + return errors.New("xz: footer checksum error") + } + + var g footer + // backward size (index size) + g.indexSize = (int64(uint32LE(data[4:])) + 1) * 4 + + // flags + if data[8] != 0 { + return errInvalidFlags + } + g.flags = data[9] + if err := verifyFlags(g.flags); err != nil { + return err + } + + *f = g + return nil +} + +/*** Block Header ***/ + +// blockHeader represents the content of an xz block header. +type blockHeader struct { + compressedSize int64 + uncompressedSize int64 + filters []filter +} + +// String converts the block header into a string. +func (h blockHeader) String() string { + var buf bytes.Buffer + first := true + if h.compressedSize >= 0 { + fmt.Fprintf(&buf, "compressed size %d", h.compressedSize) + first = false + } + if h.uncompressedSize >= 0 { + if !first { + buf.WriteString(" ") + } + fmt.Fprintf(&buf, "uncompressed size %d", h.uncompressedSize) + first = false + } + for _, f := range h.filters { + if !first { + buf.WriteString(" ") + } + fmt.Fprintf(&buf, "filter %s", f) + first = false + } + return buf.String() +} + +// Masks for the block flags. +const ( + filterCountMask = 0x03 + compressedSizePresent = 0x40 + uncompressedSizePresent = 0x80 + reservedBlockFlags = 0x3C +) + +// errIndexIndicator signals that an index indicator (0x00) has been found +// instead of an expected block header indicator. +var errIndexIndicator = errors.New("xz: found index indicator") + +// readBlockHeader reads the block header. +func readBlockHeader(r io.Reader) (h *blockHeader, n int, err error) { + var buf bytes.Buffer + buf.Grow(20) + + // block header size + z, err := io.CopyN(&buf, r, 1) + n = int(z) + if err != nil { + return nil, n, err + } + s := buf.Bytes()[0] + if s == 0 { + return nil, n, errIndexIndicator + } + + // read complete header + headerLen := (int(s) + 1) * 4 + buf.Grow(headerLen - 1) + z, err = io.CopyN(&buf, r, int64(headerLen-1)) + n += int(z) + if err != nil { + return nil, n, err + } + + // unmarshal block header + h = new(blockHeader) + if err = h.UnmarshalBinary(buf.Bytes()); err != nil { + return nil, n, err + } + + return h, n, nil +} + +// readSizeInBlockHeader reads the uncompressed or compressed size +// fields in the block header. The present value informs the function +// whether the respective field is actually present in the header. +func readSizeInBlockHeader(r io.ByteReader, present bool) (n int64, err error) { + if !present { + return -1, nil + } + x, _, err := readUvarint(r) + if err != nil { + return 0, err + } + if x >= 1<<63 { + return 0, errors.New("xz: size overflow in block header") + } + return int64(x), nil +} + +// UnmarshalBinary unmarshals the block header. +func (h *blockHeader) UnmarshalBinary(data []byte) error { + // Check header length + s := data[0] + if data[0] == 0 { + return errIndexIndicator + } + headerLen := (int(s) + 1) * 4 + if len(data) != headerLen { + return fmt.Errorf("xz: data length %d; want %d", len(data), + headerLen) + } + n := headerLen - 4 + + // Check CRC-32 + crc := crc32.NewIEEE() + crc.Write(data[:n]) + if crc.Sum32() != uint32LE(data[n:]) { + return errors.New("xz: checksum error for block header") + } + + // Block header flags + flags := data[1] + if flags&reservedBlockFlags != 0 { + return errors.New("xz: reserved block header flags set") + } + + r := bytes.NewReader(data[2:n]) + + // Compressed size + var err error + h.compressedSize, err = readSizeInBlockHeader( + r, flags&compressedSizePresent != 0) + if err != nil { + return err + } + + // Uncompressed size + h.uncompressedSize, err = readSizeInBlockHeader( + r, flags&uncompressedSizePresent != 0) + if err != nil { + return err + } + + h.filters, err = readFilters(r, int(flags&filterCountMask)+1) + if err != nil { + return err + } + + // Check padding + // Since headerLen is a multiple of 4 we don't need to check + // alignment. + k := r.Len() + // The standard spec says that the padding should have not more + // than 3 bytes. However we found paddings of 4 or 5 in the + // wild. See https://github.com/ulikunitz/xz/pull/11 and + // https://github.com/ulikunitz/xz/issues/15 + // + // The only reasonable approach seems to be to ignore the + // padding size. We still check that all padding bytes are zero. + if !allZeros(data[n-k : n]) { + return errPadding + } + return nil +} + +// MarshalBinary marshals the binary header. +func (h *blockHeader) MarshalBinary() (data []byte, err error) { + if !(minFilters <= len(h.filters) && len(h.filters) <= maxFilters) { + return nil, errors.New("xz: filter count wrong") + } + for i, f := range h.filters { + if i < len(h.filters)-1 { + if f.id() == lzmaFilterID { + return nil, errors.New( + "xz: LZMA2 filter is not the last") + } + } else { + // last filter + if f.id() != lzmaFilterID { + return nil, errors.New("xz: " + + "last filter must be the LZMA2 filter") + } + } + } + + var buf bytes.Buffer + // header size must set at the end + buf.WriteByte(0) + + // flags + flags := byte(len(h.filters) - 1) + if h.compressedSize >= 0 { + flags |= compressedSizePresent + } + if h.uncompressedSize >= 0 { + flags |= uncompressedSizePresent + } + buf.WriteByte(flags) + + p := make([]byte, 10) + if h.compressedSize >= 0 { + k := putUvarint(p, uint64(h.compressedSize)) + buf.Write(p[:k]) + } + if h.uncompressedSize >= 0 { + k := putUvarint(p, uint64(h.uncompressedSize)) + buf.Write(p[:k]) + } + + for _, f := range h.filters { + fp, err := f.MarshalBinary() + if err != nil { + return nil, err + } + buf.Write(fp) + } + + // padding + for i := padLen(int64(buf.Len())); i > 0; i-- { + buf.WriteByte(0) + } + + // crc place holder + buf.Write(p[:4]) + + data = buf.Bytes() + if len(data)%4 != 0 { + panic("data length not aligned") + } + s := len(data)/4 - 1 + if !(1 < s && s <= 255) { + panic("wrong block header size") + } + data[0] = byte(s) + + crc := crc32.NewIEEE() + crc.Write(data[:len(data)-4]) + putUint32LE(data[len(data)-4:], crc.Sum32()) + + return data, nil +} + +// Constants used for marshalling and unmarshalling filters in the xz +// block header. +const ( + minFilters = 1 + maxFilters = 4 + minReservedID = 1 << 62 +) + +// filter represents a filter in the block header. +type filter interface { + id() uint64 + UnmarshalBinary(data []byte) error + MarshalBinary() (data []byte, err error) + reader(r io.Reader, c *ReaderConfig) (fr io.Reader, err error) + writeCloser(w io.WriteCloser, c *WriterConfig) (fw io.WriteCloser, err error) + // filter must be last filter + last() bool +} + +// readFilter reads a block filter from the block header. At this point +// in time only the LZMA2 filter is supported. +func readFilter(r io.Reader) (f filter, err error) { + br := lzma.ByteReader(r) + + // index + id, _, err := readUvarint(br) + if err != nil { + return nil, err + } + + var data []byte + switch id { + case lzmaFilterID: + data = make([]byte, lzmaFilterLen) + data[0] = lzmaFilterID + if _, err = io.ReadFull(r, data[1:]); err != nil { + return nil, err + } + f = new(lzmaFilter) + default: + if id >= minReservedID { + return nil, errors.New( + "xz: reserved filter id in block stream header") + } + return nil, errors.New("xz: invalid filter id") + } + if err = f.UnmarshalBinary(data); err != nil { + return nil, err + } + return f, err +} + +// readFilters reads count filters. At this point in time only the count +// 1 is supported. +func readFilters(r io.Reader, count int) (filters []filter, err error) { + if count != 1 { + return nil, errors.New("xz: unsupported filter count") + } + f, err := readFilter(r) + if err != nil { + return nil, err + } + return []filter{f}, err +} + +/*** Index ***/ + +// record describes a block in the xz file index. +type record struct { + unpaddedSize int64 + uncompressedSize int64 +} + +// readRecord reads an index record. +func readRecord(r io.ByteReader) (rec record, n int, err error) { + u, k, err := readUvarint(r) + n += k + if err != nil { + return rec, n, err + } + rec.unpaddedSize = int64(u) + if rec.unpaddedSize < 0 { + return rec, n, errors.New("xz: unpadded size negative") + } + + u, k, err = readUvarint(r) + n += k + if err != nil { + return rec, n, err + } + rec.uncompressedSize = int64(u) + if rec.uncompressedSize < 0 { + return rec, n, errors.New("xz: uncompressed size negative") + } + + return rec, n, nil +} + +// MarshalBinary converts an index record in its binary encoding. +func (rec *record) MarshalBinary() (data []byte, err error) { + // maximum length of a uvarint is 10 + p := make([]byte, 20) + n := putUvarint(p, uint64(rec.unpaddedSize)) + n += putUvarint(p[n:], uint64(rec.uncompressedSize)) + return p[:n], nil +} + +// writeIndex writes the index, a sequence of records. +func writeIndex(w io.Writer, index []record) (n int64, err error) { + crc := crc32.NewIEEE() + mw := io.MultiWriter(w, crc) + + // index indicator + k, err := mw.Write([]byte{0}) + n += int64(k) + if err != nil { + return n, err + } + + // number of records + p := make([]byte, 10) + k = putUvarint(p, uint64(len(index))) + k, err = mw.Write(p[:k]) + n += int64(k) + if err != nil { + return n, err + } + + // list of records + for _, rec := range index { + p, err := rec.MarshalBinary() + if err != nil { + return n, err + } + k, err = mw.Write(p) + n += int64(k) + if err != nil { + return n, err + } + } + + // index padding + k, err = mw.Write(make([]byte, padLen(int64(n)))) + n += int64(k) + if err != nil { + return n, err + } + + // crc32 checksum + putUint32LE(p, crc.Sum32()) + k, err = w.Write(p[:4]) + n += int64(k) + + return n, err +} + +// readIndexBody reads the index from the reader. It assumes that the +// index indicator has already been read. +func readIndexBody(r io.Reader, expectedRecordLen int) (records []record, n int64, err error) { + crc := crc32.NewIEEE() + // index indicator + crc.Write([]byte{0}) + + br := lzma.ByteReader(io.TeeReader(r, crc)) + + // number of records + u, k, err := readUvarint(br) + n += int64(k) + if err != nil { + return nil, n, err + } + recLen := int(u) + if recLen < 0 || uint64(recLen) != u { + return nil, n, errors.New("xz: record number overflow") + } + if recLen != expectedRecordLen { + return nil, n, fmt.Errorf( + "xz: index length is %d; want %d", + recLen, expectedRecordLen) + } + + // list of records + records = make([]record, recLen) + for i := range records { + records[i], k, err = readRecord(br) + n += int64(k) + if err != nil { + return nil, n, err + } + } + + p := make([]byte, padLen(int64(n+1)), 4) + k, err = io.ReadFull(br.(io.Reader), p) + n += int64(k) + if err != nil { + return nil, n, err + } + if !allZeros(p) { + return nil, n, errors.New("xz: non-zero byte in index padding") + } + + // crc32 + s := crc.Sum32() + p = p[:4] + k, err = io.ReadFull(br.(io.Reader), p) + n += int64(k) + if err != nil { + return records, n, err + } + if uint32LE(p) != s { + return nil, n, errors.New("xz: wrong checksum for index") + } + + return records, n, nil +} diff --git a/vendor/github.com/ulikunitz/xz/fox-check-none.xz b/vendor/github.com/ulikunitz/xz/fox-check-none.xz new file mode 100644 index 0000000000000000000000000000000000000000..46043f7dc89b610dc3badb9db3426620c4c97462 GIT binary patch literal 96 zcmexsUKJ6=z`*cd=%ynRgCe6CkX@qxbTK1?PDnLRM*R tL9s%9S!$6&2~avGv8qxbB|lw{3#g5Ofzej?!NQIFY(?{`7{LOOQ2>-O93KDx literal 0 HcmV?d00001 diff --git a/vendor/github.com/ulikunitz/xz/fox.xz b/vendor/github.com/ulikunitz/xz/fox.xz new file mode 100644 index 0000000000000000000000000000000000000000..4b820bd5a16e83fe5db4fb315639a4337f862483 GIT binary patch literal 104 zcmexsUKJ6=z`*kC+7>q^21Q0O1_p)_{ill=8FWH2QWXkIGn2Cwl8W-n^AytZD-^Oy za|?dFO$zmVVdxt0+m!4eq- E0K@hlng9R* literal 0 HcmV?d00001 diff --git a/vendor/github.com/ulikunitz/xz/internal/hash/cyclic_poly.go b/vendor/github.com/ulikunitz/xz/internal/hash/cyclic_poly.go new file mode 100644 index 00000000..dae159db --- /dev/null +++ b/vendor/github.com/ulikunitz/xz/internal/hash/cyclic_poly.go @@ -0,0 +1,181 @@ +// Copyright 2014-2022 Ulrich Kunitz. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package hash + +// CyclicPoly provides a cyclic polynomial rolling hash. +type CyclicPoly struct { + h uint64 + p []uint64 + i int +} + +// ror rotates the unsigned 64-bit integer to right. The argument s must be +// less than 64. +func ror(x uint64, s uint) uint64 { + return (x >> s) | (x << (64 - s)) +} + +// NewCyclicPoly creates a new instance of the CyclicPoly structure. The +// argument n gives the number of bytes for which a hash will be executed. +// This number must be positive; the method panics if this isn't the case. +func NewCyclicPoly(n int) *CyclicPoly { + if n < 1 { + panic("argument n must be positive") + } + return &CyclicPoly{p: make([]uint64, 0, n)} +} + +// Len returns the length of the byte sequence for which a hash is generated. +func (r *CyclicPoly) Len() int { + return cap(r.p) +} + +// RollByte hashes the next byte and returns a hash value. The complete becomes +// available after at least Len() bytes have been hashed. +func (r *CyclicPoly) RollByte(x byte) uint64 { + y := hash[x] + if len(r.p) < cap(r.p) { + r.h = ror(r.h, 1) ^ y + r.p = append(r.p, y) + } else { + r.h ^= ror(r.p[r.i], uint(cap(r.p)-1)) + r.h = ror(r.h, 1) ^ y + r.p[r.i] = y + r.i = (r.i + 1) % cap(r.p) + } + return r.h +} + +// Stores the hash for the individual bytes. +var hash = [256]uint64{ + 0x2e4fc3f904065142, 0xc790984cfbc99527, + 0x879f95eb8c62f187, 0x3b61be86b5021ef2, + 0x65a896a04196f0a5, 0xc5b307b80470b59e, + 0xd3bff376a70df14b, 0xc332f04f0b3f1701, + 0x753b5f0e9abf3e0d, 0xb41538fdfe66ef53, + 0x1906a10c2c1c0208, 0xfb0c712a03421c0d, + 0x38be311a65c9552b, 0xfee7ee4ca6445c7e, + 0x71aadeded184f21e, 0xd73426fccda23b2d, + 0x29773fb5fb9600b5, 0xce410261cd32981a, + 0xfe2848b3c62dbc2d, 0x459eaaff6e43e11c, + 0xc13e35fc9c73a887, 0xf30ed5c201e76dbc, + 0xa5f10b3910482cea, 0x2945d59be02dfaad, + 0x06ee334ff70571b5, 0xbabf9d8070f44380, + 0xee3e2e9912ffd27c, 0x2a7118d1ea6b8ea7, + 0x26183cb9f7b1664c, 0xea71dac7da068f21, + 0xea92eca5bd1d0bb7, 0x415595862defcd75, + 0x248a386023c60648, 0x9cf021ab284b3c8a, + 0xfc9372df02870f6c, 0x2b92d693eeb3b3fc, + 0x73e799d139dc6975, 0x7b15ae312486363c, + 0xb70e5454a2239c80, 0x208e3fb31d3b2263, + 0x01f563cabb930f44, 0x2ac4533d2a3240d8, + 0x84231ed1064f6f7c, 0xa9f020977c2a6d19, + 0x213c227271c20122, 0x09fe8a9a0a03d07a, + 0x4236dc75bcaf910c, 0x460a8b2bead8f17e, + 0xd9b27be1aa07055f, 0xd202d5dc4b11c33e, + 0x70adb010543bea12, 0xcdae938f7ea6f579, + 0x3f3d870208672f4d, 0x8e6ccbce9d349536, + 0xe4c0871a389095ae, 0xf5f2a49152bca080, + 0x9a43f9b97269934e, 0xc17b3753cb6f475c, + 0xd56d941e8e206bd4, 0xac0a4f3e525eda00, + 0xa06d5a011912a550, 0x5537ed19537ad1df, + 0xa32fe713d611449d, 0x2a1d05b47c3b579f, + 0x991d02dbd30a2a52, 0x39e91e7e28f93eb0, + 0x40d06adb3e92c9ac, 0x9b9d3afde1c77c97, + 0x9a3f3f41c02c616f, 0x22ecd4ba00f60c44, + 0x0b63d5d801708420, 0x8f227ca8f37ffaec, + 0x0256278670887c24, 0x107e14877dbf540b, + 0x32c19f2786ac1c05, 0x1df5b12bb4bc9c61, + 0xc0cac129d0d4c4e2, 0x9fdb52ee9800b001, + 0x31f601d5d31c48c4, 0x72ff3c0928bcaec7, + 0xd99264421147eb03, 0x535a2d6d38aefcfe, + 0x6ba8b4454a916237, 0xfa39366eaae4719c, + 0x10f00fd7bbb24b6f, 0x5bd23185c76c84d4, + 0xb22c3d7e1b00d33f, 0x3efc20aa6bc830a8, + 0xd61c2503fe639144, 0x30ce625441eb92d3, + 0xe5d34cf359e93100, 0xa8e5aa13f2b9f7a5, + 0x5c2b8d851ca254a6, 0x68fb6c5e8b0d5fdf, + 0xc7ea4872c96b83ae, 0x6dd5d376f4392382, + 0x1be88681aaa9792f, 0xfef465ee1b6c10d9, + 0x1f98b65ed43fcb2e, 0x4d1ca11eb6e9a9c9, + 0x7808e902b3857d0b, 0x171c9c4ea4607972, + 0x58d66274850146df, 0x42b311c10d3981d1, + 0x647fa8c621c41a4c, 0xf472771c66ddfedc, + 0x338d27e3f847b46b, 0x6402ce3da97545ce, + 0x5162db616fc38638, 0x9c83be97bc22a50e, + 0x2d3d7478a78d5e72, 0xe621a9b938fd5397, + 0x9454614eb0f81c45, 0x395fb6e742ed39b6, + 0x77dd9179d06037bf, 0xc478d0fee4d2656d, + 0x35d9d6cb772007af, 0x83a56e92c883f0f6, + 0x27937453250c00a1, 0x27bd6ebc3a46a97d, + 0x9f543bf784342d51, 0xd158f38c48b0ed52, + 0x8dd8537c045f66b4, 0x846a57230226f6d5, + 0x6b13939e0c4e7cdf, 0xfca25425d8176758, + 0x92e5fc6cd52788e6, 0x9992e13d7a739170, + 0x518246f7a199e8ea, 0xf104c2a71b9979c7, + 0x86b3ffaabea4768f, 0x6388061cf3e351ad, + 0x09d9b5295de5bbb5, 0x38bf1638c2599e92, + 0x1d759846499e148d, 0x4c0ff015e5f96ef4, + 0xa41a94cfa270f565, 0x42d76f9cb2326c0b, + 0x0cf385dd3c9c23ba, 0x0508a6c7508d6e7a, + 0x337523aabbe6cf8d, 0x646bb14001d42b12, + 0xc178729d138adc74, 0xf900ef4491f24086, + 0xee1a90d334bb5ac4, 0x9755c92247301a50, + 0xb999bf7c4ff1b610, 0x6aeeb2f3b21e8fc9, + 0x0fa8084cf91ac6ff, 0x10d226cf136e6189, + 0xd302057a07d4fb21, 0x5f03800e20a0fcc3, + 0x80118d4ae46bd210, 0x58ab61a522843733, + 0x51edd575c5432a4b, 0x94ee6ff67f9197f7, + 0x765669e0e5e8157b, 0xa5347830737132f0, + 0x3ba485a69f01510c, 0x0b247d7b957a01c3, + 0x1b3d63449fd807dc, 0x0fdc4721c30ad743, + 0x8b535ed3829b2b14, 0xee41d0cad65d232c, + 0xe6a99ed97a6a982f, 0x65ac6194c202003d, + 0x692accf3a70573eb, 0xcc3c02c3e200d5af, + 0x0d419e8b325914a3, 0x320f160f42c25e40, + 0x00710d647a51fe7a, 0x3c947692330aed60, + 0x9288aa280d355a7a, 0xa1806a9b791d1696, + 0x5d60e38496763da1, 0x6c69e22e613fd0f4, + 0x977fc2a5aadffb17, 0xfb7bd063fc5a94ba, + 0x460c17992cbaece1, 0xf7822c5444d3297f, + 0x344a9790c69b74aa, 0xb80a42e6cae09dce, + 0x1b1361eaf2b1e757, 0xd84c1e758e236f01, + 0x88e0b7be347627cc, 0x45246009b7a99490, + 0x8011c6dd3fe50472, 0xc341d682bffb99d7, + 0x2511be93808e2d15, 0xd5bc13d7fd739840, + 0x2a3cd030679ae1ec, 0x8ad9898a4b9ee157, + 0x3245fef0a8eaf521, 0x3d6d8dbbb427d2b0, + 0x1ed146d8968b3981, 0x0c6a28bf7d45f3fc, + 0x4a1fd3dbcee3c561, 0x4210ff6a476bf67e, + 0xa559cce0d9199aac, 0xde39d47ef3723380, + 0xe5b69d848ce42e35, 0xefa24296f8e79f52, + 0x70190b59db9a5afc, 0x26f166cdb211e7bf, + 0x4deaf2df3c6b8ef5, 0xf171dbdd670f1017, + 0xb9059b05e9420d90, 0x2f0da855c9388754, + 0x611d5e9ab77949cc, 0x2912038ac01163f4, + 0x0231df50402b2fba, 0x45660fc4f3245f58, + 0xb91cc97c7c8dac50, 0xb72d2aafe4953427, + 0xfa6463f87e813d6b, 0x4515f7ee95d5c6a2, + 0x1310e1c1a48d21c3, 0xad48a7810cdd8544, + 0x4d5bdfefd5c9e631, 0xa43ed43f1fdcb7de, + 0xe70cfc8fe1ee9626, 0xef4711b0d8dda442, + 0xb80dd9bd4dab6c93, 0xa23be08d31ba4d93, + 0x9b37db9d0335a39c, 0x494b6f870f5cfebc, + 0x6d1b3c1149dda943, 0x372c943a518c1093, + 0xad27af45e77c09c4, 0x3b6f92b646044604, + 0xac2917909f5fcf4f, 0x2069a60e977e5557, + 0x353a469e71014de5, 0x24be356281f55c15, + 0x2b6d710ba8e9adea, 0x404ad1751c749c29, + 0xed7311bf23d7f185, 0xba4f6976b4acc43e, + 0x32d7198d2bc39000, 0xee667019014d6e01, + 0x494ef3e128d14c83, 0x1f95a152baecd6be, + 0x201648dff1f483a5, 0x68c28550c8384af6, + 0x5fc834a6824a7f48, 0x7cd06cb7365eaf28, + 0xd82bbd95e9b30909, 0x234f0d1694c53f6d, + 0xd2fb7f4a96d83f4a, 0xff0d5da83acac05e, + 0xf8f6b97f5585080a, 0x74236084be57b95b, + 0xa25e40c03bbc36ad, 0x6b6e5c14ce88465b, + 0x4378ffe93e1528c5, 0x94ca92a17118e2d2, +} diff --git a/vendor/github.com/ulikunitz/xz/internal/hash/doc.go b/vendor/github.com/ulikunitz/xz/internal/hash/doc.go new file mode 100644 index 00000000..b4cf8b75 --- /dev/null +++ b/vendor/github.com/ulikunitz/xz/internal/hash/doc.go @@ -0,0 +1,14 @@ +// Copyright 2014-2022 Ulrich Kunitz. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +/* +Package hash provides rolling hashes. + +Rolling hashes have to be used for maintaining the positions of n-byte +sequences in the dictionary buffer. + +The package provides currently the Rabin-Karp rolling hash and a Cyclic +Polynomial hash. Both support the Hashes method to be used with an interface. +*/ +package hash diff --git a/vendor/github.com/ulikunitz/xz/internal/hash/rabin_karp.go b/vendor/github.com/ulikunitz/xz/internal/hash/rabin_karp.go new file mode 100644 index 00000000..5322342e --- /dev/null +++ b/vendor/github.com/ulikunitz/xz/internal/hash/rabin_karp.go @@ -0,0 +1,66 @@ +// Copyright 2014-2022 Ulrich Kunitz. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package hash + +// A is the default constant for Robin-Karp rolling hash. This is a random +// prime. +const A = 0x97b548add41d5da1 + +// RabinKarp supports the computation of a rolling hash. +type RabinKarp struct { + A uint64 + // a^n + aOldest uint64 + h uint64 + p []byte + i int +} + +// NewRabinKarp creates a new RabinKarp value. The argument n defines the +// length of the byte sequence to be hashed. The default constant will will be +// used. +func NewRabinKarp(n int) *RabinKarp { + return NewRabinKarpConst(n, A) +} + +// NewRabinKarpConst creates a new RabinKarp value. The argument n defines the +// length of the byte sequence to be hashed. The argument a provides the +// constant used to compute the hash. +func NewRabinKarpConst(n int, a uint64) *RabinKarp { + if n <= 0 { + panic("number of bytes n must be positive") + } + aOldest := uint64(1) + // There are faster methods. For the small n required by the LZMA + // compressor O(n) is sufficient. + for i := 0; i < n; i++ { + aOldest *= a + } + return &RabinKarp{ + A: a, aOldest: aOldest, + p: make([]byte, 0, n), + } +} + +// Len returns the length of the byte sequence. +func (r *RabinKarp) Len() int { + return cap(r.p) +} + +// RollByte computes the hash after x has been added. +func (r *RabinKarp) RollByte(x byte) uint64 { + if len(r.p) < cap(r.p) { + r.h += uint64(x) + r.h *= r.A + r.p = append(r.p, x) + } else { + r.h -= uint64(r.p[r.i]) * r.aOldest + r.h += uint64(x) + r.h *= r.A + r.p[r.i] = x + r.i = (r.i + 1) % cap(r.p) + } + return r.h +} diff --git a/vendor/github.com/ulikunitz/xz/internal/hash/roller.go b/vendor/github.com/ulikunitz/xz/internal/hash/roller.go new file mode 100644 index 00000000..a9898335 --- /dev/null +++ b/vendor/github.com/ulikunitz/xz/internal/hash/roller.go @@ -0,0 +1,29 @@ +// Copyright 2014-2022 Ulrich Kunitz. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package hash + +// Roller provides an interface for rolling hashes. The hash value will become +// valid after hash has been called Len times. +type Roller interface { + Len() int + RollByte(x byte) uint64 +} + +// Hashes computes all hash values for the array p. Note that the state of the +// roller is changed. +func Hashes(r Roller, p []byte) []uint64 { + n := r.Len() + if len(p) < n { + return nil + } + h := make([]uint64, len(p)-n+1) + for i := 0; i < n-1; i++ { + r.RollByte(p[i]) + } + for i := range h { + h[i] = r.RollByte(p[i+n-1]) + } + return h +} diff --git a/vendor/github.com/ulikunitz/xz/internal/xlog/xlog.go b/vendor/github.com/ulikunitz/xz/internal/xlog/xlog.go new file mode 100644 index 00000000..f4627ea1 --- /dev/null +++ b/vendor/github.com/ulikunitz/xz/internal/xlog/xlog.go @@ -0,0 +1,456 @@ +// Copyright 2014-2022 Ulrich Kunitz. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package xlog provides a simple logging package that allows to disable +// certain message categories. It defines a type, Logger, with multiple +// methods for formatting output. The package has also a predefined +// 'standard' Logger accessible through helper function Print[f|ln], +// Fatal[f|ln], Panic[f|ln], Warn[f|ln], Print[f|ln] and Debug[f|ln] +// that are easier to use then creating a Logger manually. That logger +// writes to standard error and prints the date and time of each logged +// message, which can be configured using the function SetFlags. +// +// The Fatal functions call os.Exit(1) after the message is output +// unless not suppressed by the flags. The Panic functions call panic +// after the writing the log message unless suppressed. +package xlog + +import ( + "fmt" + "io" + "os" + "runtime" + "sync" + "time" +) + +// The flags define what information is prefixed to each log entry +// generated by the Logger. The Lno* versions allow the suppression of +// specific output. The bits are or'ed together to control what will be +// printed. There is no control over the order of the items printed and +// the format. The full format is: +// +// 2009-01-23 01:23:23.123123 /a/b/c/d.go:23: message +const ( + Ldate = 1 << iota // the date: 2009-01-23 + Ltime // the time: 01:23:23 + Lmicroseconds // microsecond resolution: 01:23:23.123123 + Llongfile // full file name and line number: /a/b/c/d.go:23 + Lshortfile // final file name element and line number: d.go:23 + Lnopanic // suppresses output from Panic[f|ln] but not the panic call + Lnofatal // suppresses output from Fatal[f|ln] but not the exit + Lnowarn // suppresses output from Warn[f|ln] + Lnoprint // suppresses output from Print[f|ln] + Lnodebug // suppresses output from Debug[f|ln] + // initial values for the standard logger + Lstdflags = Ldate | Ltime | Lnodebug +) + +// A Logger represents an active logging object that generates lines of +// output to an io.Writer. Each logging operation if not suppressed +// makes a single call to the Writer's Write method. A Logger can be +// used simultaneously from multiple goroutines; it guarantees to +// serialize access to the Writer. +type Logger struct { + mu sync.Mutex // ensures atomic writes; and protects the following + // fields + prefix string // prefix to write at beginning of each line + flag int // properties + out io.Writer // destination for output + buf []byte // for accumulating text to write +} + +// New creates a new Logger. The out argument sets the destination to +// which the log output will be written. The prefix appears at the +// beginning of each log line. The flag argument defines the logging +// properties. +func New(out io.Writer, prefix string, flag int) *Logger { + return &Logger{out: out, prefix: prefix, flag: flag} +} + +// std is the standard logger used by the package scope functions. +var std = New(os.Stderr, "", Lstdflags) + +// itoa converts the integer to ASCII. A negative widths will avoid +// zero-padding. The function supports only non-negative integers. +func itoa(buf *[]byte, i int, wid int) { + var u = uint(i) + if u == 0 && wid <= 1 { + *buf = append(*buf, '0') + return + } + var b [32]byte + bp := len(b) + for ; u > 0 || wid > 0; u /= 10 { + bp-- + wid-- + b[bp] = byte(u%10) + '0' + } + *buf = append(*buf, b[bp:]...) +} + +// formatHeader puts the header into the buf field of the buffer. +func (l *Logger) formatHeader(t time.Time, file string, line int) { + l.buf = append(l.buf, l.prefix...) + if l.flag&(Ldate|Ltime|Lmicroseconds) != 0 { + if l.flag&Ldate != 0 { + year, month, day := t.Date() + itoa(&l.buf, year, 4) + l.buf = append(l.buf, '-') + itoa(&l.buf, int(month), 2) + l.buf = append(l.buf, '-') + itoa(&l.buf, day, 2) + l.buf = append(l.buf, ' ') + } + if l.flag&(Ltime|Lmicroseconds) != 0 { + hour, min, sec := t.Clock() + itoa(&l.buf, hour, 2) + l.buf = append(l.buf, ':') + itoa(&l.buf, min, 2) + l.buf = append(l.buf, ':') + itoa(&l.buf, sec, 2) + if l.flag&Lmicroseconds != 0 { + l.buf = append(l.buf, '.') + itoa(&l.buf, t.Nanosecond()/1e3, 6) + } + l.buf = append(l.buf, ' ') + } + } + if l.flag&(Lshortfile|Llongfile) != 0 { + if l.flag&Lshortfile != 0 { + short := file + for i := len(file) - 1; i > 0; i-- { + if file[i] == '/' { + short = file[i+1:] + break + } + } + file = short + } + l.buf = append(l.buf, file...) + l.buf = append(l.buf, ':') + itoa(&l.buf, line, -1) + l.buf = append(l.buf, ": "...) + } +} + +func (l *Logger) output(calldepth int, now time.Time, s string) error { + var file string + var line int + if l.flag&(Lshortfile|Llongfile) != 0 { + l.mu.Unlock() + var ok bool + _, file, line, ok = runtime.Caller(calldepth) + if !ok { + file = "???" + line = 0 + } + l.mu.Lock() + } + l.buf = l.buf[:0] + l.formatHeader(now, file, line) + l.buf = append(l.buf, s...) + if len(s) == 0 || s[len(s)-1] != '\n' { + l.buf = append(l.buf, '\n') + } + _, err := l.out.Write(l.buf) + return err +} + +// Output writes the string s with the header controlled by the flags to +// the l.out writer. A newline will be appended if s doesn't end in a +// newline. Calldepth is used to recover the PC, although all current +// calls of Output use the call depth 2. Access to the function is serialized. +func (l *Logger) Output(calldepth, noflag int, v ...interface{}) error { + now := time.Now() + l.mu.Lock() + defer l.mu.Unlock() + if l.flag&noflag != 0 { + return nil + } + s := fmt.Sprint(v...) + return l.output(calldepth+1, now, s) +} + +// Outputf works like output but formats the output like Printf. +func (l *Logger) Outputf(calldepth int, noflag int, format string, v ...interface{}) error { + now := time.Now() + l.mu.Lock() + defer l.mu.Unlock() + if l.flag&noflag != 0 { + return nil + } + s := fmt.Sprintf(format, v...) + return l.output(calldepth+1, now, s) +} + +// Outputln works like output but formats the output like Println. +func (l *Logger) Outputln(calldepth int, noflag int, v ...interface{}) error { + now := time.Now() + l.mu.Lock() + defer l.mu.Unlock() + if l.flag&noflag != 0 { + return nil + } + s := fmt.Sprintln(v...) + return l.output(calldepth+1, now, s) +} + +// Panic prints the message like Print and calls panic. The printing +// might be suppressed by the flag Lnopanic. +func (l *Logger) Panic(v ...interface{}) { + l.Output(2, Lnopanic, v...) + s := fmt.Sprint(v...) + panic(s) +} + +// Panic prints the message like Print and calls panic. The printing +// might be suppressed by the flag Lnopanic. +func Panic(v ...interface{}) { + std.Output(2, Lnopanic, v...) + s := fmt.Sprint(v...) + panic(s) +} + +// Panicf prints the message like Printf and calls panic. The printing +// might be suppressed by the flag Lnopanic. +func (l *Logger) Panicf(format string, v ...interface{}) { + l.Outputf(2, Lnopanic, format, v...) + s := fmt.Sprintf(format, v...) + panic(s) +} + +// Panicf prints the message like Printf and calls panic. The printing +// might be suppressed by the flag Lnopanic. +func Panicf(format string, v ...interface{}) { + std.Outputf(2, Lnopanic, format, v...) + s := fmt.Sprintf(format, v...) + panic(s) +} + +// Panicln prints the message like Println and calls panic. The printing +// might be suppressed by the flag Lnopanic. +func (l *Logger) Panicln(v ...interface{}) { + l.Outputln(2, Lnopanic, v...) + s := fmt.Sprintln(v...) + panic(s) +} + +// Panicln prints the message like Println and calls panic. The printing +// might be suppressed by the flag Lnopanic. +func Panicln(v ...interface{}) { + std.Outputln(2, Lnopanic, v...) + s := fmt.Sprintln(v...) + panic(s) +} + +// Fatal prints the message like Print and calls os.Exit(1). The +// printing might be suppressed by the flag Lnofatal. +func (l *Logger) Fatal(v ...interface{}) { + l.Output(2, Lnofatal, v...) + os.Exit(1) +} + +// Fatal prints the message like Print and calls os.Exit(1). The +// printing might be suppressed by the flag Lnofatal. +func Fatal(v ...interface{}) { + std.Output(2, Lnofatal, v...) + os.Exit(1) +} + +// Fatalf prints the message like Printf and calls os.Exit(1). The +// printing might be suppressed by the flag Lnofatal. +func (l *Logger) Fatalf(format string, v ...interface{}) { + l.Outputf(2, Lnofatal, format, v...) + os.Exit(1) +} + +// Fatalf prints the message like Printf and calls os.Exit(1). The +// printing might be suppressed by the flag Lnofatal. +func Fatalf(format string, v ...interface{}) { + std.Outputf(2, Lnofatal, format, v...) + os.Exit(1) +} + +// Fatalln prints the message like Println and calls os.Exit(1). The +// printing might be suppressed by the flag Lnofatal. +func (l *Logger) Fatalln(format string, v ...interface{}) { + l.Outputln(2, Lnofatal, v...) + os.Exit(1) +} + +// Fatalln prints the message like Println and calls os.Exit(1). The +// printing might be suppressed by the flag Lnofatal. +func Fatalln(format string, v ...interface{}) { + std.Outputln(2, Lnofatal, v...) + os.Exit(1) +} + +// Warn prints the message like Print. The printing might be suppressed +// by the flag Lnowarn. +func (l *Logger) Warn(v ...interface{}) { + l.Output(2, Lnowarn, v...) +} + +// Warn prints the message like Print. The printing might be suppressed +// by the flag Lnowarn. +func Warn(v ...interface{}) { + std.Output(2, Lnowarn, v...) +} + +// Warnf prints the message like Printf. The printing might be suppressed +// by the flag Lnowarn. +func (l *Logger) Warnf(format string, v ...interface{}) { + l.Outputf(2, Lnowarn, format, v...) +} + +// Warnf prints the message like Printf. The printing might be suppressed +// by the flag Lnowarn. +func Warnf(format string, v ...interface{}) { + std.Outputf(2, Lnowarn, format, v...) +} + +// Warnln prints the message like Println. The printing might be suppressed +// by the flag Lnowarn. +func (l *Logger) Warnln(v ...interface{}) { + l.Outputln(2, Lnowarn, v...) +} + +// Warnln prints the message like Println. The printing might be suppressed +// by the flag Lnowarn. +func Warnln(v ...interface{}) { + std.Outputln(2, Lnowarn, v...) +} + +// Print prints the message like fmt.Print. The printing might be suppressed +// by the flag Lnoprint. +func (l *Logger) Print(v ...interface{}) { + l.Output(2, Lnoprint, v...) +} + +// Print prints the message like fmt.Print. The printing might be suppressed +// by the flag Lnoprint. +func Print(v ...interface{}) { + std.Output(2, Lnoprint, v...) +} + +// Printf prints the message like fmt.Printf. The printing might be suppressed +// by the flag Lnoprint. +func (l *Logger) Printf(format string, v ...interface{}) { + l.Outputf(2, Lnoprint, format, v...) +} + +// Printf prints the message like fmt.Printf. The printing might be suppressed +// by the flag Lnoprint. +func Printf(format string, v ...interface{}) { + std.Outputf(2, Lnoprint, format, v...) +} + +// Println prints the message like fmt.Println. The printing might be +// suppressed by the flag Lnoprint. +func (l *Logger) Println(v ...interface{}) { + l.Outputln(2, Lnoprint, v...) +} + +// Println prints the message like fmt.Println. The printing might be +// suppressed by the flag Lnoprint. +func Println(v ...interface{}) { + std.Outputln(2, Lnoprint, v...) +} + +// Debug prints the message like Print. The printing might be suppressed +// by the flag Lnodebug. +func (l *Logger) Debug(v ...interface{}) { + l.Output(2, Lnodebug, v...) +} + +// Debug prints the message like Print. The printing might be suppressed +// by the flag Lnodebug. +func Debug(v ...interface{}) { + std.Output(2, Lnodebug, v...) +} + +// Debugf prints the message like Printf. The printing might be suppressed +// by the flag Lnodebug. +func (l *Logger) Debugf(format string, v ...interface{}) { + l.Outputf(2, Lnodebug, format, v...) +} + +// Debugf prints the message like Printf. The printing might be suppressed +// by the flag Lnodebug. +func Debugf(format string, v ...interface{}) { + std.Outputf(2, Lnodebug, format, v...) +} + +// Debugln prints the message like Println. The printing might be suppressed +// by the flag Lnodebug. +func (l *Logger) Debugln(v ...interface{}) { + l.Outputln(2, Lnodebug, v...) +} + +// Debugln prints the message like Println. The printing might be suppressed +// by the flag Lnodebug. +func Debugln(v ...interface{}) { + std.Outputln(2, Lnodebug, v...) +} + +// Flags returns the current flags used by the logger. +func (l *Logger) Flags() int { + l.mu.Lock() + defer l.mu.Unlock() + return l.flag +} + +// Flags returns the current flags used by the standard logger. +func Flags() int { + return std.Flags() +} + +// SetFlags sets the flags of the logger. +func (l *Logger) SetFlags(flag int) { + l.mu.Lock() + defer l.mu.Unlock() + l.flag = flag +} + +// SetFlags sets the flags for the standard logger. +func SetFlags(flag int) { + std.SetFlags(flag) +} + +// Prefix returns the prefix used by the logger. +func (l *Logger) Prefix() string { + l.mu.Lock() + defer l.mu.Unlock() + return l.prefix +} + +// Prefix returns the prefix used by the standard logger of the package. +func Prefix() string { + return std.Prefix() +} + +// SetPrefix sets the prefix for the logger. +func (l *Logger) SetPrefix(prefix string) { + l.mu.Lock() + defer l.mu.Unlock() + l.prefix = prefix +} + +// SetPrefix sets the prefix of the standard logger of the package. +func SetPrefix(prefix string) { + std.SetPrefix(prefix) +} + +// SetOutput sets the output of the logger. +func (l *Logger) SetOutput(w io.Writer) { + l.mu.Lock() + defer l.mu.Unlock() + l.out = w +} + +// SetOutput sets the output for the standard logger of the package. +func SetOutput(w io.Writer) { + std.SetOutput(w) +} diff --git a/vendor/github.com/ulikunitz/xz/lzma/bintree.go b/vendor/github.com/ulikunitz/xz/lzma/bintree.go new file mode 100644 index 00000000..2b39da6f --- /dev/null +++ b/vendor/github.com/ulikunitz/xz/lzma/bintree.go @@ -0,0 +1,522 @@ +// Copyright 2014-2022 Ulrich Kunitz. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package lzma + +import ( + "errors" + "unicode" +) + +// node represents a node in the binary tree. +type node struct { + // x is the search value + x uint32 + // p parent node + p uint32 + // l left child + l uint32 + // r right child + r uint32 +} + +// wordLen is the number of bytes represented by the v field of a node. +const wordLen = 4 + +// binTree supports the identification of the next operation based on a +// binary tree. +// +// Nodes will be identified by their index into the ring buffer. +type binTree struct { + dict *encoderDict + // ring buffer of nodes + node []node + // absolute offset of the entry for the next node. Position 4 + // byte larger. + hoff int64 + // front position in the node ring buffer + front uint32 + // index of the root node + root uint32 + // current x value + x uint32 + // preallocated array + data []byte +} + +// null represents the nonexistent index. We can't use zero because it +// would always exist or we would need to decrease the index for each +// reference. +const null uint32 = 1<<32 - 1 + +// newBinTree initializes the binTree structure. The capacity defines +// the size of the buffer and defines the maximum distance for which +// matches will be found. +func newBinTree(capacity int) (t *binTree, err error) { + if capacity < 1 { + return nil, errors.New( + "newBinTree: capacity must be larger than zero") + } + if int64(capacity) >= int64(null) { + return nil, errors.New( + "newBinTree: capacity must less 2^{32}-1") + } + t = &binTree{ + node: make([]node, capacity), + hoff: -int64(wordLen), + root: null, + data: make([]byte, maxMatchLen), + } + return t, nil +} + +func (t *binTree) SetDict(d *encoderDict) { t.dict = d } + +// WriteByte writes a single byte into the binary tree. +func (t *binTree) WriteByte(c byte) error { + t.x = (t.x << 8) | uint32(c) + t.hoff++ + if t.hoff < 0 { + return nil + } + v := t.front + if int64(v) < t.hoff { + // We are overwriting old nodes stored in the tree. + t.remove(v) + } + t.node[v].x = t.x + t.add(v) + t.front++ + if int64(t.front) >= int64(len(t.node)) { + t.front = 0 + } + return nil +} + +// Writes writes a sequence of bytes into the binTree structure. +func (t *binTree) Write(p []byte) (n int, err error) { + for _, c := range p { + t.WriteByte(c) + } + return len(p), nil +} + +// add puts the node v into the tree. The node must not be part of the +// tree before. +func (t *binTree) add(v uint32) { + vn := &t.node[v] + // Set left and right to null indices. + vn.l, vn.r = null, null + // If the binary tree is empty make v the root. + if t.root == null { + t.root = v + vn.p = null + return + } + x := vn.x + p := t.root + // Search for the right leave link and add the new node. + for { + pn := &t.node[p] + if x <= pn.x { + if pn.l == null { + pn.l = v + vn.p = p + return + } + p = pn.l + } else { + if pn.r == null { + pn.r = v + vn.p = p + return + } + p = pn.r + } + } +} + +// parent returns the parent node index of v and the pointer to v value +// in the parent. +func (t *binTree) parent(v uint32) (p uint32, ptr *uint32) { + if t.root == v { + return null, &t.root + } + p = t.node[v].p + if t.node[p].l == v { + ptr = &t.node[p].l + } else { + ptr = &t.node[p].r + } + return +} + +// Remove node v. +func (t *binTree) remove(v uint32) { + vn := &t.node[v] + p, ptr := t.parent(v) + l, r := vn.l, vn.r + if l == null { + // Move the right child up. + *ptr = r + if r != null { + t.node[r].p = p + } + return + } + if r == null { + // Move the left child up. + *ptr = l + t.node[l].p = p + return + } + + // Search the in-order predecessor u. + un := &t.node[l] + ur := un.r + if ur == null { + // In order predecessor is l. Move it up. + un.r = r + t.node[r].p = l + un.p = p + *ptr = l + return + } + var u uint32 + for { + // Look for the max value in the tree where l is root. + u = ur + ur = t.node[u].r + if ur == null { + break + } + } + // replace u with ul + un = &t.node[u] + ul := un.l + up := un.p + t.node[up].r = ul + if ul != null { + t.node[ul].p = up + } + + // replace v by u + un.l, un.r = l, r + t.node[l].p = u + t.node[r].p = u + *ptr = u + un.p = p +} + +// search looks for the node that have the value x or for the nodes that +// brace it. The node highest in the tree with the value x will be +// returned. All other nodes with the same value live in left subtree of +// the returned node. +func (t *binTree) search(v uint32, x uint32) (a, b uint32) { + a, b = null, null + if v == null { + return + } + for { + vn := &t.node[v] + if x <= vn.x { + if x == vn.x { + return v, v + } + b = v + if vn.l == null { + return + } + v = vn.l + } else { + a = v + if vn.r == null { + return + } + v = vn.r + } + } +} + +// max returns the node with maximum value in the subtree with v as +// root. +func (t *binTree) max(v uint32) uint32 { + if v == null { + return null + } + for { + r := t.node[v].r + if r == null { + return v + } + v = r + } +} + +// min returns the node with the minimum value in the subtree with v as +// root. +func (t *binTree) min(v uint32) uint32 { + if v == null { + return null + } + for { + l := t.node[v].l + if l == null { + return v + } + v = l + } +} + +// pred returns the in-order predecessor of node v. +func (t *binTree) pred(v uint32) uint32 { + if v == null { + return null + } + u := t.max(t.node[v].l) + if u != null { + return u + } + for { + p := t.node[v].p + if p == null { + return null + } + if t.node[p].r == v { + return p + } + v = p + } +} + +// succ returns the in-order successor of node v. +func (t *binTree) succ(v uint32) uint32 { + if v == null { + return null + } + u := t.min(t.node[v].r) + if u != null { + return u + } + for { + p := t.node[v].p + if p == null { + return null + } + if t.node[p].l == v { + return p + } + v = p + } +} + +// xval converts the first four bytes of a into an 32-bit unsigned +// integer in big-endian order. +func xval(a []byte) uint32 { + var x uint32 + switch len(a) { + default: + x |= uint32(a[3]) + fallthrough + case 3: + x |= uint32(a[2]) << 8 + fallthrough + case 2: + x |= uint32(a[1]) << 16 + fallthrough + case 1: + x |= uint32(a[0]) << 24 + case 0: + } + return x +} + +// dumpX converts value x into a four-letter string. +func dumpX(x uint32) string { + a := make([]byte, 4) + for i := 0; i < 4; i++ { + c := byte(x >> uint((3-i)*8)) + if unicode.IsGraphic(rune(c)) { + a[i] = c + } else { + a[i] = '.' + } + } + return string(a) +} + +/* +// dumpNode writes a representation of the node v into the io.Writer. +func (t *binTree) dumpNode(w io.Writer, v uint32, indent int) { + if v == null { + return + } + + vn := &t.node[v] + + t.dumpNode(w, vn.r, indent+2) + + for i := 0; i < indent; i++ { + fmt.Fprint(w, " ") + } + if vn.p == null { + fmt.Fprintf(w, "node %d %q parent null\n", v, dumpX(vn.x)) + } else { + fmt.Fprintf(w, "node %d %q parent %d\n", v, dumpX(vn.x), vn.p) + } + + t.dumpNode(w, vn.l, indent+2) +} + +// dump prints a representation of the binary tree into the writer. +func (t *binTree) dump(w io.Writer) error { + bw := bufio.NewWriter(w) + t.dumpNode(bw, t.root, 0) + return bw.Flush() +} +*/ + +func (t *binTree) distance(v uint32) int { + dist := int(t.front) - int(v) + if dist <= 0 { + dist += len(t.node) + } + return dist +} + +type matchParams struct { + rep [4]uint32 + // length when match will be accepted + nAccept int + // nodes to check + check int + // finish if length get shorter + stopShorter bool +} + +func (t *binTree) match(m match, distIter func() (int, bool), p matchParams, +) (r match, checked int, accepted bool) { + buf := &t.dict.buf + for { + if checked >= p.check { + return m, checked, true + } + dist, ok := distIter() + if !ok { + return m, checked, false + } + checked++ + if m.n > 0 { + i := buf.rear - dist + m.n - 1 + if i < 0 { + i += len(buf.data) + } else if i >= len(buf.data) { + i -= len(buf.data) + } + if buf.data[i] != t.data[m.n-1] { + if p.stopShorter { + return m, checked, false + } + continue + } + } + n := buf.matchLen(dist, t.data) + switch n { + case 0: + if p.stopShorter { + return m, checked, false + } + continue + case 1: + if uint32(dist-minDistance) != p.rep[0] { + continue + } + } + if n < m.n || (n == m.n && int64(dist) >= m.distance) { + continue + } + m = match{int64(dist), n} + if n >= p.nAccept { + return m, checked, true + } + } +} + +func (t *binTree) NextOp(rep [4]uint32) operation { + // retrieve maxMatchLen data + n, _ := t.dict.buf.Peek(t.data[:maxMatchLen]) + if n == 0 { + panic("no data in buffer") + } + t.data = t.data[:n] + + var ( + m match + x, u, v uint32 + iterPred, iterSucc func() (int, bool) + ) + p := matchParams{ + rep: rep, + nAccept: maxMatchLen, + check: 32, + } + i := 4 + iterSmall := func() (dist int, ok bool) { + i-- + if i <= 0 { + return 0, false + } + return i, true + } + m, checked, accepted := t.match(m, iterSmall, p) + if accepted { + goto end + } + p.check -= checked + x = xval(t.data) + u, v = t.search(t.root, x) + if u == v && len(t.data) == 4 { + iter := func() (dist int, ok bool) { + if u == null { + return 0, false + } + dist = t.distance(u) + u, v = t.search(t.node[u].l, x) + if u != v { + u = null + } + return dist, true + } + m, _, _ = t.match(m, iter, p) + goto end + } + p.stopShorter = true + iterSucc = func() (dist int, ok bool) { + if v == null { + return 0, false + } + dist = t.distance(v) + v = t.succ(v) + return dist, true + } + m, checked, accepted = t.match(m, iterSucc, p) + if accepted { + goto end + } + p.check -= checked + iterPred = func() (dist int, ok bool) { + if u == null { + return 0, false + } + dist = t.distance(u) + u = t.pred(u) + return dist, true + } + m, _, _ = t.match(m, iterPred, p) +end: + if m.n == 0 { + return lit{t.data[0]} + } + return m +} diff --git a/vendor/github.com/ulikunitz/xz/lzma/bitops.go b/vendor/github.com/ulikunitz/xz/lzma/bitops.go new file mode 100644 index 00000000..20109170 --- /dev/null +++ b/vendor/github.com/ulikunitz/xz/lzma/bitops.go @@ -0,0 +1,47 @@ +// Copyright 2014-2022 Ulrich Kunitz. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package lzma + +/* Naming conventions follows the CodeReviewComments in the Go Wiki. */ + +// ntz32Const is used by the functions NTZ and NLZ. +const ntz32Const = 0x04d7651f + +// ntz32Table is a helper table for de Bruijn algorithm by Danny Dubé. +// See Henry S. Warren, Jr. "Hacker's Delight" section 5-1 figure 5-26. +var ntz32Table = [32]int8{ + 0, 1, 2, 24, 3, 19, 6, 25, + 22, 4, 20, 10, 16, 7, 12, 26, + 31, 23, 18, 5, 21, 9, 15, 11, + 30, 17, 8, 14, 29, 13, 28, 27, +} + +/* +// ntz32 computes the number of trailing zeros for an unsigned 32-bit integer. +func ntz32(x uint32) int { + if x == 0 { + return 32 + } + x = (x & -x) * ntz32Const + return int(ntz32Table[x>>27]) +} +*/ + +// nlz32 computes the number of leading zeros for an unsigned 32-bit integer. +func nlz32(x uint32) int { + // Smear left most bit to the right + x |= x >> 1 + x |= x >> 2 + x |= x >> 4 + x |= x >> 8 + x |= x >> 16 + // Use ntz mechanism to calculate nlz. + x++ + if x == 0 { + return 0 + } + x *= ntz32Const + return 32 - int(ntz32Table[x>>27]) +} diff --git a/vendor/github.com/ulikunitz/xz/lzma/breader.go b/vendor/github.com/ulikunitz/xz/lzma/breader.go new file mode 100644 index 00000000..81bd6718 --- /dev/null +++ b/vendor/github.com/ulikunitz/xz/lzma/breader.go @@ -0,0 +1,40 @@ +// Copyright 2014-2022 Ulrich Kunitz. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package lzma + +import ( + "io" +) + +// breader provides the ReadByte function for a Reader. It doesn't read +// more data from the reader than absolutely necessary. +type breader struct { + io.Reader + // helper slice to save allocations + p []byte +} + +// ByteReader converts an io.Reader into an io.ByteReader. +func ByteReader(r io.Reader) io.ByteReader { + br, ok := r.(io.ByteReader) + if !ok { + return &breader{r, make([]byte, 1)} + } + return br +} + +// ReadByte read byte function. +func (r *breader) ReadByte() (c byte, err error) { + for i := 0; i < 100; i++ { + n, err := r.Reader.Read(r.p) + if n > 0 { + return r.p[0], nil + } + if err != nil { + return 0, err + } + } + return 0, io.ErrNoProgress +} diff --git a/vendor/github.com/ulikunitz/xz/lzma/buffer.go b/vendor/github.com/ulikunitz/xz/lzma/buffer.go new file mode 100644 index 00000000..af41d5b2 --- /dev/null +++ b/vendor/github.com/ulikunitz/xz/lzma/buffer.go @@ -0,0 +1,171 @@ +// Copyright 2014-2022 Ulrich Kunitz. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package lzma + +import ( + "errors" +) + +// buffer provides a circular buffer of bytes. If the front index equals +// the rear index the buffer is empty. As a consequence front cannot be +// equal rear for a full buffer. So a full buffer has a length that is +// one byte less the the length of the data slice. +type buffer struct { + data []byte + front int + rear int +} + +// newBuffer creates a buffer with the given size. +func newBuffer(size int) *buffer { + return &buffer{data: make([]byte, size+1)} +} + +// Cap returns the capacity of the buffer. +func (b *buffer) Cap() int { + return len(b.data) - 1 +} + +// Resets the buffer. The front and rear index are set to zero. +func (b *buffer) Reset() { + b.front = 0 + b.rear = 0 +} + +// Buffered returns the number of bytes buffered. +func (b *buffer) Buffered() int { + delta := b.front - b.rear + if delta < 0 { + delta += len(b.data) + } + return delta +} + +// Available returns the number of bytes available for writing. +func (b *buffer) Available() int { + delta := b.rear - 1 - b.front + if delta < 0 { + delta += len(b.data) + } + return delta +} + +// addIndex adds a non-negative integer to the index i and returns the +// resulting index. The function takes care of wrapping the index as +// well as potential overflow situations. +func (b *buffer) addIndex(i int, n int) int { + // subtraction of len(b.data) prevents overflow + i += n - len(b.data) + if i < 0 { + i += len(b.data) + } + return i +} + +// Read reads bytes from the buffer into p and returns the number of +// bytes read. The function never returns an error but might return less +// data than requested. +func (b *buffer) Read(p []byte) (n int, err error) { + n, err = b.Peek(p) + b.rear = b.addIndex(b.rear, n) + return n, err +} + +// Peek reads bytes from the buffer into p without changing the buffer. +// Peek will never return an error but might return less data than +// requested. +func (b *buffer) Peek(p []byte) (n int, err error) { + m := b.Buffered() + n = len(p) + if m < n { + n = m + p = p[:n] + } + k := copy(p, b.data[b.rear:]) + if k < n { + copy(p[k:], b.data) + } + return n, nil +} + +// Discard skips the n next bytes to read from the buffer, returning the +// bytes discarded. +// +// If Discards skips fewer than n bytes, it returns an error. +func (b *buffer) Discard(n int) (discarded int, err error) { + if n < 0 { + return 0, errors.New("buffer.Discard: negative argument") + } + m := b.Buffered() + if m < n { + n = m + err = errors.New( + "buffer.Discard: discarded less bytes then requested") + } + b.rear = b.addIndex(b.rear, n) + return n, err +} + +// ErrNoSpace indicates that there is insufficient space for the Write +// operation. +var ErrNoSpace = errors.New("insufficient space") + +// Write puts data into the buffer. If less bytes are written than +// requested ErrNoSpace is returned. +func (b *buffer) Write(p []byte) (n int, err error) { + m := b.Available() + n = len(p) + if m < n { + n = m + p = p[:m] + err = ErrNoSpace + } + k := copy(b.data[b.front:], p) + if k < n { + copy(b.data, p[k:]) + } + b.front = b.addIndex(b.front, n) + return n, err +} + +// WriteByte writes a single byte into the buffer. The error ErrNoSpace +// is returned if no single byte is available in the buffer for writing. +func (b *buffer) WriteByte(c byte) error { + if b.Available() < 1 { + return ErrNoSpace + } + b.data[b.front] = c + b.front = b.addIndex(b.front, 1) + return nil +} + +// prefixLen returns the length of the common prefix of a and b. +func prefixLen(a, b []byte) int { + if len(a) > len(b) { + a, b = b, a + } + for i, c := range a { + if b[i] != c { + return i + } + } + return len(a) +} + +// matchLen returns the length of the common prefix for the given +// distance from the rear and the byte slice p. +func (b *buffer) matchLen(distance int, p []byte) int { + var n int + i := b.rear - distance + if i < 0 { + if n = prefixLen(p, b.data[len(b.data)+i:]); n < -i { + return n + } + p = p[n:] + i = 0 + } + n += prefixLen(p, b.data[i:]) + return n +} diff --git a/vendor/github.com/ulikunitz/xz/lzma/bytewriter.go b/vendor/github.com/ulikunitz/xz/lzma/bytewriter.go new file mode 100644 index 00000000..f27e31a4 --- /dev/null +++ b/vendor/github.com/ulikunitz/xz/lzma/bytewriter.go @@ -0,0 +1,37 @@ +// Copyright 2014-2022 Ulrich Kunitz. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package lzma + +import ( + "errors" + "io" +) + +// ErrLimit indicates that the limit of the LimitedByteWriter has been +// reached. +var ErrLimit = errors.New("limit reached") + +// LimitedByteWriter provides a byte writer that can be written until a +// limit is reached. The field N provides the number of remaining +// bytes. +type LimitedByteWriter struct { + BW io.ByteWriter + N int64 +} + +// WriteByte writes a single byte to the limited byte writer. It returns +// ErrLimit if the limit has been reached. If the byte is successfully +// written the field N of the LimitedByteWriter will be decremented by +// one. +func (l *LimitedByteWriter) WriteByte(c byte) error { + if l.N <= 0 { + return ErrLimit + } + if err := l.BW.WriteByte(c); err != nil { + return err + } + l.N-- + return nil +} diff --git a/vendor/github.com/ulikunitz/xz/lzma/decoder.go b/vendor/github.com/ulikunitz/xz/lzma/decoder.go new file mode 100644 index 00000000..3765484e --- /dev/null +++ b/vendor/github.com/ulikunitz/xz/lzma/decoder.go @@ -0,0 +1,277 @@ +// Copyright 2014-2022 Ulrich Kunitz. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package lzma + +import ( + "errors" + "fmt" + "io" +) + +// decoder decodes a raw LZMA stream without any header. +type decoder struct { + // dictionary; the rear pointer of the buffer will be used for + // reading the data. + Dict *decoderDict + // decoder state + State *state + // range decoder + rd *rangeDecoder + // start stores the head value of the dictionary for the LZMA + // stream + start int64 + // size of uncompressed data + size int64 + // end-of-stream encountered + eos bool + // EOS marker found + eosMarker bool +} + +// newDecoder creates a new decoder instance. The parameter size provides +// the expected byte size of the decompressed data. If the size is +// unknown use a negative value. In that case the decoder will look for +// a terminating end-of-stream marker. +func newDecoder(br io.ByteReader, state *state, dict *decoderDict, size int64) (d *decoder, err error) { + rd, err := newRangeDecoder(br) + if err != nil { + return nil, err + } + d = &decoder{ + State: state, + Dict: dict, + rd: rd, + size: size, + start: dict.pos(), + } + return d, nil +} + +// Reopen restarts the decoder with a new byte reader and a new size. Reopen +// resets the Decompressed counter to zero. +func (d *decoder) Reopen(br io.ByteReader, size int64) error { + var err error + if d.rd, err = newRangeDecoder(br); err != nil { + return err + } + d.start = d.Dict.pos() + d.size = size + d.eos = false + return nil +} + +// decodeLiteral decodes a single literal from the LZMA stream. +func (d *decoder) decodeLiteral() (op operation, err error) { + litState := d.State.litState(d.Dict.byteAt(1), d.Dict.head) + match := d.Dict.byteAt(int(d.State.rep[0]) + 1) + s, err := d.State.litCodec.Decode(d.rd, d.State.state, match, litState) + if err != nil { + return nil, err + } + return lit{s}, nil +} + +// errEOS indicates that an EOS marker has been found. +var errEOS = errors.New("EOS marker found") + +// readOp decodes the next operation from the compressed stream. It +// returns the operation. If an explicit end of stream marker is +// identified the eos error is returned. +func (d *decoder) readOp() (op operation, err error) { + // Value of the end of stream (EOS) marker + const eosDist = 1<<32 - 1 + + state, state2, posState := d.State.states(d.Dict.head) + + b, err := d.State.isMatch[state2].Decode(d.rd) + if err != nil { + return nil, err + } + if b == 0 { + // literal + op, err := d.decodeLiteral() + if err != nil { + return nil, err + } + d.State.updateStateLiteral() + return op, nil + } + b, err = d.State.isRep[state].Decode(d.rd) + if err != nil { + return nil, err + } + if b == 0 { + // simple match + d.State.rep[3], d.State.rep[2], d.State.rep[1] = + d.State.rep[2], d.State.rep[1], d.State.rep[0] + + d.State.updateStateMatch() + // The length decoder returns the length offset. + n, err := d.State.lenCodec.Decode(d.rd, posState) + if err != nil { + return nil, err + } + // The dist decoder returns the distance offset. The actual + // distance is 1 higher. + d.State.rep[0], err = d.State.distCodec.Decode(d.rd, n) + if err != nil { + return nil, err + } + if d.State.rep[0] == eosDist { + d.eosMarker = true + return nil, errEOS + } + op = match{n: int(n) + minMatchLen, + distance: int64(d.State.rep[0]) + minDistance} + return op, nil + } + b, err = d.State.isRepG0[state].Decode(d.rd) + if err != nil { + return nil, err + } + dist := d.State.rep[0] + if b == 0 { + // rep match 0 + b, err = d.State.isRepG0Long[state2].Decode(d.rd) + if err != nil { + return nil, err + } + if b == 0 { + d.State.updateStateShortRep() + op = match{n: 1, distance: int64(dist) + minDistance} + return op, nil + } + } else { + b, err = d.State.isRepG1[state].Decode(d.rd) + if err != nil { + return nil, err + } + if b == 0 { + dist = d.State.rep[1] + } else { + b, err = d.State.isRepG2[state].Decode(d.rd) + if err != nil { + return nil, err + } + if b == 0 { + dist = d.State.rep[2] + } else { + dist = d.State.rep[3] + d.State.rep[3] = d.State.rep[2] + } + d.State.rep[2] = d.State.rep[1] + } + d.State.rep[1] = d.State.rep[0] + d.State.rep[0] = dist + } + n, err := d.State.repLenCodec.Decode(d.rd, posState) + if err != nil { + return nil, err + } + d.State.updateStateRep() + op = match{n: int(n) + minMatchLen, distance: int64(dist) + minDistance} + return op, nil +} + +// apply takes the operation and transforms the decoder dictionary accordingly. +func (d *decoder) apply(op operation) error { + var err error + switch x := op.(type) { + case match: + err = d.Dict.writeMatch(x.distance, x.n) + case lit: + err = d.Dict.WriteByte(x.b) + default: + panic("op is neither a match nor a literal") + } + return err +} + +// decompress fills the dictionary unless no space for new data is +// available. If the end of the LZMA stream has been reached io.EOF will +// be returned. +func (d *decoder) decompress() error { + if d.eos { + return io.EOF + } + for d.Dict.Available() >= maxMatchLen { + op, err := d.readOp() + switch err { + case nil: + // break + case errEOS: + d.eos = true + if !d.rd.possiblyAtEnd() { + return errDataAfterEOS + } + if d.size >= 0 && d.size != d.Decompressed() { + return errSize + } + return io.EOF + case io.EOF: + d.eos = true + return io.ErrUnexpectedEOF + default: + return err + } + if err = d.apply(op); err != nil { + return err + } + if d.size >= 0 && d.Decompressed() >= d.size { + d.eos = true + if d.Decompressed() > d.size { + return errSize + } + if !d.rd.possiblyAtEnd() { + switch _, err = d.readOp(); err { + case nil: + return errSize + case io.EOF: + return io.ErrUnexpectedEOF + case errEOS: + break + default: + return err + } + } + return io.EOF + } + } + return nil +} + +// Errors that may be returned while decoding data. +var ( + errDataAfterEOS = errors.New("lzma: data after end of stream marker") + errSize = errors.New("lzma: wrong uncompressed data size") +) + +// Read reads data from the buffer. If no more data is available io.EOF is +// returned. +func (d *decoder) Read(p []byte) (n int, err error) { + var k int + for { + // Read of decoder dict never returns an error. + k, err = d.Dict.Read(p[n:]) + if err != nil { + panic(fmt.Errorf("dictionary read error %s", err)) + } + if k == 0 && d.eos { + return n, io.EOF + } + n += k + if n >= len(p) { + return n, nil + } + if err = d.decompress(); err != nil && err != io.EOF { + return n, err + } + } +} + +// Decompressed returns the number of bytes decompressed by the decoder. +func (d *decoder) Decompressed() int64 { + return d.Dict.pos() - d.start +} diff --git a/vendor/github.com/ulikunitz/xz/lzma/decoderdict.go b/vendor/github.com/ulikunitz/xz/lzma/decoderdict.go new file mode 100644 index 00000000..d5b814f0 --- /dev/null +++ b/vendor/github.com/ulikunitz/xz/lzma/decoderdict.go @@ -0,0 +1,128 @@ +// Copyright 2014-2022 Ulrich Kunitz. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package lzma + +import ( + "errors" + "fmt" +) + +// decoderDict provides the dictionary for the decoder. The whole +// dictionary is used as reader buffer. +type decoderDict struct { + buf buffer + head int64 +} + +// newDecoderDict creates a new decoder dictionary. The whole dictionary +// will be used as reader buffer. +func newDecoderDict(dictCap int) (d *decoderDict, err error) { + // lower limit supports easy test cases + if !(1 <= dictCap && int64(dictCap) <= MaxDictCap) { + return nil, errors.New("lzma: dictCap out of range") + } + d = &decoderDict{buf: *newBuffer(dictCap)} + return d, nil +} + +// Reset clears the dictionary. The read buffer is not changed, so the +// buffered data can still be read. +func (d *decoderDict) Reset() { + d.head = 0 +} + +// WriteByte writes a single byte into the dictionary. It is used to +// write literals into the dictionary. +func (d *decoderDict) WriteByte(c byte) error { + if err := d.buf.WriteByte(c); err != nil { + return err + } + d.head++ + return nil +} + +// pos returns the position of the dictionary head. +func (d *decoderDict) pos() int64 { return d.head } + +// dictLen returns the actual length of the dictionary. +func (d *decoderDict) dictLen() int { + capacity := d.buf.Cap() + if d.head >= int64(capacity) { + return capacity + } + return int(d.head) +} + +// byteAt returns a byte stored in the dictionary. If the distance is +// non-positive or exceeds the current length of the dictionary the zero +// byte is returned. +func (d *decoderDict) byteAt(dist int) byte { + if !(0 < dist && dist <= d.dictLen()) { + return 0 + } + i := d.buf.front - dist + if i < 0 { + i += len(d.buf.data) + } + return d.buf.data[i] +} + +// writeMatch writes the match at the top of the dictionary. The given +// distance must point in the current dictionary and the length must not +// exceed the maximum length 273 supported in LZMA. +// +// The error value ErrNoSpace indicates that no space is available in +// the dictionary for writing. You need to read from the dictionary +// first. +func (d *decoderDict) writeMatch(dist int64, length int) error { + if !(0 < dist && dist <= int64(d.dictLen())) { + return errors.New("writeMatch: distance out of range") + } + if !(0 < length && length <= maxMatchLen) { + return errors.New("writeMatch: length out of range") + } + if length > d.buf.Available() { + return ErrNoSpace + } + d.head += int64(length) + + i := d.buf.front - int(dist) + if i < 0 { + i += len(d.buf.data) + } + for length > 0 { + var p []byte + if i >= d.buf.front { + p = d.buf.data[i:] + i = 0 + } else { + p = d.buf.data[i:d.buf.front] + i = d.buf.front + } + if len(p) > length { + p = p[:length] + } + if _, err := d.buf.Write(p); err != nil { + panic(fmt.Errorf("d.buf.Write returned error %s", err)) + } + length -= len(p) + } + return nil +} + +// Write writes the given bytes into the dictionary and advances the +// head. +func (d *decoderDict) Write(p []byte) (n int, err error) { + n, err = d.buf.Write(p) + d.head += int64(n) + return n, err +} + +// Available returns the number of available bytes for writing into the +// decoder dictionary. +func (d *decoderDict) Available() int { return d.buf.Available() } + +// Read reads data from the buffer contained in the decoder dictionary. +func (d *decoderDict) Read(p []byte) (n int, err error) { return d.buf.Read(p) } diff --git a/vendor/github.com/ulikunitz/xz/lzma/directcodec.go b/vendor/github.com/ulikunitz/xz/lzma/directcodec.go new file mode 100644 index 00000000..76b71310 --- /dev/null +++ b/vendor/github.com/ulikunitz/xz/lzma/directcodec.go @@ -0,0 +1,38 @@ +// Copyright 2014-2022 Ulrich Kunitz. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package lzma + +// directCodec allows the encoding and decoding of values with a fixed number +// of bits. The number of bits must be in the range [1,32]. +type directCodec byte + +// Bits returns the number of bits supported by this codec. +func (dc directCodec) Bits() int { + return int(dc) +} + +// Encode uses the range encoder to encode a value with the fixed number of +// bits. The most-significant bit is encoded first. +func (dc directCodec) Encode(e *rangeEncoder, v uint32) error { + for i := int(dc) - 1; i >= 0; i-- { + if err := e.DirectEncodeBit(v >> uint(i)); err != nil { + return err + } + } + return nil +} + +// Decode uses the range decoder to decode a value with the given number of +// given bits. The most-significant bit is decoded first. +func (dc directCodec) Decode(d *rangeDecoder) (v uint32, err error) { + for i := int(dc) - 1; i >= 0; i-- { + x, err := d.DirectDecodeBit() + if err != nil { + return 0, err + } + v = (v << 1) | x + } + return v, nil +} diff --git a/vendor/github.com/ulikunitz/xz/lzma/distcodec.go b/vendor/github.com/ulikunitz/xz/lzma/distcodec.go new file mode 100644 index 00000000..b447d8ec --- /dev/null +++ b/vendor/github.com/ulikunitz/xz/lzma/distcodec.go @@ -0,0 +1,140 @@ +// Copyright 2014-2022 Ulrich Kunitz. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package lzma + +// Constants used by the distance codec. +const ( + // minimum supported distance + minDistance = 1 + // maximum supported distance, value is used for the eos marker. + maxDistance = 1 << 32 + // number of the supported len states + lenStates = 4 + // start for the position models + startPosModel = 4 + // first index with align bits support + endPosModel = 14 + // bits for the position slots + posSlotBits = 6 + // number of align bits + alignBits = 4 +) + +// distCodec provides encoding and decoding of distance values. +type distCodec struct { + posSlotCodecs [lenStates]treeCodec + posModel [endPosModel - startPosModel]treeReverseCodec + alignCodec treeReverseCodec +} + +// deepcopy initializes dc as deep copy of the source. +func (dc *distCodec) deepcopy(src *distCodec) { + if dc == src { + return + } + for i := range dc.posSlotCodecs { + dc.posSlotCodecs[i].deepcopy(&src.posSlotCodecs[i]) + } + for i := range dc.posModel { + dc.posModel[i].deepcopy(&src.posModel[i]) + } + dc.alignCodec.deepcopy(&src.alignCodec) +} + +// newDistCodec creates a new distance codec. +func (dc *distCodec) init() { + for i := range dc.posSlotCodecs { + dc.posSlotCodecs[i] = makeTreeCodec(posSlotBits) + } + for i := range dc.posModel { + posSlot := startPosModel + i + bits := (posSlot >> 1) - 1 + dc.posModel[i] = makeTreeReverseCodec(bits) + } + dc.alignCodec = makeTreeReverseCodec(alignBits) +} + +// lenState converts the value l to a supported lenState value. +func lenState(l uint32) uint32 { + if l >= lenStates { + l = lenStates - 1 + } + return l +} + +// Encode encodes the distance using the parameter l. Dist can have values from +// the full range of uint32 values. To get the distance offset the actual match +// distance has to be decreased by 1. A distance offset of 0xffffffff (eos) +// indicates the end of the stream. +func (dc *distCodec) Encode(e *rangeEncoder, dist uint32, l uint32) (err error) { + // Compute the posSlot using nlz32 + var posSlot uint32 + var bits uint32 + if dist < startPosModel { + posSlot = dist + } else { + bits = uint32(30 - nlz32(dist)) + posSlot = startPosModel - 2 + (bits << 1) + posSlot += (dist >> uint(bits)) & 1 + } + + if err = dc.posSlotCodecs[lenState(l)].Encode(e, posSlot); err != nil { + return + } + + switch { + case posSlot < startPosModel: + return nil + case posSlot < endPosModel: + tc := &dc.posModel[posSlot-startPosModel] + return tc.Encode(dist, e) + } + dic := directCodec(bits - alignBits) + if err = dic.Encode(e, dist>>alignBits); err != nil { + return + } + return dc.alignCodec.Encode(dist, e) +} + +// Decode decodes the distance offset using the parameter l. The dist value +// 0xffffffff (eos) indicates the end of the stream. Add one to the distance +// offset to get the actual match distance. +func (dc *distCodec) Decode(d *rangeDecoder, l uint32) (dist uint32, err error) { + posSlot, err := dc.posSlotCodecs[lenState(l)].Decode(d) + if err != nil { + return + } + + // posSlot equals distance + if posSlot < startPosModel { + return posSlot, nil + } + + // posSlot uses the individual models + bits := (posSlot >> 1) - 1 + dist = (2 | (posSlot & 1)) << bits + var u uint32 + if posSlot < endPosModel { + tc := &dc.posModel[posSlot-startPosModel] + if u, err = tc.Decode(d); err != nil { + return 0, err + } + dist += u + return dist, nil + } + + // posSlots use direct encoding and a single model for the four align + // bits. + dic := directCodec(bits - alignBits) + if u, err = dic.Decode(d); err != nil { + return 0, err + } + dist += u << alignBits + if u, err = dc.alignCodec.Decode(d); err != nil { + return 0, err + } + dist += u + return dist, nil +} diff --git a/vendor/github.com/ulikunitz/xz/lzma/encoder.go b/vendor/github.com/ulikunitz/xz/lzma/encoder.go new file mode 100644 index 00000000..e4093831 --- /dev/null +++ b/vendor/github.com/ulikunitz/xz/lzma/encoder.go @@ -0,0 +1,268 @@ +// Copyright 2014-2022 Ulrich Kunitz. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package lzma + +import ( + "fmt" + "io" +) + +// opLenMargin provides the upper limit of the number of bytes required +// to encode a single operation. +const opLenMargin = 16 + +// compressFlags control the compression process. +type compressFlags uint32 + +// Values for compressFlags. +const ( + // all data should be compressed, even if compression is not + // optimal. + all compressFlags = 1 << iota +) + +// encoderFlags provide the flags for an encoder. +type encoderFlags uint32 + +// Flags for the encoder. +const ( + // eosMarker requests an EOS marker to be written. + eosMarker encoderFlags = 1 << iota +) + +// Encoder compresses data buffered in the encoder dictionary and writes +// it into a byte writer. +type encoder struct { + dict *encoderDict + state *state + re *rangeEncoder + start int64 + // generate eos marker + marker bool + limit bool + margin int +} + +// newEncoder creates a new encoder. If the byte writer must be +// limited use LimitedByteWriter provided by this package. The flags +// argument supports the eosMarker flag, controlling whether a +// terminating end-of-stream marker must be written. +func newEncoder(bw io.ByteWriter, state *state, dict *encoderDict, + flags encoderFlags) (e *encoder, err error) { + + re, err := newRangeEncoder(bw) + if err != nil { + return nil, err + } + e = &encoder{ + dict: dict, + state: state, + re: re, + marker: flags&eosMarker != 0, + start: dict.Pos(), + margin: opLenMargin, + } + if e.marker { + e.margin += 5 + } + return e, nil +} + +// Write writes the bytes from p into the dictionary. If not enough +// space is available the data in the dictionary buffer will be +// compressed to make additional space available. If the limit of the +// underlying writer has been reached ErrLimit will be returned. +func (e *encoder) Write(p []byte) (n int, err error) { + for { + k, err := e.dict.Write(p[n:]) + n += k + if err == ErrNoSpace { + if err = e.compress(0); err != nil { + return n, err + } + continue + } + return n, err + } +} + +// Reopen reopens the encoder with a new byte writer. +func (e *encoder) Reopen(bw io.ByteWriter) error { + var err error + if e.re, err = newRangeEncoder(bw); err != nil { + return err + } + e.start = e.dict.Pos() + e.limit = false + return nil +} + +// writeLiteral writes a literal into the LZMA stream +func (e *encoder) writeLiteral(l lit) error { + var err error + state, state2, _ := e.state.states(e.dict.Pos()) + if err = e.state.isMatch[state2].Encode(e.re, 0); err != nil { + return err + } + litState := e.state.litState(e.dict.ByteAt(1), e.dict.Pos()) + match := e.dict.ByteAt(int(e.state.rep[0]) + 1) + err = e.state.litCodec.Encode(e.re, l.b, state, match, litState) + if err != nil { + return err + } + e.state.updateStateLiteral() + return nil +} + +// iverson implements the Iverson operator as proposed by Donald Knuth in his +// book Concrete Mathematics. +func iverson(ok bool) uint32 { + if ok { + return 1 + } + return 0 +} + +// writeMatch writes a repetition operation into the operation stream +func (e *encoder) writeMatch(m match) error { + var err error + if !(minDistance <= m.distance && m.distance <= maxDistance) { + panic(fmt.Errorf("match distance %d out of range", m.distance)) + } + dist := uint32(m.distance - minDistance) + if !(minMatchLen <= m.n && m.n <= maxMatchLen) && + !(dist == e.state.rep[0] && m.n == 1) { + panic(fmt.Errorf( + "match length %d out of range; dist %d rep[0] %d", + m.n, dist, e.state.rep[0])) + } + state, state2, posState := e.state.states(e.dict.Pos()) + if err = e.state.isMatch[state2].Encode(e.re, 1); err != nil { + return err + } + g := 0 + for ; g < 4; g++ { + if e.state.rep[g] == dist { + break + } + } + b := iverson(g < 4) + if err = e.state.isRep[state].Encode(e.re, b); err != nil { + return err + } + n := uint32(m.n - minMatchLen) + if b == 0 { + // simple match + e.state.rep[3], e.state.rep[2], e.state.rep[1], e.state.rep[0] = + e.state.rep[2], e.state.rep[1], e.state.rep[0], dist + e.state.updateStateMatch() + if err = e.state.lenCodec.Encode(e.re, n, posState); err != nil { + return err + } + return e.state.distCodec.Encode(e.re, dist, n) + } + b = iverson(g != 0) + if err = e.state.isRepG0[state].Encode(e.re, b); err != nil { + return err + } + if b == 0 { + // g == 0 + b = iverson(m.n != 1) + if err = e.state.isRepG0Long[state2].Encode(e.re, b); err != nil { + return err + } + if b == 0 { + e.state.updateStateShortRep() + return nil + } + } else { + // g in {1,2,3} + b = iverson(g != 1) + if err = e.state.isRepG1[state].Encode(e.re, b); err != nil { + return err + } + if b == 1 { + // g in {2,3} + b = iverson(g != 2) + err = e.state.isRepG2[state].Encode(e.re, b) + if err != nil { + return err + } + if b == 1 { + e.state.rep[3] = e.state.rep[2] + } + e.state.rep[2] = e.state.rep[1] + } + e.state.rep[1] = e.state.rep[0] + e.state.rep[0] = dist + } + e.state.updateStateRep() + return e.state.repLenCodec.Encode(e.re, n, posState) +} + +// writeOp writes a single operation to the range encoder. The function +// checks whether there is enough space available to close the LZMA +// stream. +func (e *encoder) writeOp(op operation) error { + if e.re.Available() < int64(e.margin) { + return ErrLimit + } + switch x := op.(type) { + case lit: + return e.writeLiteral(x) + case match: + return e.writeMatch(x) + default: + panic("unexpected operation") + } +} + +// compress compressed data from the dictionary buffer. If the flag all +// is set, all data in the dictionary buffer will be compressed. The +// function returns ErrLimit if the underlying writer has reached its +// limit. +func (e *encoder) compress(flags compressFlags) error { + n := 0 + if flags&all == 0 { + n = maxMatchLen - 1 + } + d := e.dict + m := d.m + for d.Buffered() > n { + op := m.NextOp(e.state.rep) + if err := e.writeOp(op); err != nil { + return err + } + d.Discard(op.Len()) + } + return nil +} + +// eosMatch is a pseudo operation that indicates the end of the stream. +var eosMatch = match{distance: maxDistance, n: minMatchLen} + +// Close terminates the LZMA stream. If requested the end-of-stream +// marker will be written. If the byte writer limit has been or will be +// reached during compression of the remaining data in the buffer the +// LZMA stream will be closed and data will remain in the buffer. +func (e *encoder) Close() error { + err := e.compress(all) + if err != nil && err != ErrLimit { + return err + } + if e.marker { + if err := e.writeMatch(eosMatch); err != nil { + return err + } + } + err = e.re.Close() + return err +} + +// Compressed returns the number bytes of the input data that been +// compressed. +func (e *encoder) Compressed() int64 { + return e.dict.Pos() - e.start +} diff --git a/vendor/github.com/ulikunitz/xz/lzma/encoderdict.go b/vendor/github.com/ulikunitz/xz/lzma/encoderdict.go new file mode 100644 index 00000000..4b3916ea --- /dev/null +++ b/vendor/github.com/ulikunitz/xz/lzma/encoderdict.go @@ -0,0 +1,149 @@ +// Copyright 2014-2022 Ulrich Kunitz. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package lzma + +import ( + "errors" + "fmt" + "io" +) + +// matcher is an interface that supports the identification of the next +// operation. +type matcher interface { + io.Writer + SetDict(d *encoderDict) + NextOp(rep [4]uint32) operation +} + +// encoderDict provides the dictionary of the encoder. It includes an +// additional buffer atop of the actual dictionary. +type encoderDict struct { + buf buffer + m matcher + head int64 + capacity int + // preallocated array + data [maxMatchLen]byte +} + +// newEncoderDict creates the encoder dictionary. The argument bufSize +// defines the size of the additional buffer. +func newEncoderDict(dictCap, bufSize int, m matcher) (d *encoderDict, err error) { + if !(1 <= dictCap && int64(dictCap) <= MaxDictCap) { + return nil, errors.New( + "lzma: dictionary capacity out of range") + } + if bufSize < 1 { + return nil, errors.New( + "lzma: buffer size must be larger than zero") + } + d = &encoderDict{ + buf: *newBuffer(dictCap + bufSize), + capacity: dictCap, + m: m, + } + m.SetDict(d) + return d, nil +} + +// Discard discards n bytes. Note that n must not be larger than +// MaxMatchLen. +func (d *encoderDict) Discard(n int) { + p := d.data[:n] + k, _ := d.buf.Read(p) + if k < n { + panic(fmt.Errorf("lzma: can't discard %d bytes", n)) + } + d.head += int64(n) + d.m.Write(p) +} + +// Len returns the data available in the encoder dictionary. +func (d *encoderDict) Len() int { + n := d.buf.Available() + if int64(n) > d.head { + return int(d.head) + } + return n +} + +// DictLen returns the actual length of data in the dictionary. +func (d *encoderDict) DictLen() int { + if d.head < int64(d.capacity) { + return int(d.head) + } + return d.capacity +} + +// Available returns the number of bytes that can be written by a +// following Write call. +func (d *encoderDict) Available() int { + return d.buf.Available() - d.DictLen() +} + +// Write writes data into the dictionary buffer. Note that the position +// of the dictionary head will not be moved. If there is not enough +// space in the buffer ErrNoSpace will be returned. +func (d *encoderDict) Write(p []byte) (n int, err error) { + m := d.Available() + if len(p) > m { + p = p[:m] + err = ErrNoSpace + } + var e error + if n, e = d.buf.Write(p); e != nil { + err = e + } + return n, err +} + +// Pos returns the position of the head. +func (d *encoderDict) Pos() int64 { return d.head } + +// ByteAt returns the byte at the given distance. +func (d *encoderDict) ByteAt(distance int) byte { + if !(0 < distance && distance <= d.Len()) { + return 0 + } + i := d.buf.rear - distance + if i < 0 { + i += len(d.buf.data) + } + return d.buf.data[i] +} + +// CopyN copies the last n bytes from the dictionary into the provided +// writer. This is used for copying uncompressed data into an +// uncompressed segment. +func (d *encoderDict) CopyN(w io.Writer, n int) (written int, err error) { + if n <= 0 { + return 0, nil + } + m := d.Len() + if n > m { + n = m + err = ErrNoSpace + } + i := d.buf.rear - n + var e error + if i < 0 { + i += len(d.buf.data) + if written, e = w.Write(d.buf.data[i:]); e != nil { + return written, e + } + i = 0 + } + var k int + k, e = w.Write(d.buf.data[i:d.buf.rear]) + written += k + if e != nil { + err = e + } + return written, err +} + +// Buffered returns the number of bytes in the buffer. +func (d *encoderDict) Buffered() int { return d.buf.Buffered() } diff --git a/vendor/github.com/ulikunitz/xz/lzma/fox.lzma b/vendor/github.com/ulikunitz/xz/lzma/fox.lzma new file mode 100644 index 0000000000000000000000000000000000000000..5edad633266eb5173a7c39761dc8b9e71efbfe80 GIT binary patch literal 67 zcma!LU}#|Y4+RWbQXGqzRntCtR~%i$`d{za%}WYWYfXMUl6~Q5_UjH?=5CuO0w(I5 UuQ#VXelz{mI_3ZW`W7$%0HEw6g#Z8m literal 0 HcmV?d00001 diff --git a/vendor/github.com/ulikunitz/xz/lzma/hashtable.go b/vendor/github.com/ulikunitz/xz/lzma/hashtable.go new file mode 100644 index 00000000..f66e9cdd --- /dev/null +++ b/vendor/github.com/ulikunitz/xz/lzma/hashtable.go @@ -0,0 +1,309 @@ +// Copyright 2014-2022 Ulrich Kunitz. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package lzma + +import ( + "errors" + "fmt" + + "github.com/ulikunitz/xz/internal/hash" +) + +/* For compression we need to find byte sequences that match the byte + * sequence at the dictionary head. A hash table is a simple method to + * provide this capability. + */ + +// maxMatches limits the number of matches requested from the Matches +// function. This controls the speed of the overall encoding. +const maxMatches = 16 + +// shortDists defines the number of short distances supported by the +// implementation. +const shortDists = 8 + +// The minimum is somehow arbitrary but the maximum is limited by the +// memory requirements of the hash table. +const ( + minTableExponent = 9 + maxTableExponent = 20 +) + +// newRoller contains the function used to create an instance of the +// hash.Roller. +var newRoller = func(n int) hash.Roller { return hash.NewCyclicPoly(n) } + +// hashTable stores the hash table including the rolling hash method. +// +// We implement chained hashing into a circular buffer. Each entry in +// the circular buffer stores the delta distance to the next position with a +// word that has the same hash value. +type hashTable struct { + dict *encoderDict + // actual hash table + t []int64 + // circular list data with the offset to the next word + data []uint32 + front int + // mask for computing the index for the hash table + mask uint64 + // hash offset; initial value is -int64(wordLen) + hoff int64 + // length of the hashed word + wordLen int + // hash roller for computing the hash values for the Write + // method + wr hash.Roller + // hash roller for computing arbitrary hashes + hr hash.Roller + // preallocated slices + p [maxMatches]int64 + distances [maxMatches + shortDists]int +} + +// hashTableExponent derives the hash table exponent from the dictionary +// capacity. +func hashTableExponent(n uint32) int { + e := 30 - nlz32(n) + switch { + case e < minTableExponent: + e = minTableExponent + case e > maxTableExponent: + e = maxTableExponent + } + return e +} + +// newHashTable creates a new hash table for words of length wordLen +func newHashTable(capacity int, wordLen int) (t *hashTable, err error) { + if !(0 < capacity) { + return nil, errors.New( + "newHashTable: capacity must not be negative") + } + exp := hashTableExponent(uint32(capacity)) + if !(1 <= wordLen && wordLen <= 4) { + return nil, errors.New("newHashTable: " + + "argument wordLen out of range") + } + n := 1 << uint(exp) + if n <= 0 { + panic("newHashTable: exponent is too large") + } + t = &hashTable{ + t: make([]int64, n), + data: make([]uint32, capacity), + mask: (uint64(1) << uint(exp)) - 1, + hoff: -int64(wordLen), + wordLen: wordLen, + wr: newRoller(wordLen), + hr: newRoller(wordLen), + } + return t, nil +} + +func (t *hashTable) SetDict(d *encoderDict) { t.dict = d } + +// buffered returns the number of bytes that are currently hashed. +func (t *hashTable) buffered() int { + n := t.hoff + 1 + switch { + case n <= 0: + return 0 + case n >= int64(len(t.data)): + return len(t.data) + } + return int(n) +} + +// addIndex adds n to an index ensuring that is stays inside the +// circular buffer for the hash chain. +func (t *hashTable) addIndex(i, n int) int { + i += n - len(t.data) + if i < 0 { + i += len(t.data) + } + return i +} + +// putDelta puts the delta instance at the current front of the circular +// chain buffer. +func (t *hashTable) putDelta(delta uint32) { + t.data[t.front] = delta + t.front = t.addIndex(t.front, 1) +} + +// putEntry puts a new entry into the hash table. If there is already a +// value stored it is moved into the circular chain buffer. +func (t *hashTable) putEntry(h uint64, pos int64) { + if pos < 0 { + return + } + i := h & t.mask + old := t.t[i] - 1 + t.t[i] = pos + 1 + var delta int64 + if old >= 0 { + delta = pos - old + if delta > 1<<32-1 || delta > int64(t.buffered()) { + delta = 0 + } + } + t.putDelta(uint32(delta)) +} + +// WriteByte converts a single byte into a hash and puts them into the hash +// table. +func (t *hashTable) WriteByte(b byte) error { + h := t.wr.RollByte(b) + t.hoff++ + t.putEntry(h, t.hoff) + return nil +} + +// Write converts the bytes provided into hash tables and stores the +// abbreviated offsets into the hash table. The method will never return an +// error. +func (t *hashTable) Write(p []byte) (n int, err error) { + for _, b := range p { + // WriteByte doesn't generate an error. + t.WriteByte(b) + } + return len(p), nil +} + +// getMatches the matches for a specific hash. The functions returns the +// number of positions found. +// +// TODO: Make a getDistances because that we are actually interested in. +func (t *hashTable) getMatches(h uint64, positions []int64) (n int) { + if t.hoff < 0 || len(positions) == 0 { + return 0 + } + buffered := t.buffered() + tailPos := t.hoff + 1 - int64(buffered) + rear := t.front - buffered + if rear >= 0 { + rear -= len(t.data) + } + // get the slot for the hash + pos := t.t[h&t.mask] - 1 + delta := pos - tailPos + for { + if delta < 0 { + return n + } + positions[n] = tailPos + delta + n++ + if n >= len(positions) { + return n + } + i := rear + int(delta) + if i < 0 { + i += len(t.data) + } + u := t.data[i] + if u == 0 { + return n + } + delta -= int64(u) + } +} + +// hash computes the rolling hash for the word stored in p. For correct +// results its length must be equal to t.wordLen. +func (t *hashTable) hash(p []byte) uint64 { + var h uint64 + for _, b := range p { + h = t.hr.RollByte(b) + } + return h +} + +// Matches fills the positions slice with potential matches. The +// functions returns the number of positions filled into positions. The +// byte slice p must have word length of the hash table. +func (t *hashTable) Matches(p []byte, positions []int64) int { + if len(p) != t.wordLen { + panic(fmt.Errorf( + "byte slice must have length %d", t.wordLen)) + } + h := t.hash(p) + return t.getMatches(h, positions) +} + +// NextOp identifies the next operation using the hash table. +// +// TODO: Use all repetitions to find matches. +func (t *hashTable) NextOp(rep [4]uint32) operation { + // get positions + data := t.dict.data[:maxMatchLen] + n, _ := t.dict.buf.Peek(data) + data = data[:n] + var p []int64 + if n < t.wordLen { + p = t.p[:0] + } else { + p = t.p[:maxMatches] + n = t.Matches(data[:t.wordLen], p) + p = p[:n] + } + + // convert positions in potential distances + head := t.dict.head + dists := append(t.distances[:0], 1, 2, 3, 4, 5, 6, 7, 8) + for _, pos := range p { + dis := int(head - pos) + if dis > shortDists { + dists = append(dists, dis) + } + } + + // check distances + var m match + dictLen := t.dict.DictLen() + for _, dist := range dists { + if dist > dictLen { + continue + } + + // Here comes a trick. We are only interested in matches + // that are longer than the matches we have been found + // before. So before we test the whole byte sequence at + // the given distance, we test the first byte that would + // make the match longer. If it doesn't match the byte + // to match, we don't to care any longer. + i := t.dict.buf.rear - dist + m.n + if i < 0 { + i += len(t.dict.buf.data) + } + if t.dict.buf.data[i] != data[m.n] { + // We can't get a longer match. Jump to the next + // distance. + continue + } + + n := t.dict.buf.matchLen(dist, data) + switch n { + case 0: + continue + case 1: + if uint32(dist-minDistance) != rep[0] { + continue + } + } + if n > m.n { + m = match{int64(dist), n} + if n == len(data) { + // No better match will be found. + break + } + } + } + + if m.n == 0 { + return lit{data[0]} + } + return m +} diff --git a/vendor/github.com/ulikunitz/xz/lzma/header.go b/vendor/github.com/ulikunitz/xz/lzma/header.go new file mode 100644 index 00000000..34aa097e --- /dev/null +++ b/vendor/github.com/ulikunitz/xz/lzma/header.go @@ -0,0 +1,170 @@ +// Copyright 2014-2022 Ulrich Kunitz. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package lzma + +import ( + "errors" + "fmt" +) + +// uint32LE reads an uint32 integer from a byte slice +func uint32LE(b []byte) uint32 { + x := uint32(b[3]) << 24 + x |= uint32(b[2]) << 16 + x |= uint32(b[1]) << 8 + x |= uint32(b[0]) + return x +} + +// uint64LE converts the uint64 value stored as little endian to an uint64 +// value. +func uint64LE(b []byte) uint64 { + x := uint64(b[7]) << 56 + x |= uint64(b[6]) << 48 + x |= uint64(b[5]) << 40 + x |= uint64(b[4]) << 32 + x |= uint64(b[3]) << 24 + x |= uint64(b[2]) << 16 + x |= uint64(b[1]) << 8 + x |= uint64(b[0]) + return x +} + +// putUint32LE puts an uint32 integer into a byte slice that must have at least +// a length of 4 bytes. +func putUint32LE(b []byte, x uint32) { + b[0] = byte(x) + b[1] = byte(x >> 8) + b[2] = byte(x >> 16) + b[3] = byte(x >> 24) +} + +// putUint64LE puts the uint64 value into the byte slice as little endian +// value. The byte slice b must have at least place for 8 bytes. +func putUint64LE(b []byte, x uint64) { + b[0] = byte(x) + b[1] = byte(x >> 8) + b[2] = byte(x >> 16) + b[3] = byte(x >> 24) + b[4] = byte(x >> 32) + b[5] = byte(x >> 40) + b[6] = byte(x >> 48) + b[7] = byte(x >> 56) +} + +// noHeaderSize defines the value of the length field in the LZMA header. +const noHeaderSize uint64 = 1<<64 - 1 + +// HeaderLen provides the length of the LZMA file header. +const HeaderLen = 13 + +// Header represents the Header of an LZMA file. +type Header struct { + Properties Properties + DictSize uint32 + // uncompressed Size; negative value if no Size is given + Size int64 +} + +// marshalBinary marshals the header. +func (h *Header) marshalBinary() (data []byte, err error) { + if err = h.Properties.verify(); err != nil { + return nil, err + } + if !(h.DictSize <= MaxDictCap) { + return nil, fmt.Errorf("lzma: DictCap %d out of range", + h.DictSize) + } + + data = make([]byte, 13) + + // property byte + data[0] = h.Properties.Code() + + // dictionary capacity + putUint32LE(data[1:5], uint32(h.DictSize)) + + // uncompressed size + var s uint64 + if h.Size > 0 { + s = uint64(h.Size) + } else { + s = noHeaderSize + } + putUint64LE(data[5:], s) + + return data, nil +} + +// unmarshalBinary unmarshals the header. +func (h *Header) unmarshalBinary(data []byte) error { + if len(data) != HeaderLen { + return errors.New("lzma.unmarshalBinary: data has wrong length") + } + + // properties + var err error + if h.Properties, err = PropertiesForCode(data[0]); err != nil { + return err + } + + // dictionary capacity + h.DictSize = uint32LE(data[1:]) + if int(h.DictSize) < 0 { + return errors.New( + "LZMA header: dictionary capacity exceeds maximum " + + "integer") + } + + // uncompressed size + s := uint64LE(data[5:]) + if s == noHeaderSize { + h.Size = -1 + } else { + h.Size = int64(s) + if h.Size < 0 { + return errors.New( + "LZMA header: uncompressed size " + + "out of int64 range") + } + } + + return nil +} + +// validDictSize checks whether the dictionary capacity is correct. This +// is used to weed out wrong file headers. +func validDictSize(dictcap int) bool { + if int64(dictcap) == MaxDictCap { + return true + } + for n := uint(10); n < 32; n++ { + if dictcap == 1<= 10 or 2^32-1. If +// there is an explicit size it must not exceed 256 GiB. The length of +// the data argument must be HeaderLen. +// +// This function should be disregarded because there is no guarantee that LZMA +// files follow the constraints. +func ValidHeader(data []byte) bool { + var h Header + if err := h.unmarshalBinary(data); err != nil { + return false + } + if !validDictSize(int(h.DictSize)) { + return false + } + return h.Size < 0 || h.Size <= 1<<38 +} diff --git a/vendor/github.com/ulikunitz/xz/lzma/header2.go b/vendor/github.com/ulikunitz/xz/lzma/header2.go new file mode 100644 index 00000000..081fc840 --- /dev/null +++ b/vendor/github.com/ulikunitz/xz/lzma/header2.go @@ -0,0 +1,398 @@ +// Copyright 2014-2022 Ulrich Kunitz. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package lzma + +import ( + "errors" + "fmt" + "io" +) + +const ( + // maximum size of compressed data in a chunk + maxCompressed = 1 << 16 + // maximum size of uncompressed data in a chunk + maxUncompressed = 1 << 21 +) + +// chunkType represents the type of an LZMA2 chunk. Note that this +// value is an internal representation and no actual encoding of a LZMA2 +// chunk header. +type chunkType byte + +// Possible values for the chunk type. +const ( + // end of stream + cEOS chunkType = iota + // uncompressed; reset dictionary + cUD + // uncompressed; no reset of dictionary + cU + // LZMA compressed; no reset + cL + // LZMA compressed; reset state + cLR + // LZMA compressed; reset state; new property value + cLRN + // LZMA compressed; reset state; new property value; reset dictionary + cLRND +) + +// chunkTypeStrings provide a string representation for the chunk types. +var chunkTypeStrings = [...]string{ + cEOS: "EOS", + cU: "U", + cUD: "UD", + cL: "L", + cLR: "LR", + cLRN: "LRN", + cLRND: "LRND", +} + +// String returns a string representation of the chunk type. +func (c chunkType) String() string { + if !(cEOS <= c && c <= cLRND) { + return "unknown" + } + return chunkTypeStrings[c] +} + +// Actual encodings for the chunk types in the value. Note that the high +// uncompressed size bits are stored in the header byte additionally. +const ( + hEOS = 0 + hUD = 1 + hU = 2 + hL = 1 << 7 + hLR = 1<<7 | 1<<5 + hLRN = 1<<7 | 1<<6 + hLRND = 1<<7 | 1<<6 | 1<<5 +) + +// errHeaderByte indicates an unsupported value for the chunk header +// byte. These bytes starts the variable-length chunk header. +var errHeaderByte = errors.New("lzma: unsupported chunk header byte") + +// headerChunkType converts the header byte into a chunk type. It +// ignores the uncompressed size bits in the chunk header byte. +func headerChunkType(h byte) (c chunkType, err error) { + if h&hL == 0 { + // no compression + switch h { + case hEOS: + c = cEOS + case hUD: + c = cUD + case hU: + c = cU + default: + return 0, errHeaderByte + } + return + } + switch h & hLRND { + case hL: + c = cL + case hLR: + c = cLR + case hLRN: + c = cLRN + case hLRND: + c = cLRND + default: + return 0, errHeaderByte + } + return +} + +// uncompressedHeaderLen provides the length of an uncompressed header +const uncompressedHeaderLen = 3 + +// headerLen returns the length of the LZMA2 header for a given chunk +// type. +func headerLen(c chunkType) int { + switch c { + case cEOS: + return 1 + case cU, cUD: + return uncompressedHeaderLen + case cL, cLR: + return 5 + case cLRN, cLRND: + return 6 + } + panic(fmt.Errorf("unsupported chunk type %d", c)) +} + +// chunkHeader represents the contents of a chunk header. +type chunkHeader struct { + ctype chunkType + uncompressed uint32 + compressed uint16 + props Properties +} + +// String returns a string representation of the chunk header. +func (h *chunkHeader) String() string { + return fmt.Sprintf("%s %d %d %s", h.ctype, h.uncompressed, + h.compressed, &h.props) +} + +// UnmarshalBinary reads the content of the chunk header from the data +// slice. The slice must have the correct length. +func (h *chunkHeader) UnmarshalBinary(data []byte) error { + if len(data) == 0 { + return errors.New("no data") + } + c, err := headerChunkType(data[0]) + if err != nil { + return err + } + + n := headerLen(c) + if len(data) < n { + return errors.New("incomplete data") + } + if len(data) > n { + return errors.New("invalid data length") + } + + *h = chunkHeader{ctype: c} + if c == cEOS { + return nil + } + + h.uncompressed = uint32(uint16BE(data[1:3])) + if c <= cU { + return nil + } + h.uncompressed |= uint32(data[0]&^hLRND) << 16 + + h.compressed = uint16BE(data[3:5]) + if c <= cLR { + return nil + } + + h.props, err = PropertiesForCode(data[5]) + return err +} + +// MarshalBinary encodes the chunk header value. The function checks +// whether the content of the chunk header is correct. +func (h *chunkHeader) MarshalBinary() (data []byte, err error) { + if h.ctype > cLRND { + return nil, errors.New("invalid chunk type") + } + if err = h.props.verify(); err != nil { + return nil, err + } + + data = make([]byte, headerLen(h.ctype)) + + switch h.ctype { + case cEOS: + return data, nil + case cUD: + data[0] = hUD + case cU: + data[0] = hU + case cL: + data[0] = hL + case cLR: + data[0] = hLR + case cLRN: + data[0] = hLRN + case cLRND: + data[0] = hLRND + } + + putUint16BE(data[1:3], uint16(h.uncompressed)) + if h.ctype <= cU { + return data, nil + } + data[0] |= byte(h.uncompressed>>16) &^ hLRND + + putUint16BE(data[3:5], h.compressed) + if h.ctype <= cLR { + return data, nil + } + + data[5] = h.props.Code() + return data, nil +} + +// readChunkHeader reads the chunk header from the IO reader. +func readChunkHeader(r io.Reader) (h *chunkHeader, err error) { + p := make([]byte, 1, 6) + if _, err = io.ReadFull(r, p); err != nil { + return + } + c, err := headerChunkType(p[0]) + if err != nil { + return + } + p = p[:headerLen(c)] + if _, err = io.ReadFull(r, p[1:]); err != nil { + return + } + h = new(chunkHeader) + if err = h.UnmarshalBinary(p); err != nil { + return nil, err + } + return h, nil +} + +// uint16BE converts a big-endian uint16 representation to an uint16 +// value. +func uint16BE(p []byte) uint16 { + return uint16(p[0])<<8 | uint16(p[1]) +} + +// putUint16BE puts the big-endian uint16 presentation into the given +// slice. +func putUint16BE(p []byte, x uint16) { + p[0] = byte(x >> 8) + p[1] = byte(x) +} + +// chunkState is used to manage the state of the chunks +type chunkState byte + +// start and stop define the initial and terminating state of the chunk +// state +const ( + start chunkState = 'S' + stop chunkState = 'T' +) + +// errors for the chunk state handling +var ( + errChunkType = errors.New("lzma: unexpected chunk type") + errState = errors.New("lzma: wrong chunk state") +) + +// next transitions state based on chunk type input +func (c *chunkState) next(ctype chunkType) error { + switch *c { + // start state + case 'S': + switch ctype { + case cEOS: + *c = 'T' + case cUD: + *c = 'R' + case cLRND: + *c = 'L' + default: + return errChunkType + } + // normal LZMA mode + case 'L': + switch ctype { + case cEOS: + *c = 'T' + case cUD: + *c = 'R' + case cU: + *c = 'U' + case cL, cLR, cLRN, cLRND: + break + default: + return errChunkType + } + // reset required + case 'R': + switch ctype { + case cEOS: + *c = 'T' + case cUD, cU: + break + case cLRN, cLRND: + *c = 'L' + default: + return errChunkType + } + // uncompressed + case 'U': + switch ctype { + case cEOS: + *c = 'T' + case cUD: + *c = 'R' + case cU: + break + case cL, cLR, cLRN, cLRND: + *c = 'L' + default: + return errChunkType + } + // terminal state + case 'T': + return errChunkType + default: + return errState + } + return nil +} + +// defaultChunkType returns the default chunk type for each chunk state. +func (c chunkState) defaultChunkType() chunkType { + switch c { + case 'S': + return cLRND + case 'L', 'U': + return cL + case 'R': + return cLRN + default: + // no error + return cEOS + } +} + +// maxDictCap defines the maximum dictionary capacity supported by the +// LZMA2 dictionary capacity encoding. +const maxDictCap = 1<<32 - 1 + +// maxDictCapCode defines the maximum dictionary capacity code. +const maxDictCapCode = 40 + +// The function decodes the dictionary capacity byte, but doesn't change +// for the correct range of the given byte. +func decodeDictCap(c byte) int64 { + return (2 | int64(c)&1) << (11 + (c>>1)&0x1f) +} + +// DecodeDictCap decodes the encoded dictionary capacity. The function +// returns an error if the code is out of range. +func DecodeDictCap(c byte) (n int64, err error) { + if c >= maxDictCapCode { + if c == maxDictCapCode { + return maxDictCap, nil + } + return 0, errors.New("lzma: invalid dictionary size code") + } + return decodeDictCap(c), nil +} + +// EncodeDictCap encodes a dictionary capacity. The function returns the +// code for the capacity that is greater or equal n. If n exceeds the +// maximum support dictionary capacity, the maximum value is returned. +func EncodeDictCap(n int64) byte { + a, b := byte(0), byte(40) + for a < b { + c := a + (b-a)>>1 + m := decodeDictCap(c) + if n <= m { + if n == m { + return c + } + b = c + } else { + a = c + 1 + } + } + return a +} diff --git a/vendor/github.com/ulikunitz/xz/lzma/lengthcodec.go b/vendor/github.com/ulikunitz/xz/lzma/lengthcodec.go new file mode 100644 index 00000000..1ea5320a --- /dev/null +++ b/vendor/github.com/ulikunitz/xz/lzma/lengthcodec.go @@ -0,0 +1,115 @@ +// Copyright 2014-2022 Ulrich Kunitz. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package lzma + +import "errors" + +// maxPosBits defines the number of bits of the position value that are used to +// to compute the posState value. The value is used to select the tree codec +// for length encoding and decoding. +const maxPosBits = 4 + +// minMatchLen and maxMatchLen give the minimum and maximum values for +// encoding and decoding length values. minMatchLen is also used as base +// for the encoded length values. +const ( + minMatchLen = 2 + maxMatchLen = minMatchLen + 16 + 256 - 1 +) + +// lengthCodec support the encoding of the length value. +type lengthCodec struct { + choice [2]prob + low [1 << maxPosBits]treeCodec + mid [1 << maxPosBits]treeCodec + high treeCodec +} + +// deepcopy initializes the lc value as deep copy of the source value. +func (lc *lengthCodec) deepcopy(src *lengthCodec) { + if lc == src { + return + } + lc.choice = src.choice + for i := range lc.low { + lc.low[i].deepcopy(&src.low[i]) + } + for i := range lc.mid { + lc.mid[i].deepcopy(&src.mid[i]) + } + lc.high.deepcopy(&src.high) +} + +// init initializes a new length codec. +func (lc *lengthCodec) init() { + for i := range lc.choice { + lc.choice[i] = probInit + } + for i := range lc.low { + lc.low[i] = makeTreeCodec(3) + } + for i := range lc.mid { + lc.mid[i] = makeTreeCodec(3) + } + lc.high = makeTreeCodec(8) +} + +// Encode encodes the length offset. The length offset l can be compute by +// subtracting minMatchLen (2) from the actual length. +// +// l = length - minMatchLen +func (lc *lengthCodec) Encode(e *rangeEncoder, l uint32, posState uint32, +) (err error) { + if l > maxMatchLen-minMatchLen { + return errors.New("lengthCodec.Encode: l out of range") + } + if l < 8 { + if err = lc.choice[0].Encode(e, 0); err != nil { + return + } + return lc.low[posState].Encode(e, l) + } + if err = lc.choice[0].Encode(e, 1); err != nil { + return + } + if l < 16 { + if err = lc.choice[1].Encode(e, 0); err != nil { + return + } + return lc.mid[posState].Encode(e, l-8) + } + if err = lc.choice[1].Encode(e, 1); err != nil { + return + } + if err = lc.high.Encode(e, l-16); err != nil { + return + } + return nil +} + +// Decode reads the length offset. Add minMatchLen to compute the actual length +// to the length offset l. +func (lc *lengthCodec) Decode(d *rangeDecoder, posState uint32, +) (l uint32, err error) { + var b uint32 + if b, err = lc.choice[0].Decode(d); err != nil { + return + } + if b == 0 { + l, err = lc.low[posState].Decode(d) + return + } + if b, err = lc.choice[1].Decode(d); err != nil { + return + } + if b == 0 { + l, err = lc.mid[posState].Decode(d) + l += 8 + return + } + l, err = lc.high.Decode(d) + l += 16 + return +} diff --git a/vendor/github.com/ulikunitz/xz/lzma/literalcodec.go b/vendor/github.com/ulikunitz/xz/lzma/literalcodec.go new file mode 100644 index 00000000..e4ef5fc5 --- /dev/null +++ b/vendor/github.com/ulikunitz/xz/lzma/literalcodec.go @@ -0,0 +1,125 @@ +// Copyright 2014-2022 Ulrich Kunitz. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package lzma + +// literalCodec supports the encoding of literal. It provides 768 probability +// values per literal state. The upper 512 probabilities are used with the +// context of a match bit. +type literalCodec struct { + probs []prob +} + +// deepcopy initializes literal codec c as a deep copy of the source. +func (c *literalCodec) deepcopy(src *literalCodec) { + if c == src { + return + } + c.probs = make([]prob, len(src.probs)) + copy(c.probs, src.probs) +} + +// init initializes the literal codec. +func (c *literalCodec) init(lc, lp int) { + switch { + case !(minLC <= lc && lc <= maxLC): + panic("lc out of range") + case !(minLP <= lp && lp <= maxLP): + panic("lp out of range") + } + c.probs = make([]prob, 0x300<= 7 { + m := uint32(match) + for { + matchBit := (m >> 7) & 1 + m <<= 1 + bit := (r >> 7) & 1 + r <<= 1 + i := ((1 + matchBit) << 8) | symbol + if err = probs[i].Encode(e, bit); err != nil { + return + } + symbol = (symbol << 1) | bit + if matchBit != bit { + break + } + if symbol >= 0x100 { + break + } + } + } + for symbol < 0x100 { + bit := (r >> 7) & 1 + r <<= 1 + if err = probs[symbol].Encode(e, bit); err != nil { + return + } + symbol = (symbol << 1) | bit + } + return nil +} + +// Decode decodes a literal byte using the range decoder as well as the LZMA +// state, a match byte, and the literal state. +func (c *literalCodec) Decode(d *rangeDecoder, + state uint32, match byte, litState uint32, +) (s byte, err error) { + k := litState * 0x300 + probs := c.probs[k : k+0x300] + symbol := uint32(1) + if state >= 7 { + m := uint32(match) + for { + matchBit := (m >> 7) & 1 + m <<= 1 + i := ((1 + matchBit) << 8) | symbol + bit, err := d.DecodeBit(&probs[i]) + if err != nil { + return 0, err + } + symbol = (symbol << 1) | bit + if matchBit != bit { + break + } + if symbol >= 0x100 { + break + } + } + } + for symbol < 0x100 { + bit, err := d.DecodeBit(&probs[symbol]) + if err != nil { + return 0, err + } + symbol = (symbol << 1) | bit + } + s = byte(symbol - 0x100) + return s, nil +} + +// minLC and maxLC define the range for LC values. +const ( + minLC = 0 + maxLC = 8 +) + +// minLC and maxLC define the range for LP values. +const ( + minLP = 0 + maxLP = 4 +) diff --git a/vendor/github.com/ulikunitz/xz/lzma/matchalgorithm.go b/vendor/github.com/ulikunitz/xz/lzma/matchalgorithm.go new file mode 100644 index 00000000..02dfb8bf --- /dev/null +++ b/vendor/github.com/ulikunitz/xz/lzma/matchalgorithm.go @@ -0,0 +1,52 @@ +// Copyright 2014-2022 Ulrich Kunitz. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package lzma + +import "errors" + +// MatchAlgorithm identifies an algorithm to find matches in the +// dictionary. +type MatchAlgorithm byte + +// Supported matcher algorithms. +const ( + HashTable4 MatchAlgorithm = iota + BinaryTree +) + +// maStrings are used by the String method. +var maStrings = map[MatchAlgorithm]string{ + HashTable4: "HashTable4", + BinaryTree: "BinaryTree", +} + +// String returns a string representation of the Matcher. +func (a MatchAlgorithm) String() string { + if s, ok := maStrings[a]; ok { + return s + } + return "unknown" +} + +var errUnsupportedMatchAlgorithm = errors.New( + "lzma: unsupported match algorithm value") + +// verify checks whether the matcher value is supported. +func (a MatchAlgorithm) verify() error { + if _, ok := maStrings[a]; !ok { + return errUnsupportedMatchAlgorithm + } + return nil +} + +func (a MatchAlgorithm) new(dictCap int) (m matcher, err error) { + switch a { + case HashTable4: + return newHashTable(dictCap, 4) + case BinaryTree: + return newBinTree(dictCap) + } + return nil, errUnsupportedMatchAlgorithm +} diff --git a/vendor/github.com/ulikunitz/xz/lzma/operation.go b/vendor/github.com/ulikunitz/xz/lzma/operation.go new file mode 100644 index 00000000..7b7eddc3 --- /dev/null +++ b/vendor/github.com/ulikunitz/xz/lzma/operation.go @@ -0,0 +1,55 @@ +// Copyright 2014-2022 Ulrich Kunitz. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package lzma + +import ( + "fmt" + "unicode" +) + +// operation represents an operation on the dictionary during encoding or +// decoding. +type operation interface { + Len() int +} + +// rep represents a repetition at the given distance and the given length +type match struct { + // supports all possible distance values, including the eos marker + distance int64 + // length + n int +} + +// Len returns the number of bytes matched. +func (m match) Len() int { + return m.n +} + +// String returns a string representation for the repetition. +func (m match) String() string { + return fmt.Sprintf("M{%d,%d}", m.distance, m.n) +} + +// lit represents a single byte literal. +type lit struct { + b byte +} + +// Len returns 1 for the single byte literal. +func (l lit) Len() int { + return 1 +} + +// String returns a string representation for the literal. +func (l lit) String() string { + var c byte + if unicode.IsPrint(rune(l.b)) { + c = l.b + } else { + c = '.' + } + return fmt.Sprintf("L{%c/%02x}", c, l.b) +} diff --git a/vendor/github.com/ulikunitz/xz/lzma/prob.go b/vendor/github.com/ulikunitz/xz/lzma/prob.go new file mode 100644 index 00000000..2feccba1 --- /dev/null +++ b/vendor/github.com/ulikunitz/xz/lzma/prob.go @@ -0,0 +1,53 @@ +// Copyright 2014-2022 Ulrich Kunitz. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package lzma + +// movebits defines the number of bits used for the updates of probability +// values. +const movebits = 5 + +// probbits defines the number of bits of a probability value. +const probbits = 11 + +// probInit defines 0.5 as initial value for prob values. +const probInit prob = 1 << (probbits - 1) + +// Type prob represents probabilities. The type can also be used to encode and +// decode single bits. +type prob uint16 + +// Dec decreases the probability. The decrease is proportional to the +// probability value. +func (p *prob) dec() { + *p -= *p >> movebits +} + +// Inc increases the probability. The Increase is proportional to the +// difference of 1 and the probability value. +func (p *prob) inc() { + *p += ((1 << probbits) - *p) >> movebits +} + +// Computes the new bound for a given range using the probability value. +func (p prob) bound(r uint32) uint32 { + return (r >> probbits) * uint32(p) +} + +// Bits returns 1. One is the number of bits that can be encoded or decoded +// with a single prob value. +func (p prob) Bits() int { + return 1 +} + +// Encode encodes the least-significant bit of v. Note that the p value will be +// changed. +func (p *prob) Encode(e *rangeEncoder, v uint32) error { + return e.EncodeBit(v, p) +} + +// Decode decodes a single bit. Note that the p value will change. +func (p *prob) Decode(d *rangeDecoder) (v uint32, err error) { + return d.DecodeBit(p) +} diff --git a/vendor/github.com/ulikunitz/xz/lzma/properties.go b/vendor/github.com/ulikunitz/xz/lzma/properties.go new file mode 100644 index 00000000..15b754cc --- /dev/null +++ b/vendor/github.com/ulikunitz/xz/lzma/properties.go @@ -0,0 +1,69 @@ +// Copyright 2014-2022 Ulrich Kunitz. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package lzma + +import ( + "errors" + "fmt" +) + +// maximum and minimum values for the LZMA properties. +const ( + minPB = 0 + maxPB = 4 +) + +// maxPropertyCode is the possible maximum of a properties code byte. +const maxPropertyCode = (maxPB+1)*(maxLP+1)*(maxLC+1) - 1 + +// Properties contains the parameters LC, LP and PB. The parameter LC +// defines the number of literal context bits; parameter LP the number +// of literal position bits and PB the number of position bits. +type Properties struct { + LC int + LP int + PB int +} + +// String returns the properties in a string representation. +func (p *Properties) String() string { + return fmt.Sprintf("LC %d LP %d PB %d", p.LC, p.LP, p.PB) +} + +// PropertiesForCode converts a properties code byte into a Properties value. +func PropertiesForCode(code byte) (p Properties, err error) { + if code > maxPropertyCode { + return p, errors.New("lzma: invalid properties code") + } + p.LC = int(code % 9) + code /= 9 + p.LP = int(code % 5) + code /= 5 + p.PB = int(code % 5) + return p, err +} + +// verify checks the properties for correctness. +func (p *Properties) verify() error { + if p == nil { + return errors.New("lzma: properties are nil") + } + if !(minLC <= p.LC && p.LC <= maxLC) { + return errors.New("lzma: lc out of range") + } + if !(minLP <= p.LP && p.LP <= maxLP) { + return errors.New("lzma: lp out of range") + } + if !(minPB <= p.PB && p.PB <= maxPB) { + return errors.New("lzma: pb out of range") + } + return nil +} + +// Code converts the properties to a byte. The function assumes that +// the properties components are all in range. +func (p Properties) Code() byte { + return byte((p.PB*5+p.LP)*9 + p.LC) +} diff --git a/vendor/github.com/ulikunitz/xz/lzma/rangecodec.go b/vendor/github.com/ulikunitz/xz/lzma/rangecodec.go new file mode 100644 index 00000000..4b0fee3f --- /dev/null +++ b/vendor/github.com/ulikunitz/xz/lzma/rangecodec.go @@ -0,0 +1,222 @@ +// Copyright 2014-2022 Ulrich Kunitz. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package lzma + +import ( + "errors" + "io" +) + +// rangeEncoder implements range encoding of single bits. The low value can +// overflow therefore we need uint64. The cache value is used to handle +// overflows. +type rangeEncoder struct { + lbw *LimitedByteWriter + nrange uint32 + low uint64 + cacheLen int64 + cache byte +} + +// maxInt64 provides the maximal value of the int64 type +const maxInt64 = 1<<63 - 1 + +// newRangeEncoder creates a new range encoder. +func newRangeEncoder(bw io.ByteWriter) (re *rangeEncoder, err error) { + lbw, ok := bw.(*LimitedByteWriter) + if !ok { + lbw = &LimitedByteWriter{BW: bw, N: maxInt64} + } + return &rangeEncoder{ + lbw: lbw, + nrange: 0xffffffff, + cacheLen: 1}, nil +} + +// Available returns the number of bytes that still can be written. The +// method takes the bytes that will be currently written by Close into +// account. +func (e *rangeEncoder) Available() int64 { + return e.lbw.N - (e.cacheLen + 4) +} + +// writeByte writes a single byte to the underlying writer. An error is +// returned if the limit is reached. The written byte will be counted if +// the underlying writer doesn't return an error. +func (e *rangeEncoder) writeByte(c byte) error { + if e.Available() < 1 { + return ErrLimit + } + return e.lbw.WriteByte(c) +} + +// DirectEncodeBit encodes the least-significant bit of b with probability 1/2. +func (e *rangeEncoder) DirectEncodeBit(b uint32) error { + e.nrange >>= 1 + e.low += uint64(e.nrange) & (0 - (uint64(b) & 1)) + + // normalize + const top = 1 << 24 + if e.nrange >= top { + return nil + } + e.nrange <<= 8 + return e.shiftLow() +} + +// EncodeBit encodes the least significant bit of b. The p value will be +// updated by the function depending on the bit encoded. +func (e *rangeEncoder) EncodeBit(b uint32, p *prob) error { + bound := p.bound(e.nrange) + if b&1 == 0 { + e.nrange = bound + p.inc() + } else { + e.low += uint64(bound) + e.nrange -= bound + p.dec() + } + + // normalize + const top = 1 << 24 + if e.nrange >= top { + return nil + } + e.nrange <<= 8 + return e.shiftLow() +} + +// Close writes a complete copy of the low value. +func (e *rangeEncoder) Close() error { + for i := 0; i < 5; i++ { + if err := e.shiftLow(); err != nil { + return err + } + } + return nil +} + +// shiftLow shifts the low value for 8 bit. The shifted byte is written into +// the byte writer. The cache value is used to handle overflows. +func (e *rangeEncoder) shiftLow() error { + if uint32(e.low) < 0xff000000 || (e.low>>32) != 0 { + tmp := e.cache + for { + err := e.writeByte(tmp + byte(e.low>>32)) + if err != nil { + return err + } + tmp = 0xff + e.cacheLen-- + if e.cacheLen <= 0 { + if e.cacheLen < 0 { + panic("negative cacheLen") + } + break + } + } + e.cache = byte(uint32(e.low) >> 24) + } + e.cacheLen++ + e.low = uint64(uint32(e.low) << 8) + return nil +} + +// rangeDecoder decodes single bits of the range encoding stream. +type rangeDecoder struct { + br io.ByteReader + nrange uint32 + code uint32 +} + +// newRangeDecoder initializes a range decoder. It reads five bytes from the +// reader and therefore may return an error. +func newRangeDecoder(br io.ByteReader) (d *rangeDecoder, err error) { + d = &rangeDecoder{br: br, nrange: 0xffffffff} + + b, err := d.br.ReadByte() + if err != nil { + return nil, err + } + if b != 0 { + return nil, errors.New("newRangeDecoder: first byte not zero") + } + + for i := 0; i < 4; i++ { + if err = d.updateCode(); err != nil { + return nil, err + } + } + + if d.code >= d.nrange { + return nil, errors.New("newRangeDecoder: d.code >= d.nrange") + } + + return d, nil +} + +// possiblyAtEnd checks whether the decoder may be at the end of the stream. +func (d *rangeDecoder) possiblyAtEnd() bool { + return d.code == 0 +} + +// DirectDecodeBit decodes a bit with probability 1/2. The return value b will +// contain the bit at the least-significant position. All other bits will be +// zero. +func (d *rangeDecoder) DirectDecodeBit() (b uint32, err error) { + d.nrange >>= 1 + d.code -= d.nrange + t := 0 - (d.code >> 31) + d.code += d.nrange & t + b = (t + 1) & 1 + + // d.code will stay less then d.nrange + + // normalize + // assume d.code < d.nrange + const top = 1 << 24 + if d.nrange >= top { + return b, nil + } + d.nrange <<= 8 + // d.code < d.nrange will be maintained + return b, d.updateCode() +} + +// decodeBit decodes a single bit. The bit will be returned at the +// least-significant position. All other bits will be zero. The probability +// value will be updated. +func (d *rangeDecoder) DecodeBit(p *prob) (b uint32, err error) { + bound := p.bound(d.nrange) + if d.code < bound { + d.nrange = bound + p.inc() + b = 0 + } else { + d.code -= bound + d.nrange -= bound + p.dec() + b = 1 + } + // normalize + // assume d.code < d.nrange + const top = 1 << 24 + if d.nrange >= top { + return b, nil + } + d.nrange <<= 8 + // d.code < d.nrange will be maintained + return b, d.updateCode() +} + +// updateCode reads a new byte into the code. +func (d *rangeDecoder) updateCode() error { + b, err := d.br.ReadByte() + if err != nil { + return err + } + d.code = (d.code << 8) | uint32(b) + return nil +} diff --git a/vendor/github.com/ulikunitz/xz/lzma/reader.go b/vendor/github.com/ulikunitz/xz/lzma/reader.go new file mode 100644 index 00000000..eef6bea7 --- /dev/null +++ b/vendor/github.com/ulikunitz/xz/lzma/reader.go @@ -0,0 +1,193 @@ +// Copyright 2014-2022 Ulrich Kunitz. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package lzma supports the decoding and encoding of LZMA streams. +// Reader and Writer support the classic LZMA format. Reader2 and +// Writer2 support the decoding and encoding of LZMA2 streams. +// +// The package is written completely in Go and does not rely on any external +// library. +package lzma + +import ( + "errors" + "fmt" + "io" +) + +// ReaderConfig stores the parameters for the reader of the classic LZMA +// format. +type ReaderConfig struct { + // Since v0.5.14 this parameter sets an upper limit for a .lzma file's + // dictionary size. This helps to mitigate problems with mangled + // headers. + DictCap int +} + +// fill converts the zero values of the configuration to the default values. +func (c *ReaderConfig) fill() { + if c.DictCap == 0 { + // set an upper limit of 2 GiB-1 for dictionary capacity + // to address the zero prefix security issue. + c.DictCap = (1 << 31) - 1 + // original: c.DictCap = 8 * 1024 * 1024 + } +} + +// Verify checks the reader configuration for errors. Zero values will +// be replaced by default values. +func (c *ReaderConfig) Verify() error { + c.fill() + if !(MinDictCap <= c.DictCap && int64(c.DictCap) <= MaxDictCap) { + return errors.New("lzma: dictionary capacity is out of range") + } + return nil +} + +// Reader provides a reader for LZMA files or streams. +// +// # Security concerns +// +// Note that LZMA format doesn't support a magic marker in the header. So +// [NewReader] cannot determine whether it reads the actual header. For instance +// the LZMA stream might have a zero byte in front of the reader, leading to +// larger dictionary sizes and file sizes. The code will detect later that there +// are problems with the stream, but the dictionary has already been allocated +// and this might consume a lot of memory. +// +// Version 0.5.14 introduces built-in mitigations: +// +// - The [ReaderConfig] DictCap field is now interpreted as a limit for the +// dictionary size. +// - The default is 2 Gigabytes minus 1 byte (2^31-1 bytes). +// - Users can check with the [Reader.Header] method what the actual values are in +// their LZMA files and set a smaller limit using [ReaderConfig]. +// - The dictionary size doesn't exceed the larger of the file size and +// the minimum dictionary size. This is another measure to prevent huge +// memory allocations for the dictionary. +// - The code supports stream sizes only up to a pebibyte (1024^5). +type Reader struct { + lzma io.Reader + header Header + // headerOrig stores the original header read from the stream. + headerOrig Header + d *decoder +} + +// NewReader creates a new reader for an LZMA stream using the classic +// format. NewReader reads and checks the header of the LZMA stream. +func NewReader(lzma io.Reader) (r *Reader, err error) { + return ReaderConfig{}.NewReader(lzma) +} + +// ErrDictSize reports about an error of the dictionary size. +type ErrDictSize struct { + ConfigDictCap int + HeaderDictSize uint32 + Message string +} + +// Error returns the error message. +func (e *ErrDictSize) Error() string { + return e.Message +} + +func newErrDictSize(messageformat string, + configDictCap int, headerDictSize uint32, + args ...interface{}) *ErrDictSize { + newArgs := make([]interface{}, len(args)+2) + newArgs[0] = configDictCap + newArgs[1] = headerDictSize + copy(newArgs[2:], args) + return &ErrDictSize{ + ConfigDictCap: configDictCap, + HeaderDictSize: headerDictSize, + Message: fmt.Sprintf(messageformat, newArgs...), + } +} + +// We support only files not larger than 1 << 50 bytes (a pebibyte, 1024^5). +const maxStreamSize = 1 << 50 + +// NewReader creates a new reader for an LZMA stream in the classic +// format. The function reads and verifies the header of the LZMA +// stream. +func (c ReaderConfig) NewReader(lzma io.Reader) (r *Reader, err error) { + if err = c.Verify(); err != nil { + return nil, err + } + data := make([]byte, HeaderLen) + if _, err := io.ReadFull(lzma, data); err != nil { + if err == io.EOF { + return nil, errors.New("lzma: unexpected EOF") + } + return nil, err + } + r = &Reader{lzma: lzma} + if err = r.header.unmarshalBinary(data); err != nil { + return nil, err + } + r.headerOrig = r.header + dictSize := int64(r.header.DictSize) + if int64(c.DictCap) < dictSize { + return nil, newErrDictSize( + "lzma: header dictionary size %[2]d exceeds configured dictionary capacity %[1]d", + c.DictCap, uint32(dictSize), + ) + } + if dictSize < MinDictCap { + dictSize = MinDictCap + } + // original code: disabled this because there is no point in increasing + // the dictionary above what is stated in the file. + /* + if int64(c.DictCap) > int64(dictSize) { + dictSize = int64(c.DictCap) + } + */ + size := r.header.Size + if size >= 0 && size < dictSize { + dictSize = size + } + // Protect against modified or malicious headers. + if size > maxStreamSize { + return nil, fmt.Errorf( + "lzma: stream size %d exceeds a pebibyte (1024^5)", + size) + } + if dictSize < MinDictCap { + dictSize = MinDictCap + } + + r.header.DictSize = uint32(dictSize) + + state := newState(r.header.Properties) + dict, err := newDecoderDict(int(dictSize)) + if err != nil { + return nil, err + } + r.d, err = newDecoder(ByteReader(lzma), state, dict, r.header.Size) + if err != nil { + return nil, err + } + return r, nil +} + +// Header returns the header as read from the LZMA stream. It is intended to +// allow the user to understand what parameters are typically provided in the +// headers of the LZMA files and set the DictCap field in [ReaderConfig] +// accordingly. +func (r *Reader) Header() (h Header, ok bool) { + return r.headerOrig, r.d != nil +} + +// EOSMarker indicates that an EOS marker has been encountered. +func (r *Reader) EOSMarker() bool { + return r.d.eosMarker +} + +// Read returns uncompressed data. +func (r *Reader) Read(p []byte) (n int, err error) { + return r.d.Read(p) +} diff --git a/vendor/github.com/ulikunitz/xz/lzma/reader2.go b/vendor/github.com/ulikunitz/xz/lzma/reader2.go new file mode 100644 index 00000000..f36e2650 --- /dev/null +++ b/vendor/github.com/ulikunitz/xz/lzma/reader2.go @@ -0,0 +1,231 @@ +// Copyright 2014-2022 Ulrich Kunitz. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package lzma + +import ( + "errors" + "io" + + "github.com/ulikunitz/xz/internal/xlog" +) + +// Reader2Config stores the parameters for the LZMA2 reader. +// format. +type Reader2Config struct { + DictCap int +} + +// fill converts the zero values of the configuration to the default values. +func (c *Reader2Config) fill() { + if c.DictCap == 0 { + c.DictCap = 8 * 1024 * 1024 + } +} + +// Verify checks the reader configuration for errors. Zero configuration values +// will be replaced by default values. +func (c *Reader2Config) Verify() error { + c.fill() + if !(MinDictCap <= c.DictCap && int64(c.DictCap) <= MaxDictCap) { + return errors.New("lzma: dictionary capacity is out of range") + } + return nil +} + +// Reader2 supports the reading of LZMA2 chunk sequences. Note that the +// first chunk should have a dictionary reset and the first compressed +// chunk a properties reset. The chunk sequence may not be terminated by +// an end-of-stream chunk. +type Reader2 struct { + r io.Reader + err error + + dict *decoderDict + ur *uncompressedReader + decoder *decoder + chunkReader io.Reader + + cstate chunkState +} + +// NewReader2 creates a reader for an LZMA2 chunk sequence. +func NewReader2(lzma2 io.Reader) (r *Reader2, err error) { + return Reader2Config{}.NewReader2(lzma2) +} + +// NewReader2 creates an LZMA2 reader using the given configuration. +func (c Reader2Config) NewReader2(lzma2 io.Reader) (r *Reader2, err error) { + if err = c.Verify(); err != nil { + return nil, err + } + r = &Reader2{r: lzma2, cstate: start} + r.dict, err = newDecoderDict(c.DictCap) + if err != nil { + return nil, err + } + if err = r.startChunk(); err != nil { + r.err = err + } + return r, nil +} + +// uncompressed tests whether the chunk type specifies an uncompressed +// chunk. +func uncompressed(ctype chunkType) bool { + return ctype == cU || ctype == cUD +} + +// startChunk parses a new chunk. +func (r *Reader2) startChunk() error { + r.chunkReader = nil + header, err := readChunkHeader(r.r) + if err != nil { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + return err + } + xlog.Debugf("chunk header %v", header) + if err = r.cstate.next(header.ctype); err != nil { + return err + } + if r.cstate == stop { + return io.EOF + } + if header.ctype == cUD || header.ctype == cLRND { + r.dict.Reset() + } + size := int64(header.uncompressed) + 1 + if uncompressed(header.ctype) { + if r.ur != nil { + r.ur.Reopen(r.r, size) + } else { + r.ur = newUncompressedReader(r.r, r.dict, size) + } + r.chunkReader = r.ur + return nil + } + br := ByteReader(io.LimitReader(r.r, int64(header.compressed)+1)) + if r.decoder == nil { + state := newState(header.props) + r.decoder, err = newDecoder(br, state, r.dict, size) + if err != nil { + return err + } + r.chunkReader = r.decoder + return nil + } + switch header.ctype { + case cLR: + r.decoder.State.Reset() + case cLRN, cLRND: + r.decoder.State = newState(header.props) + } + err = r.decoder.Reopen(br, size) + if err != nil { + return err + } + r.chunkReader = r.decoder + return nil +} + +// Read reads data from the LZMA2 chunk sequence. +func (r *Reader2) Read(p []byte) (n int, err error) { + if r.err != nil { + return 0, r.err + } + for n < len(p) { + var k int + k, err = r.chunkReader.Read(p[n:]) + n += k + if err != nil { + if err == io.EOF { + err = r.startChunk() + if err == nil { + continue + } + } + r.err = err + return n, err + } + if k == 0 { + r.err = errors.New("lzma: Reader2 doesn't get data") + return n, r.err + } + } + return n, nil +} + +// EOS returns whether the LZMA2 stream has been terminated by an +// end-of-stream chunk. +func (r *Reader2) EOS() bool { + return r.cstate == stop +} + +// uncompressedReader is used to read uncompressed chunks. +type uncompressedReader struct { + lr io.LimitedReader + Dict *decoderDict + eof bool + err error +} + +// newUncompressedReader initializes a new uncompressedReader. +func newUncompressedReader(r io.Reader, dict *decoderDict, size int64) *uncompressedReader { + ur := &uncompressedReader{ + lr: io.LimitedReader{R: r, N: size}, + Dict: dict, + } + return ur +} + +// Reopen reinitializes an uncompressed reader. +func (ur *uncompressedReader) Reopen(r io.Reader, size int64) { + ur.err = nil + ur.eof = false + ur.lr = io.LimitedReader{R: r, N: size} +} + +// fill reads uncompressed data into the dictionary. +func (ur *uncompressedReader) fill() error { + if !ur.eof { + n, err := io.CopyN(ur.Dict, &ur.lr, int64(ur.Dict.Available())) + if err != io.EOF { + return err + } + ur.eof = true + if n > 0 { + return nil + } + } + if ur.lr.N != 0 { + return io.ErrUnexpectedEOF + } + return io.EOF +} + +// Read reads uncompressed data from the limited reader. +func (ur *uncompressedReader) Read(p []byte) (n int, err error) { + if ur.err != nil { + return 0, ur.err + } + for { + var k int + k, err = ur.Dict.Read(p[n:]) + n += k + if n >= len(p) { + return n, nil + } + if err != nil { + break + } + err = ur.fill() + if err != nil { + break + } + } + ur.err = err + return n, err +} diff --git a/vendor/github.com/ulikunitz/xz/lzma/state.go b/vendor/github.com/ulikunitz/xz/lzma/state.go new file mode 100644 index 00000000..34779c51 --- /dev/null +++ b/vendor/github.com/ulikunitz/xz/lzma/state.go @@ -0,0 +1,145 @@ +// Copyright 2014-2022 Ulrich Kunitz. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package lzma + +// states defines the overall state count +const states = 12 + +// State maintains the full state of the operation encoding or decoding +// process. +type state struct { + rep [4]uint32 + isMatch [states << maxPosBits]prob + isRepG0Long [states << maxPosBits]prob + isRep [states]prob + isRepG0 [states]prob + isRepG1 [states]prob + isRepG2 [states]prob + litCodec literalCodec + lenCodec lengthCodec + repLenCodec lengthCodec + distCodec distCodec + state uint32 + posBitMask uint32 + Properties Properties +} + +// initProbSlice initializes a slice of probabilities. +func initProbSlice(p []prob) { + for i := range p { + p[i] = probInit + } +} + +// Reset sets all state information to the original values. +func (s *state) Reset() { + p := s.Properties + *s = state{ + Properties: p, + // dict: s.dict, + posBitMask: (uint32(1) << uint(p.PB)) - 1, + } + initProbSlice(s.isMatch[:]) + initProbSlice(s.isRep[:]) + initProbSlice(s.isRepG0[:]) + initProbSlice(s.isRepG1[:]) + initProbSlice(s.isRepG2[:]) + initProbSlice(s.isRepG0Long[:]) + s.litCodec.init(p.LC, p.LP) + s.lenCodec.init() + s.repLenCodec.init() + s.distCodec.init() +} + +// newState creates a new state from the give Properties. +func newState(p Properties) *state { + s := &state{Properties: p} + s.Reset() + return s +} + +// deepcopy initializes s as a deep copy of the source. +func (s *state) deepcopy(src *state) { + if s == src { + return + } + s.rep = src.rep + s.isMatch = src.isMatch + s.isRepG0Long = src.isRepG0Long + s.isRep = src.isRep + s.isRepG0 = src.isRepG0 + s.isRepG1 = src.isRepG1 + s.isRepG2 = src.isRepG2 + s.litCodec.deepcopy(&src.litCodec) + s.lenCodec.deepcopy(&src.lenCodec) + s.repLenCodec.deepcopy(&src.repLenCodec) + s.distCodec.deepcopy(&src.distCodec) + s.state = src.state + s.posBitMask = src.posBitMask + s.Properties = src.Properties +} + +// cloneState creates a new clone of the give state. +func cloneState(src *state) *state { + s := new(state) + s.deepcopy(src) + return s +} + +// updateStateLiteral updates the state for a literal. +func (s *state) updateStateLiteral() { + switch { + case s.state < 4: + s.state = 0 + return + case s.state < 10: + s.state -= 3 + return + } + s.state -= 6 +} + +// updateStateMatch updates the state for a match. +func (s *state) updateStateMatch() { + if s.state < 7 { + s.state = 7 + } else { + s.state = 10 + } +} + +// updateStateRep updates the state for a repetition. +func (s *state) updateStateRep() { + if s.state < 7 { + s.state = 8 + } else { + s.state = 11 + } +} + +// updateStateShortRep updates the state for a short repetition. +func (s *state) updateStateShortRep() { + if s.state < 7 { + s.state = 9 + } else { + s.state = 11 + } +} + +// states computes the states of the operation codec. +func (s *state) states(dictHead int64) (state1, state2, posState uint32) { + state1 = s.state + posState = uint32(dictHead) & s.posBitMask + state2 = (s.state << maxPosBits) | posState + return +} + +// litState computes the literal state. +func (s *state) litState(prev byte, dictHead int64) uint32 { + lp, lc := uint(s.Properties.LP), uint(s.Properties.LC) + litState := ((uint32(dictHead) & ((1 << lp) - 1)) << lc) | + (uint32(prev) >> (8 - lc)) + return litState +} diff --git a/vendor/github.com/ulikunitz/xz/lzma/treecodecs.go b/vendor/github.com/ulikunitz/xz/lzma/treecodecs.go new file mode 100644 index 00000000..36b29b59 --- /dev/null +++ b/vendor/github.com/ulikunitz/xz/lzma/treecodecs.go @@ -0,0 +1,133 @@ +// Copyright 2014-2022 Ulrich Kunitz. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package lzma + +// treeCodec encodes or decodes values with a fixed bit size. It is using a +// tree of probability value. The root of the tree is the most-significant bit. +type treeCodec struct { + probTree +} + +// makeTreeCodec makes a tree codec. The bits value must be inside the range +// [1,32]. +func makeTreeCodec(bits int) treeCodec { + return treeCodec{makeProbTree(bits)} +} + +// deepcopy initializes tc as a deep copy of the source. +func (tc *treeCodec) deepcopy(src *treeCodec) { + tc.probTree.deepcopy(&src.probTree) +} + +// Encode uses the range encoder to encode a fixed-bit-size value. +func (tc *treeCodec) Encode(e *rangeEncoder, v uint32) (err error) { + m := uint32(1) + for i := int(tc.bits) - 1; i >= 0; i-- { + b := (v >> uint(i)) & 1 + if err := e.EncodeBit(b, &tc.probs[m]); err != nil { + return err + } + m = (m << 1) | b + } + return nil +} + +// Decodes uses the range decoder to decode a fixed-bit-size value. Errors may +// be caused by the range decoder. +func (tc *treeCodec) Decode(d *rangeDecoder) (v uint32, err error) { + m := uint32(1) + for j := 0; j < int(tc.bits); j++ { + b, err := d.DecodeBit(&tc.probs[m]) + if err != nil { + return 0, err + } + m = (m << 1) | b + } + return m - (1 << uint(tc.bits)), nil +} + +// treeReverseCodec is another tree codec, where the least-significant bit is +// the start of the probability tree. +type treeReverseCodec struct { + probTree +} + +// deepcopy initializes the treeReverseCodec as a deep copy of the +// source. +func (tc *treeReverseCodec) deepcopy(src *treeReverseCodec) { + tc.probTree.deepcopy(&src.probTree) +} + +// makeTreeReverseCodec creates treeReverseCodec value. The bits argument must +// be in the range [1,32]. +func makeTreeReverseCodec(bits int) treeReverseCodec { + return treeReverseCodec{makeProbTree(bits)} +} + +// Encode uses range encoder to encode a fixed-bit-size value. The range +// encoder may cause errors. +func (tc *treeReverseCodec) Encode(v uint32, e *rangeEncoder) (err error) { + m := uint32(1) + for i := uint(0); i < uint(tc.bits); i++ { + b := (v >> i) & 1 + if err := e.EncodeBit(b, &tc.probs[m]); err != nil { + return err + } + m = (m << 1) | b + } + return nil +} + +// Decodes uses the range decoder to decode a fixed-bit-size value. Errors +// returned by the range decoder will be returned. +func (tc *treeReverseCodec) Decode(d *rangeDecoder) (v uint32, err error) { + m := uint32(1) + for j := uint(0); j < uint(tc.bits); j++ { + b, err := d.DecodeBit(&tc.probs[m]) + if err != nil { + return 0, err + } + m = (m << 1) | b + v |= b << j + } + return v, nil +} + +// probTree stores enough probability values to be used by the treeEncode and +// treeDecode methods of the range coder types. +type probTree struct { + probs []prob + bits byte +} + +// deepcopy initializes the probTree value as a deep copy of the source. +func (t *probTree) deepcopy(src *probTree) { + if t == src { + return + } + t.probs = make([]prob, len(src.probs)) + copy(t.probs, src.probs) + t.bits = src.bits +} + +// makeProbTree initializes a probTree structure. +func makeProbTree(bits int) probTree { + if !(1 <= bits && bits <= 32) { + panic("bits outside of range [1,32]") + } + t := probTree{ + bits: byte(bits), + probs: make([]prob, 1< 0 { + c.SizeInHeader = true + } + if !c.SizeInHeader { + c.EOSMarker = true + } +} + +// Verify checks WriterConfig for errors. Verify will replace zero +// values with default values. +func (c *WriterConfig) Verify() error { + c.fill() + var err error + if c == nil { + return errors.New("lzma: WriterConfig is nil") + } + if c.Properties == nil { + return errors.New("lzma: WriterConfig has no Properties set") + } + if err = c.Properties.verify(); err != nil { + return err + } + if !(MinDictCap <= c.DictCap && int64(c.DictCap) <= MaxDictCap) { + return errors.New("lzma: dictionary capacity is out of range") + } + if !(maxMatchLen <= c.BufSize) { + return errors.New("lzma: lookahead buffer size too small") + } + if c.SizeInHeader { + if c.Size < 0 { + return errors.New("lzma: negative size not supported") + } + } else if !c.EOSMarker { + return errors.New("lzma: EOS marker is required") + } + if err = c.Matcher.verify(); err != nil { + return err + } + + return nil +} + +// header returns the header structure for this configuration. +func (c *WriterConfig) header() Header { + h := Header{ + Properties: *c.Properties, + DictSize: uint32(c.DictCap), + Size: -1, + } + if c.SizeInHeader { + h.Size = c.Size + } + return h +} + +// Writer writes an LZMA stream in the classic format. +type Writer struct { + h Header + bw io.ByteWriter + buf *bufio.Writer + e *encoder +} + +// NewWriter creates a new LZMA writer for the classic format. The +// method will write the header to the underlying stream. +func (c WriterConfig) NewWriter(lzma io.Writer) (w *Writer, err error) { + if err = c.Verify(); err != nil { + return nil, err + } + w = &Writer{h: c.header()} + + var ok bool + w.bw, ok = lzma.(io.ByteWriter) + if !ok { + w.buf = bufio.NewWriter(lzma) + w.bw = w.buf + } + state := newState(w.h.Properties) + m, err := c.Matcher.new(int(w.h.DictSize)) + if err != nil { + return nil, err + } + dict, err := newEncoderDict(int(w.h.DictSize), c.BufSize, m) + if err != nil { + return nil, err + } + var flags encoderFlags + if c.EOSMarker { + flags = eosMarker + } + if w.e, err = newEncoder(w.bw, state, dict, flags); err != nil { + return nil, err + } + + if err = w.writeHeader(); err != nil { + return nil, err + } + return w, nil +} + +// NewWriter creates a new LZMA writer using the classic format. The +// function writes the header to the underlying stream. +func NewWriter(lzma io.Writer) (w *Writer, err error) { + return WriterConfig{}.NewWriter(lzma) +} + +// writeHeader writes the LZMA header into the stream. +func (w *Writer) writeHeader() error { + data, err := w.h.marshalBinary() + if err != nil { + return err + } + _, err = w.bw.(io.Writer).Write(data) + return err +} + +// Write puts data into the Writer. +func (w *Writer) Write(p []byte) (n int, err error) { + if w.h.Size >= 0 { + m := w.h.Size + m -= w.e.Compressed() + int64(w.e.dict.Buffered()) + if m < 0 { + m = 0 + } + if m < int64(len(p)) { + p = p[:m] + err = ErrNoSpace + } + } + var werr error + if n, werr = w.e.Write(p); werr != nil { + err = werr + } + return n, err +} + +// Close closes the writer stream. It ensures that all data from the +// buffer will be compressed and the LZMA stream will be finished. +func (w *Writer) Close() error { + if w.h.Size >= 0 { + n := w.e.Compressed() + int64(w.e.dict.Buffered()) + if n != w.h.Size { + return errSize + } + } + err := w.e.Close() + if w.buf != nil { + ferr := w.buf.Flush() + if err == nil { + err = ferr + } + } + return err +} diff --git a/vendor/github.com/ulikunitz/xz/lzma/writer2.go b/vendor/github.com/ulikunitz/xz/lzma/writer2.go new file mode 100644 index 00000000..97bbafa1 --- /dev/null +++ b/vendor/github.com/ulikunitz/xz/lzma/writer2.go @@ -0,0 +1,305 @@ +// Copyright 2014-2022 Ulrich Kunitz. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package lzma + +import ( + "bytes" + "errors" + "io" +) + +// Writer2Config is used to create a Writer2 using parameters. +type Writer2Config struct { + // The properties for the encoding. If the it is nil the value + // {LC: 3, LP: 0, PB: 2} will be chosen. + Properties *Properties + // The capacity of the dictionary. If DictCap is zero, the value + // 8 MiB will be chosen. + DictCap int + // Size of the lookahead buffer; value 0 indicates default size + // 4096 + BufSize int + // Match algorithm + Matcher MatchAlgorithm +} + +// fill replaces zero values with default values. +func (c *Writer2Config) fill() { + if c.Properties == nil { + c.Properties = &Properties{LC: 3, LP: 0, PB: 2} + } + if c.DictCap == 0 { + c.DictCap = 8 * 1024 * 1024 + } + if c.BufSize == 0 { + c.BufSize = 4096 + } +} + +// Verify checks the Writer2Config for correctness. Zero values will be +// replaced by default values. +func (c *Writer2Config) Verify() error { + c.fill() + var err error + if c == nil { + return errors.New("lzma: WriterConfig is nil") + } + if c.Properties == nil { + return errors.New("lzma: WriterConfig has no Properties set") + } + if err = c.Properties.verify(); err != nil { + return err + } + if !(MinDictCap <= c.DictCap && int64(c.DictCap) <= MaxDictCap) { + return errors.New("lzma: dictionary capacity is out of range") + } + if !(maxMatchLen <= c.BufSize) { + return errors.New("lzma: lookahead buffer size too small") + } + if c.Properties.LC+c.Properties.LP > 4 { + return errors.New("lzma: sum of lc and lp exceeds 4") + } + if err = c.Matcher.verify(); err != nil { + return err + } + return nil +} + +// Writer2 supports the creation of an LZMA2 stream. But note that +// written data is buffered, so call Flush or Close to write data to the +// underlying writer. The Close method writes the end-of-stream marker +// to the stream. So you may be able to concatenate the output of two +// writers as long the output of the first writer has only been flushed +// but not closed. +// +// Any change to the fields Properties, DictCap must be done before the +// first call to Write, Flush or Close. +type Writer2 struct { + w io.Writer + + start *state + encoder *encoder + + cstate chunkState + ctype chunkType + + buf bytes.Buffer + lbw LimitedByteWriter +} + +// NewWriter2 creates an LZMA2 chunk sequence writer with the default +// parameters and options. +func NewWriter2(lzma2 io.Writer) (w *Writer2, err error) { + return Writer2Config{}.NewWriter2(lzma2) +} + +// NewWriter2 creates a new LZMA2 writer using the given configuration. +func (c Writer2Config) NewWriter2(lzma2 io.Writer) (w *Writer2, err error) { + if err = c.Verify(); err != nil { + return nil, err + } + w = &Writer2{ + w: lzma2, + start: newState(*c.Properties), + cstate: start, + ctype: start.defaultChunkType(), + } + w.buf.Grow(maxCompressed) + w.lbw = LimitedByteWriter{BW: &w.buf, N: maxCompressed} + m, err := c.Matcher.new(c.DictCap) + if err != nil { + return nil, err + } + d, err := newEncoderDict(c.DictCap, c.BufSize, m) + if err != nil { + return nil, err + } + w.encoder, err = newEncoder(&w.lbw, cloneState(w.start), d, 0) + if err != nil { + return nil, err + } + return w, nil +} + +// written returns the number of bytes written to the current chunk +func (w *Writer2) written() int { + if w.encoder == nil { + return 0 + } + return int(w.encoder.Compressed()) + w.encoder.dict.Buffered() +} + +// errClosed indicates that the writer is closed. +var errClosed = errors.New("lzma: writer closed") + +// Writes data to LZMA2 stream. Note that written data will be buffered. +// Use Flush or Close to ensure that data is written to the underlying +// writer. +func (w *Writer2) Write(p []byte) (n int, err error) { + if w.cstate == stop { + return 0, errClosed + } + for n < len(p) { + m := maxUncompressed - w.written() + if m <= 0 { + panic("lzma: maxUncompressed reached") + } + var q []byte + if n+m < len(p) { + q = p[n : n+m] + } else { + q = p[n:] + } + k, err := w.encoder.Write(q) + n += k + if err != nil && err != ErrLimit { + return n, err + } + if err == ErrLimit || k == m { + if err = w.flushChunk(); err != nil { + return n, err + } + } + } + return n, nil +} + +// writeUncompressedChunk writes an uncompressed chunk to the LZMA2 +// stream. +func (w *Writer2) writeUncompressedChunk() error { + u := w.encoder.Compressed() + if u <= 0 { + return errors.New("lzma: can't write empty uncompressed chunk") + } + if u > maxUncompressed { + panic("overrun of uncompressed data limit") + } + switch w.ctype { + case cLRND: + w.ctype = cUD + default: + w.ctype = cU + } + w.encoder.state = w.start + + header := chunkHeader{ + ctype: w.ctype, + uncompressed: uint32(u - 1), + } + hdata, err := header.MarshalBinary() + if err != nil { + return err + } + if _, err = w.w.Write(hdata); err != nil { + return err + } + _, err = w.encoder.dict.CopyN(w.w, int(u)) + return err +} + +// writeCompressedChunk writes a compressed chunk to the underlying +// writer. +func (w *Writer2) writeCompressedChunk() error { + if w.ctype == cU || w.ctype == cUD { + panic("chunk type uncompressed") + } + + u := w.encoder.Compressed() + if u <= 0 { + return errors.New("writeCompressedChunk: empty chunk") + } + if u > maxUncompressed { + panic("overrun of uncompressed data limit") + } + c := w.buf.Len() + if c <= 0 { + panic("no compressed data") + } + if c > maxCompressed { + panic("overrun of compressed data limit") + } + header := chunkHeader{ + ctype: w.ctype, + uncompressed: uint32(u - 1), + compressed: uint16(c - 1), + props: w.encoder.state.Properties, + } + hdata, err := header.MarshalBinary() + if err != nil { + return err + } + if _, err = w.w.Write(hdata); err != nil { + return err + } + _, err = io.Copy(w.w, &w.buf) + return err +} + +// writes a single chunk to the underlying writer. +func (w *Writer2) writeChunk() error { + u := int(uncompressedHeaderLen + w.encoder.Compressed()) + c := headerLen(w.ctype) + w.buf.Len() + if u < c { + return w.writeUncompressedChunk() + } + return w.writeCompressedChunk() +} + +// flushChunk terminates the current chunk. The encoder will be reset +// to support the next chunk. +func (w *Writer2) flushChunk() error { + if w.written() == 0 { + return nil + } + var err error + if err = w.encoder.Close(); err != nil { + return err + } + if err = w.writeChunk(); err != nil { + return err + } + w.buf.Reset() + w.lbw.N = maxCompressed + if err = w.encoder.Reopen(&w.lbw); err != nil { + return err + } + if err = w.cstate.next(w.ctype); err != nil { + return err + } + w.ctype = w.cstate.defaultChunkType() + w.start = cloneState(w.encoder.state) + return nil +} + +// Flush writes all buffered data out to the underlying stream. This +// could result in multiple chunks to be created. +func (w *Writer2) Flush() error { + if w.cstate == stop { + return errClosed + } + for w.written() > 0 { + if err := w.flushChunk(); err != nil { + return err + } + } + return nil +} + +// Close terminates the LZMA2 stream with an EOS chunk. +func (w *Writer2) Close() error { + if w.cstate == stop { + return errClosed + } + if err := w.Flush(); err != nil { + return nil + } + // write zero byte EOS chunk + _, err := w.w.Write([]byte{0}) + if err != nil { + return err + } + w.cstate = stop + return nil +} diff --git a/vendor/github.com/ulikunitz/xz/lzmafilter.go b/vendor/github.com/ulikunitz/xz/lzmafilter.go new file mode 100644 index 00000000..bd5f42ee --- /dev/null +++ b/vendor/github.com/ulikunitz/xz/lzmafilter.go @@ -0,0 +1,117 @@ +// Copyright 2014-2022 Ulrich Kunitz. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package xz + +import ( + "errors" + "fmt" + "io" + + "github.com/ulikunitz/xz/lzma" +) + +// LZMA filter constants. +const ( + lzmaFilterID = 0x21 + lzmaFilterLen = 3 +) + +// lzmaFilter declares the LZMA2 filter information stored in an xz +// block header. +type lzmaFilter struct { + dictCap int64 +} + +// String returns a representation of the LZMA filter. +func (f lzmaFilter) String() string { + return fmt.Sprintf("LZMA dict cap %#x", f.dictCap) +} + +// id returns the ID for the LZMA2 filter. +func (f lzmaFilter) id() uint64 { return lzmaFilterID } + +// MarshalBinary converts the lzmaFilter in its encoded representation. +func (f lzmaFilter) MarshalBinary() (data []byte, err error) { + c := lzma.EncodeDictCap(f.dictCap) + return []byte{lzmaFilterID, 1, c}, nil +} + +// UnmarshalBinary unmarshals the given data representation of the LZMA2 +// filter. +func (f *lzmaFilter) UnmarshalBinary(data []byte) error { + if len(data) != lzmaFilterLen { + return errors.New("xz: data for LZMA2 filter has wrong length") + } + if data[0] != lzmaFilterID { + return errors.New("xz: wrong LZMA2 filter id") + } + if data[1] != 1 { + return errors.New("xz: wrong LZMA2 filter size") + } + dc, err := lzma.DecodeDictCap(data[2]) + if err != nil { + return errors.New("xz: wrong LZMA2 dictionary size property") + } + + f.dictCap = dc + return nil +} + +// reader creates a new reader for the LZMA2 filter. +func (f lzmaFilter) reader(r io.Reader, c *ReaderConfig) (fr io.Reader, + err error) { + + config := new(lzma.Reader2Config) + if c != nil { + config.DictCap = c.DictCap + } + dc := int(f.dictCap) + if dc < 1 { + return nil, errors.New("xz: LZMA2 filter parameter " + + "dictionary capacity overflow") + } + if dc > config.DictCap { + config.DictCap = dc + } + + fr, err = config.NewReader2(r) + if err != nil { + return nil, err + } + return fr, nil +} + +// writeCloser creates a io.WriteCloser for the LZMA2 filter. +func (f lzmaFilter) writeCloser(w io.WriteCloser, c *WriterConfig, +) (fw io.WriteCloser, err error) { + config := new(lzma.Writer2Config) + if c != nil { + *config = lzma.Writer2Config{ + Properties: c.Properties, + DictCap: c.DictCap, + BufSize: c.BufSize, + Matcher: c.Matcher, + } + } + + dc := int(f.dictCap) + if dc < 1 { + return nil, errors.New("xz: LZMA2 filter parameter " + + "dictionary capacity overflow") + } + if dc > config.DictCap { + config.DictCap = dc + } + + fw, err = config.NewWriter2(w) + if err != nil { + return nil, err + } + return fw, nil +} + +// last returns true, because an LZMA2 filter must be the last filter in +// the filter list. +func (f lzmaFilter) last() bool { return true } diff --git a/vendor/github.com/ulikunitz/xz/make-docs b/vendor/github.com/ulikunitz/xz/make-docs new file mode 100644 index 00000000..a8c612ce --- /dev/null +++ b/vendor/github.com/ulikunitz/xz/make-docs @@ -0,0 +1,5 @@ +#!/bin/sh + +set -x +pandoc -t html5 -f markdown -s --css=doc/md.css -o README.html README.md +pandoc -t html5 -f markdown -s --css=doc/md.css -o TODO.html TODO.md diff --git a/vendor/github.com/ulikunitz/xz/none-check.go b/vendor/github.com/ulikunitz/xz/none-check.go new file mode 100644 index 00000000..6a56a261 --- /dev/null +++ b/vendor/github.com/ulikunitz/xz/none-check.go @@ -0,0 +1,23 @@ +// Copyright 2014-2022 Ulrich Kunitz. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package xz + +import "hash" + +type noneHash struct{} + +func (h noneHash) Write(p []byte) (n int, err error) { return len(p), nil } + +func (h noneHash) Sum(b []byte) []byte { return b } + +func (h noneHash) Reset() {} + +func (h noneHash) Size() int { return 0 } + +func (h noneHash) BlockSize() int { return 0 } + +func newNoneHash() hash.Hash { + return &noneHash{} +} diff --git a/vendor/github.com/ulikunitz/xz/reader.go b/vendor/github.com/ulikunitz/xz/reader.go new file mode 100644 index 00000000..bde1412c --- /dev/null +++ b/vendor/github.com/ulikunitz/xz/reader.go @@ -0,0 +1,359 @@ +// Copyright 2014-2022 Ulrich Kunitz. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package xz supports the compression and decompression of xz files. It +// supports version 1.0.4 of the specification without the non-LZMA2 +// filters. See http://tukaani.org/xz/xz-file-format-1.0.4.txt +package xz + +import ( + "bytes" + "errors" + "fmt" + "hash" + "io" + + "github.com/ulikunitz/xz/internal/xlog" + "github.com/ulikunitz/xz/lzma" +) + +// ReaderConfig defines the parameters for the xz reader. The +// SingleStream parameter requests the reader to assume that the +// underlying stream contains only a single stream. +type ReaderConfig struct { + DictCap int + SingleStream bool +} + +// Verify checks the reader parameters for Validity. Zero values will be +// replaced by default values. +func (c *ReaderConfig) Verify() error { + if c == nil { + return errors.New("xz: reader parameters are nil") + } + lc := lzma.Reader2Config{DictCap: c.DictCap} + if err := lc.Verify(); err != nil { + return err + } + return nil +} + +// Reader supports the reading of one or multiple xz streams. +type Reader struct { + ReaderConfig + + xz io.Reader + sr *streamReader +} + +// streamReader decodes a single xz stream +type streamReader struct { + ReaderConfig + + xz io.Reader + br *blockReader + newHash func() hash.Hash + h header + index []record +} + +// NewReader creates a new xz reader using the default parameters. +// The function reads and checks the header of the first XZ stream. The +// reader will process multiple streams including padding. +func NewReader(xz io.Reader) (r *Reader, err error) { + return ReaderConfig{}.NewReader(xz) +} + +// NewReader creates an xz stream reader. The created reader will be +// able to process multiple streams and padding unless a SingleStream +// has been set in the reader configuration c. +func (c ReaderConfig) NewReader(xz io.Reader) (r *Reader, err error) { + if err = c.Verify(); err != nil { + return nil, err + } + r = &Reader{ + ReaderConfig: c, + xz: xz, + } + if r.sr, err = c.newStreamReader(xz); err != nil { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + return nil, err + } + return r, nil +} + +var errUnexpectedData = errors.New("xz: unexpected data after stream") + +// Read reads uncompressed data from the stream. +func (r *Reader) Read(p []byte) (n int, err error) { + for n < len(p) { + if r.sr == nil { + if r.SingleStream { + data := make([]byte, 1) + _, err = io.ReadFull(r.xz, data) + if err != io.EOF { + return n, errUnexpectedData + } + return n, io.EOF + } + for { + r.sr, err = r.ReaderConfig.newStreamReader(r.xz) + if err != errPadding { + break + } + } + if err != nil { + return n, err + } + } + k, err := r.sr.Read(p[n:]) + n += k + if err != nil { + if err == io.EOF { + r.sr = nil + continue + } + return n, err + } + } + return n, nil +} + +var errPadding = errors.New("xz: padding (4 zero bytes) encountered") + +// newStreamReader creates a new xz stream reader using the given configuration +// parameters. NewReader reads and checks the header of the xz stream. +func (c ReaderConfig) newStreamReader(xz io.Reader) (r *streamReader, err error) { + if err = c.Verify(); err != nil { + return nil, err + } + data := make([]byte, HeaderLen) + if _, err := io.ReadFull(xz, data[:4]); err != nil { + return nil, err + } + if bytes.Equal(data[:4], []byte{0, 0, 0, 0}) { + return nil, errPadding + } + if _, err = io.ReadFull(xz, data[4:]); err != nil { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + return nil, err + } + r = &streamReader{ + ReaderConfig: c, + xz: xz, + index: make([]record, 0, 4), + } + if err = r.h.UnmarshalBinary(data); err != nil { + return nil, err + } + xlog.Debugf("xz header %s", r.h) + if r.newHash, err = newHashFunc(r.h.flags); err != nil { + return nil, err + } + return r, nil +} + +// readTail reads the index body and the xz footer. +func (r *streamReader) readTail() error { + index, n, err := readIndexBody(r.xz, len(r.index)) + if err != nil { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + return err + } + + for i, rec := range r.index { + if rec != index[i] { + return fmt.Errorf("xz: record %d is %v; want %v", + i, rec, index[i]) + } + } + + p := make([]byte, footerLen) + if _, err = io.ReadFull(r.xz, p); err != nil { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + return err + } + var f footer + if err = f.UnmarshalBinary(p); err != nil { + return err + } + xlog.Debugf("xz footer %s", f) + if f.flags != r.h.flags { + return errors.New("xz: footer flags incorrect") + } + if f.indexSize != int64(n)+1 { + return errors.New("xz: index size in footer wrong") + } + return nil +} + +// Read reads actual data from the xz stream. +func (r *streamReader) Read(p []byte) (n int, err error) { + for n < len(p) { + if r.br == nil { + bh, hlen, err := readBlockHeader(r.xz) + if err != nil { + if err == errIndexIndicator { + if err = r.readTail(); err != nil { + return n, err + } + return n, io.EOF + } + return n, err + } + xlog.Debugf("block %v", *bh) + r.br, err = r.ReaderConfig.newBlockReader(r.xz, bh, + hlen, r.newHash()) + if err != nil { + return n, err + } + } + k, err := r.br.Read(p[n:]) + n += k + if err != nil { + if err == io.EOF { + r.index = append(r.index, r.br.record()) + r.br = nil + } else { + return n, err + } + } + } + return n, nil +} + +// countingReader is a reader that counts the bytes read. +type countingReader struct { + r io.Reader + n int64 +} + +// Read reads data from the wrapped reader and adds it to the n field. +func (lr *countingReader) Read(p []byte) (n int, err error) { + n, err = lr.r.Read(p) + lr.n += int64(n) + return n, err +} + +// blockReader supports the reading of a block. +type blockReader struct { + lxz countingReader + header *blockHeader + headerLen int + n int64 + hash hash.Hash + r io.Reader +} + +// newBlockReader creates a new block reader. +func (c *ReaderConfig) newBlockReader(xz io.Reader, h *blockHeader, + hlen int, hash hash.Hash) (br *blockReader, err error) { + + br = &blockReader{ + lxz: countingReader{r: xz}, + header: h, + headerLen: hlen, + hash: hash, + } + + fr, err := c.newFilterReader(&br.lxz, h.filters) + if err != nil { + return nil, err + } + if br.hash.Size() != 0 { + br.r = io.TeeReader(fr, br.hash) + } else { + br.r = fr + } + + return br, nil +} + +// uncompressedSize returns the uncompressed size of the block. +func (br *blockReader) uncompressedSize() int64 { + return br.n +} + +// compressedSize returns the compressed size of the block. +func (br *blockReader) compressedSize() int64 { + return br.lxz.n +} + +// unpaddedSize computes the unpadded size for the block. +func (br *blockReader) unpaddedSize() int64 { + n := int64(br.headerLen) + n += br.compressedSize() + n += int64(br.hash.Size()) + return n +} + +// record returns the index record for the current block. +func (br *blockReader) record() record { + return record{br.unpaddedSize(), br.uncompressedSize()} +} + +// Read reads data from the block. +func (br *blockReader) Read(p []byte) (n int, err error) { + n, err = br.r.Read(p) + br.n += int64(n) + + u := br.header.uncompressedSize + if u >= 0 && br.uncompressedSize() > u { + return n, errors.New("xz: wrong uncompressed size for block") + } + c := br.header.compressedSize + if c >= 0 && br.compressedSize() > c { + return n, errors.New("xz: wrong compressed size for block") + } + if err != io.EOF { + return n, err + } + if br.uncompressedSize() < u || br.compressedSize() < c { + return n, io.ErrUnexpectedEOF + } + + s := br.hash.Size() + k := padLen(br.lxz.n) + q := make([]byte, k+s, k+2*s) + if _, err = io.ReadFull(br.lxz.r, q); err != nil { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + return n, err + } + if !allZeros(q[:k]) { + return n, errors.New("xz: non-zero block padding") + } + checkSum := q[k:] + computedSum := br.hash.Sum(checkSum[s:]) + if !bytes.Equal(checkSum, computedSum) { + return n, errors.New("xz: checksum error for block") + } + return n, io.EOF +} + +func (c *ReaderConfig) newFilterReader(r io.Reader, f []filter) (fr io.Reader, + err error) { + + if err = verifyFilters(f); err != nil { + return nil, err + } + + fr = r + for i := len(f) - 1; i >= 0; i-- { + fr, err = f[i].reader(fr, c) + if err != nil { + return nil, err + } + } + return fr, nil +} diff --git a/vendor/github.com/ulikunitz/xz/writer.go b/vendor/github.com/ulikunitz/xz/writer.go new file mode 100644 index 00000000..f693e0ae --- /dev/null +++ b/vendor/github.com/ulikunitz/xz/writer.go @@ -0,0 +1,399 @@ +// Copyright 2014-2022 Ulrich Kunitz. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package xz + +import ( + "errors" + "fmt" + "hash" + "io" + + "github.com/ulikunitz/xz/lzma" +) + +// WriterConfig describe the parameters for an xz writer. +type WriterConfig struct { + Properties *lzma.Properties + DictCap int + BufSize int + BlockSize int64 + // checksum method: CRC32, CRC64 or SHA256 (default: CRC64) + CheckSum byte + // Forces NoChecksum (default: false) + NoCheckSum bool + // match algorithm + Matcher lzma.MatchAlgorithm +} + +// fill replaces zero values with default values. +func (c *WriterConfig) fill() { + if c.Properties == nil { + c.Properties = &lzma.Properties{LC: 3, LP: 0, PB: 2} + } + if c.DictCap == 0 { + c.DictCap = 8 * 1024 * 1024 + } + if c.BufSize == 0 { + c.BufSize = 4096 + } + if c.BlockSize == 0 { + c.BlockSize = maxInt64 + } + if c.CheckSum == 0 { + c.CheckSum = CRC64 + } + if c.NoCheckSum { + c.CheckSum = None + } +} + +// Verify checks the configuration for errors. Zero values will be +// replaced by default values. +func (c *WriterConfig) Verify() error { + if c == nil { + return errors.New("xz: writer configuration is nil") + } + c.fill() + lc := lzma.Writer2Config{ + Properties: c.Properties, + DictCap: c.DictCap, + BufSize: c.BufSize, + Matcher: c.Matcher, + } + if err := lc.Verify(); err != nil { + return err + } + if c.BlockSize <= 0 { + return errors.New("xz: block size out of range") + } + if err := verifyFlags(c.CheckSum); err != nil { + return err + } + return nil +} + +// filters creates the filter list for the given parameters. +func (c *WriterConfig) filters() []filter { + return []filter{&lzmaFilter{int64(c.DictCap)}} +} + +// maxInt64 defines the maximum 64-bit signed integer. +const maxInt64 = 1<<63 - 1 + +// verifyFilters checks the filter list for the length and the right +// sequence of filters. +func verifyFilters(f []filter) error { + if len(f) == 0 { + return errors.New("xz: no filters") + } + if len(f) > 4 { + return errors.New("xz: more than four filters") + } + for _, g := range f[:len(f)-1] { + if g.last() { + return errors.New("xz: last filter is not last") + } + } + if !f[len(f)-1].last() { + return errors.New("xz: wrong last filter") + } + return nil +} + +// newFilterWriteCloser converts a filter list into a WriteCloser that +// can be used by a blockWriter. +func (c *WriterConfig) newFilterWriteCloser(w io.Writer, f []filter) (fw io.WriteCloser, err error) { + if err = verifyFilters(f); err != nil { + return nil, err + } + fw = nopWriteCloser(w) + for i := len(f) - 1; i >= 0; i-- { + fw, err = f[i].writeCloser(fw, c) + if err != nil { + return nil, err + } + } + return fw, nil +} + +// nopWCloser implements a WriteCloser with a Close method not doing +// anything. +type nopWCloser struct { + io.Writer +} + +// Close returns nil and doesn't do anything else. +func (c nopWCloser) Close() error { + return nil +} + +// nopWriteCloser converts the Writer into a WriteCloser with a Close +// function that does nothing beside returning nil. +func nopWriteCloser(w io.Writer) io.WriteCloser { + return nopWCloser{w} +} + +// Writer compresses data written to it. It is an io.WriteCloser. +type Writer struct { + WriterConfig + + xz io.Writer + bw *blockWriter + newHash func() hash.Hash + h header + index []record + closed bool +} + +// newBlockWriter creates a new block writer writes the header out. +func (w *Writer) newBlockWriter() error { + var err error + w.bw, err = w.WriterConfig.newBlockWriter(w.xz, w.newHash()) + if err != nil { + return err + } + if err = w.bw.writeHeader(w.xz); err != nil { + return err + } + return nil +} + +// closeBlockWriter closes a block writer and records the sizes in the +// index. +func (w *Writer) closeBlockWriter() error { + var err error + if err = w.bw.Close(); err != nil { + return err + } + w.index = append(w.index, w.bw.record()) + return nil +} + +// NewWriter creates a new xz writer using default parameters. +func NewWriter(xz io.Writer) (w *Writer, err error) { + return WriterConfig{}.NewWriter(xz) +} + +// NewWriter creates a new Writer using the given configuration parameters. +func (c WriterConfig) NewWriter(xz io.Writer) (w *Writer, err error) { + if err = c.Verify(); err != nil { + return nil, err + } + w = &Writer{ + WriterConfig: c, + xz: xz, + h: header{c.CheckSum}, + index: make([]record, 0, 4), + } + if w.newHash, err = newHashFunc(c.CheckSum); err != nil { + return nil, err + } + data, err := w.h.MarshalBinary() + if err != nil { + return nil, fmt.Errorf("w.h.MarshalBinary(): error %w", err) + } + if _, err = xz.Write(data); err != nil { + return nil, err + } + if err = w.newBlockWriter(); err != nil { + return nil, err + } + return w, nil + +} + +// Write compresses the uncompressed data provided. +func (w *Writer) Write(p []byte) (n int, err error) { + if w.closed { + return 0, errClosed + } + for { + k, err := w.bw.Write(p[n:]) + n += k + if err != errNoSpace { + return n, err + } + if err = w.closeBlockWriter(); err != nil { + return n, err + } + if err = w.newBlockWriter(); err != nil { + return n, err + } + } +} + +// Close closes the writer and adds the footer to the Writer. Close +// doesn't close the underlying writer. +func (w *Writer) Close() error { + if w.closed { + return errClosed + } + w.closed = true + var err error + if err = w.closeBlockWriter(); err != nil { + return err + } + + f := footer{flags: w.h.flags} + if f.indexSize, err = writeIndex(w.xz, w.index); err != nil { + return err + } + data, err := f.MarshalBinary() + if err != nil { + return err + } + if _, err = w.xz.Write(data); err != nil { + return err + } + return nil +} + +// countingWriter is a writer that counts all data written to it. +type countingWriter struct { + w io.Writer + n int64 +} + +// Write writes data to the countingWriter. +func (cw *countingWriter) Write(p []byte) (n int, err error) { + n, err = cw.w.Write(p) + cw.n += int64(n) + if err == nil && cw.n < 0 { + return n, errors.New("xz: counter overflow") + } + return +} + +// blockWriter is writes a single block. +type blockWriter struct { + cxz countingWriter + // mw combines io.WriteCloser w and the hash. + mw io.Writer + w io.WriteCloser + n int64 + blockSize int64 + closed bool + headerLen int + + filters []filter + hash hash.Hash +} + +// newBlockWriter creates a new block writer. +func (c *WriterConfig) newBlockWriter(xz io.Writer, hash hash.Hash) (bw *blockWriter, err error) { + bw = &blockWriter{ + cxz: countingWriter{w: xz}, + blockSize: c.BlockSize, + filters: c.filters(), + hash: hash, + } + bw.w, err = c.newFilterWriteCloser(&bw.cxz, bw.filters) + if err != nil { + return nil, err + } + if bw.hash.Size() != 0 { + bw.mw = io.MultiWriter(bw.w, bw.hash) + } else { + bw.mw = bw.w + } + return bw, nil +} + +// writeHeader writes the header. If the function is called after Close +// the commpressedSize and uncompressedSize fields will be filled. +func (bw *blockWriter) writeHeader(w io.Writer) error { + h := blockHeader{ + compressedSize: -1, + uncompressedSize: -1, + filters: bw.filters, + } + if bw.closed { + h.compressedSize = bw.compressedSize() + h.uncompressedSize = bw.uncompressedSize() + } + data, err := h.MarshalBinary() + if err != nil { + return err + } + if _, err = w.Write(data); err != nil { + return err + } + bw.headerLen = len(data) + return nil +} + +// compressed size returns the amount of data written to the underlying +// stream. +func (bw *blockWriter) compressedSize() int64 { + return bw.cxz.n +} + +// uncompressedSize returns the number of data written to the +// blockWriter +func (bw *blockWriter) uncompressedSize() int64 { + return bw.n +} + +// unpaddedSize returns the sum of the header length, the uncompressed +// size of the block and the hash size. +func (bw *blockWriter) unpaddedSize() int64 { + if bw.headerLen <= 0 { + panic("xz: block header not written") + } + n := int64(bw.headerLen) + n += bw.compressedSize() + n += int64(bw.hash.Size()) + return n +} + +// record returns the record for the current stream. Call Close before +// calling this method. +func (bw *blockWriter) record() record { + return record{bw.unpaddedSize(), bw.uncompressedSize()} +} + +var errClosed = errors.New("xz: writer already closed") + +var errNoSpace = errors.New("xz: no space") + +// Write writes uncompressed data to the block writer. +func (bw *blockWriter) Write(p []byte) (n int, err error) { + if bw.closed { + return 0, errClosed + } + + t := bw.blockSize - bw.n + if int64(len(p)) > t { + err = errNoSpace + p = p[:t] + } + + var werr error + n, werr = bw.mw.Write(p) + bw.n += int64(n) + if werr != nil { + return n, werr + } + return n, err +} + +// Close closes the writer. +func (bw *blockWriter) Close() error { + if bw.closed { + return errClosed + } + bw.closed = true + if err := bw.w.Close(); err != nil { + return err + } + s := bw.hash.Size() + k := padLen(bw.cxz.n) + p := make([]byte, k+s) + bw.hash.Sum(p[k:k]) + if _, err := bw.cxz.w.Write(p); err != nil { + return err + } + return nil +} diff --git a/vendor/modules.txt b/vendor/modules.txt index bb6a448e..30d04aac 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -83,6 +83,12 @@ github.com/nats-io/nuid github.com/novox/mesh-host/internal/declaration github.com/novox/mesh-host/rootsearch github.com/novox/mesh-host/validate +# github.com/ulikunitz/xz v0.5.17 +## explicit; go 1.20 +github.com/ulikunitz/xz +github.com/ulikunitz/xz/internal/hash +github.com/ulikunitz/xz/internal/xlog +github.com/ulikunitz/xz/lzma # golang.org/x/crypto v0.57.0 ## explicit; go 1.26.0 golang.org/x/crypto/acme From 77acb4009282e80a94dab4dde10732882a89f1a7 Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 12 Oct 2026 14:28:15 +0200 Subject: [PATCH 06/26] Let a directory's owner follow the account its module runs as (issue 279) A media manager runs as the puid/pgid its assignment sets per node, but a declared directory's owner was literal, so a folder the app must write was refused on a machine whose identity was not the image's default. A directory's owner now takes ${setting:} (user, or user:group), filled at composition and refused unless the value is a whole number, and the module check refuses any other shape or a key the module asks for nowhere else. --- internal/catalogue/manifest.go | 1 + internal/catalogue/setting_in_owner_test.go | 152 +++++++++++++++++++ internal/catalogue/setting_into.go | 125 ++++++++++++++- internal/catalogue/unconsumed_placeholder.go | 2 +- 4 files changed, 276 insertions(+), 4 deletions(-) create mode 100644 internal/catalogue/setting_in_owner_test.go diff --git a/internal/catalogue/manifest.go b/internal/catalogue/manifest.go index be5d13dd..024d850a 100644 --- a/internal/catalogue/manifest.go +++ b/internal/catalogue/manifest.go @@ -1859,6 +1859,7 @@ func ParseManifest(raw []byte) (Manifest, error) { problems = append(problems, invokeProblems(m)...) problems = append(problems, replacesProblems(m)...) problems = append(problems, UnitSettingProblems(m)...) + problems = append(problems, OwnerSettingProblems(m)...) problems = append(problems, endpointNameProblems(m)...) problems = append(problems, RouteProblems(m)...) for _, port := range m.Guards { diff --git a/internal/catalogue/setting_in_owner_test.go b/internal/catalogue/setting_in_owner_test.go new file mode 100644 index 00000000..e7704639 --- /dev/null +++ b/internal/catalogue/setting_in_owner_test.go @@ -0,0 +1,152 @@ +package catalogue + +import ( + "strings" + "testing" +) + +// A declared directory may be owned by the account its module runs as (novox/hq issue 279): a media +// manager runs as the puid and pgid its assignment sets per node, and a folder it must write was owned +// by the manifest's literal, so the app refused it on a machine whose identity was not the image's. + +func recycler() Manifest { + return Manifest{Module: "sonarr", + Resources: []map[string]any{ + {"id": "recycle-bin", "type": "directory", "path": "/srv/sonarr/recycle", "mode": "0755", + "owner": "${setting:puid}:${setting:pgid}"}, + {"id": "identity", "type": "file", "path": "/srv/sonarr/identity.env", "mode": "0644", + "content": "PUID=${setting:puid}\nPGID=${setting:pgid}\n"}, + }, + } +} + +func copied(r map[string]any) map[string]any { + out := map[string]any{} + for k, v := range r { + out[k] = v + } + return out +} + +// The node's identity reaches the directory's owner, the node's layer over the mesh's, a number given +// as a string or as a JSON number alike. +func TestASettingOwnsADirectory(t *testing.T) { + m := recycler() + layers := []Layer{ + {From: "the mesh", Values: map[string]any{"puid": "1000", "pgid": "1000"}}, + {From: "this node", Values: map[string]any{"puid": float64(1001), "pgid": "2000"}}, + } + r := copied(m.Resources[0]) + if err := settingInto(r, layers, m.Module); err != nil { + t.Fatal(err) + } + if r["owner"] != "1001:2000" { + t.Fatalf("owned by %q, not 1001:2000", r["owner"]) + } + user := copied(m.Resources[0]) + user["owner"] = "${setting:puid}" + if err := settingInto(user, layers, m.Module); err != nil || user["owner"] != "1001" { + t.Fatalf("user alone: %v, %v", user["owner"], err) + } +} + +// Composed for a machine, the way a push writes it: the directory reaches the node-engine owned by the +// number the node's settings name, never by the placeholder. +func TestAComposedMachineGetsTheOwnerTheSettingNames(t *testing.T) { + r := anAdoptedAnchor() + r.Modules = append(r.Modules, recycler()) + with := anchorRendering(false) + with.Settings["sonarr"] = []Layer{{From: "anchor", Values: map[string]any{"puid": "1001", "pgid": "2000"}}} + composed, err := r.Compose(with) + if err != nil { + t.Fatal(err) + } + if why, left := composed.LeftOut["sonarr"]; left { + t.Fatalf("left out: %s", why) + } + got := byID(composed.Resources)["sonarr.recycle-bin"] + if got["owner"] != "1001:2000" { + t.Fatalf("composed owned by %v, not 1001:2000", got["owner"]) + } +} + +// Only a whole number owns a directory: an account's name, text, a sign, a fraction or a value past the +// kernel's ids is refused by name, and the directory is left as it was. So is a key nothing sets. +func TestAnOwnerSettingThatIsNoNumberIsRefused(t *testing.T) { + m := recycler() + for _, bad := range []any{"abc", "root", "", "-1", "+5", "1000:1000", "10 00", "1000\nx", float64(1.5), + float64(-3), "4294967295", "99999999999", true} { + r := copied(m.Resources[0]) + err := settingInto(r, []Layer{{From: "ace", Values: map[string]any{"puid": bad, "pgid": "1000"}}}, m.Module) + if err == nil || !strings.Contains(err.Error(), `"puid"`) || !strings.Contains(err.Error(), "whole number") { + t.Errorf("%v: %v", bad, err) + } + if r["owner"] != m.Resources[0]["owner"] { + t.Errorf("%v: the owner was changed on refusal: %v", bad, r["owner"]) + } + } + r := copied(m.Resources[0]) + if err := settingInto(r, nil, m.Module); err == nil || !strings.Contains(err.Error(), "${setting:puid}") { + t.Errorf("nothing set: %v", err) + } + if err := JudgeSettings(m, []Layer{{From: "ace", Values: map[string]any{"puid": "media", "pgid": "1000"}}}, false); err == nil || + !strings.Contains(err.Error(), "whole number") { + t.Errorf("a stored name is not refused when judged: %v", err) + } +} + +// Where the manifest is read: an owner holding a setting is a whole placeholder or a whole number on +// each side of the colon, and every key is one the module asks for elsewhere or declares a default for. +func TestAnOwnerSettingIsJudgedWhereTheManifestIsRead(t *testing.T) { + ok := []string{"${setting:puid}:${setting:pgid}", "${setting:puid}", "${setting:puid}:1000", "1000:${setting:pgid}", + "1000:1000", "media:media"} + bad := []string{ + "${setting:puid}x", + "x${setting:puid}", + "${setting:puid}:${setting:pgid}:1", + "${setting:puid}${setting:pgid}", + "media:${setting:pgid}", + "${setting:puid}:", + "${setting:other}:${setting:pgid}", + } + for _, owner := range append(ok, bad...) { + m := recycler() + m.Resources[0] = copied(m.Resources[0]) + m.Resources[0]["owner"] = owner + err := parsed(t, m) + refused := err != nil && strings.Contains(err.Error(), "owner") + want := false + for _, b := range bad { + want = want || b == owner + } + if refused != want { + t.Errorf("%s: refused %v, want %v (%v)", owner, refused, want, err) + } + } + // A key with a default is declared, though nothing else asks for it. + m := recycler() + m.Resources = m.Resources[:1] + m.Resources[0] = copied(m.Resources[0]) + m.Resources[0]["owner"] = "${setting:workers}" + m.Settings = map[string]SettingDeclaration{"workers": {Kind: KindPreference, Default: float64(4), Why: "a test"}} + if err := parsed(t, m); err != nil && strings.Contains(err.Error(), "owner") { + t.Errorf("a declared key was refused: %v", err) + } + // A file's owner takes no setting: the sweep says so. + f := recycler() + f.Resources[1] = copied(f.Resources[1]) + f.Resources[1]["owner"] = "${setting:puid}" + if err := parsed(t, f); err == nil || !strings.Contains(err.Error(), "not filled in this field") { + t.Errorf("a file's owner setting passed: %v", err) + } +} + +// A key a directory's owner asks for is a destination, so setting it is not called stray. +func TestASettingAnOwnerAsksForIsNotStray(t *testing.T) { + m := recycler() + m.Resources = m.Resources[:1] + stray := strings.Join(UnusedSettings(m, []Layer{{From: "ace", Values: map[string]any{"puid": "1001", "pgid": "2000"}}}), "; ") + if strings.Contains(stray, "puid") || strings.Contains(stray, "pgid") { + t.Errorf("called stray: %s", stray) + } +} diff --git a/internal/catalogue/setting_into.go b/internal/catalogue/setting_into.go index 30967997..2c91ec63 100644 --- a/internal/catalogue/setting_into.go +++ b/internal/catalogue/setting_into.go @@ -4,6 +4,7 @@ import ( "fmt" "regexp" "sort" + "strconv" "strings" ) @@ -15,7 +16,8 @@ import ( // (novox/hq issues 122, 134). ADR 0112 names the operator as one of the four providers; this is the // operator answering. // -// `${setting:}` in a file's content, and in a service's unit name, is filled from the module's settings layers — the mesh's, +// `${setting:}` in a file's content, in a service's unit name and in a directory's owner (a uid or +// gid only, issue 279) is filled from the module's settings layers — the mesh's, // then this node's — the same layers a mergeable JSON file and a contribution already take, so // `settings set ` is the one place a person's values go. **Refused when no layer sets it**, // naming the key and the remedy: a definition that carried a default for a mail domain would be @@ -59,6 +61,9 @@ func settingInto(resource map[string]any, layers []Layer, module string) error { if fmt.Sprint(resource["type"]) == "service" { return settingIntoUnit(resource, layers, module) } + if fmt.Sprint(resource["type"]) == "directory" { + return settingIntoOwner(resource, layers, module) + } if fmt.Sprint(resource["type"]) != "file" { return nil } @@ -143,6 +148,104 @@ func settingIntoUnit(resource map[string]any, layers []Layer, module string) err return nil } +// A declared directory may be owned by the account its module runs as (novox/hq issue 279). +// +// A containerised module runs as the identity its assignment sets per node — its puid and pgid +// settings, written into the environment file its image reads (ADR 0112, issue 153) — and a directory +// its app must write, beyond the one its image re-owns at start, was owned by whatever the manifest +// spelt: right on a machine whose identity matched the image's default, refused by the app on one whose +// did not. So a directory's `owner` may name the settings that identity is: `${setting:}` for the +// user alone, `${setting:}:${setting:}` for both, filled from the same layers at +// composition. +// +// **Whole numbers only.** An owner a setting fills is a uid or a gid — never an account's name, never +// text — so no value can put anything into the ownership the node-engine applies but a number; one that +// is not is refused at composition, naming the key, and the directory is left as it was. The shape is +// judged where the manifest is read (OwnerSettingProblems): each side of the colon a whole placeholder +// or a whole number, and every key one the module itself asks for elsewhere — in a file, a unit, a +// contribution or a served fact — or declares a default for, so the owner follows the identity its +// process is told it runs as, never a key that reaches nothing else. + +// ownerSide is one side of an owner a setting may fill: a whole placeholder or a whole number. +var ownerSide = `(?:\$\{setting:[a-z0-9][a-z0-9_.-]*\}|[0-9]+)` + +// ownerWithSetting is the only shape an owner holding a setting may have: user, or user:group. +var ownerWithSetting = regexp.MustCompile(`^` + ownerSide + `(?::` + ownerSide + `)?$`) + +// wholeID is whether a value is a uid or a gid: a whole number below the kernel's "no id" (2^32-1). +func wholeID(v string) bool { + if v == "" || strings.TrimLeft(v, "0123456789") != "" { + return false + } + n, err := strconv.ParseUint(v, 10, 32) + return err == nil && n < 1<<32-1 +} + +// OwnerSettingProblems refuses, where a manifest is read, a directory whose owner carries a setting +// in any shape but a whole placeholder or number on each side of the colon, or names a key the module +// asks for nowhere else and declares no default for. +func OwnerSettingProblems(m Manifest) []string { + var problems []string + asked := settingKeysAskedOutsideOwners(m) + for k := range m.Settings { + asked[k] = true + } + for _, r := range m.Resources { + owner, _ := r["owner"].(string) + if !strings.Contains(owner, "${setting:") { + continue + } + if fmt.Sprint(r["type"]) != "directory" { + // Said by the placeholder sweep: only a directory's owner takes a setting. + continue + } + if !ownerWithSetting.MatchString(owner) { + problems = append(problems, fmt.Sprintf("%s: the directory %v's owner %q carries a setting in a shape "+ + "the mesh does not fill; an owner names the account by number only, each side of the colon a whole "+ + "${setting:key} or a whole number (${setting:puid}:${setting:pgid}) (novox/hq issue 279)", + m.Module, r["id"], owner)) + continue + } + for _, key := range settingsUsed(owner) { + if !asked[key] { + problems = append(problems, fmt.Sprintf("%s: the directory %v's owner says ${setting:%s}, and the "+ + "module asks for %q nowhere else and declares no default for it: an owner follows the identity "+ + "the module's process is told it runs as (its identity file's ${setting:%s}), never a key of its "+ + "own (novox/hq issue 279)", m.Module, r["id"], key, key, key)) + } + } + } + return problems +} + +// settingIntoOwner fills ${setting:…} in a directory's owner, refusing — with the key and why, the +// resource left as it was — a key nothing sets and a value that is not a whole number. +func settingIntoOwner(resource map[string]any, layers []Layer, module string) error { + owner, ok := resource["owner"].(string) + if !ok { + return nil + } + for _, key := range settingsUsed(owner) { + value, set := settingValue(layers, key) + if !set { + return &UnsetSettingError{Module: module, Setting: key, said: fmt.Sprintf( + "%s has a directory %v whose owner says ${setting:%s}, and nothing sets %q for it — an operator's "+ + "value is the assignment's, never the definition's (novox/hq ADR 0112): "+ + "`settings set %s ` with {%q: …}%s", + module, resource["id"], key, key, module, key, orNoSettings(layers))} + } + v := plainly(value) + if !wholeID(v) { + return fmt.Errorf("%s: the setting %q is %q, which cannot own the directory %v: an owner a setting "+ + "names is a uid or a gid, a whole number, never an account's name or other text (novox/hq issue 279)", + module, key, v, resource["id"]) + } + owner = strings.ReplaceAll(owner, "${setting:"+key+"}", v) + } + resource["owner"] = owner + return nil +} + func settingValue(layers []Layer, key string) (any, bool) { var value any set := false @@ -171,9 +274,25 @@ func orNoSettings(layers []Layer) string { return "; set today: " + strings.Join(keys, ", ") } -// settingKeysUsedBy is every key a module's files, contributions and served facts ask for, so a -// setting that lands in one is not called stray. +// settingKeysUsedBy is every key a module's files, units, directory owners, contributions and served +// facts ask for, so a setting that lands in one is not called stray. func settingKeysUsedBy(m Manifest) map[string]bool { + used := settingKeysAskedOutsideOwners(m) + for _, r := range m.Resources { + if fmt.Sprint(r["type"]) == "directory" { + if owner, ok := r["owner"].(string); ok { + for _, k := range settingsUsed(owner) { + used[k] = true + } + } + } + } + return used +} + +// settingKeysAskedOutsideOwners is every key a module's files, units, contributions and served facts +// ask for: what a directory's owner may follow (OwnerSettingProblems). +func settingKeysAskedOutsideOwners(m Manifest) map[string]bool { used := map[string]bool{} note := func(s string) { for _, k := range settingsUsed(s) { diff --git a/internal/catalogue/unconsumed_placeholder.go b/internal/catalogue/unconsumed_placeholder.go index 23f81a93..5c681f4f 100644 --- a/internal/catalogue/unconsumed_placeholder.go +++ b/internal/catalogue/unconsumed_placeholder.go @@ -56,7 +56,7 @@ type filler struct { // seatInto, machineInto and contributionsInto. A pass that comes to read another field adds it here, // or the sweep refuses the placeholder it would have filled. var fillers = []filler{ - {namespace: "setting", patterns: []*regexp.Regexp{settingRef}, fields: map[string][]string{"file": {"content"}, "service": {"unit"}}}, + {namespace: "setting", patterns: []*regexp.Regexp{settingRef}, fields: map[string][]string{"file": {"content"}, "service": {"unit"}, "directory": {"owner"}}}, {namespace: "dir", patterns: []*regexp.Regexp{dirRef}, fields: map[string][]string{"*": {"path", "content", "volumes", "env", "env-file"}}}, {namespace: "access", patterns: []*regexp.Regexp{accessRef}, fields: map[string][]string{"*": {"path", "content", "volumes", "env", "env-file"}}}, {namespace: "secret", patterns: []*regexp.Regexp{placeholder}, fields: map[string][]string{"file": {"content"}}, From a371de37fef6274a18aca247e616eb4396e78174 Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 12 Oct 2026 14:48:58 +0200 Subject: [PATCH 07/26] Keep a directory owner's settings at the controller's terminal (issue 279) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Root chowns a directory to the number its owner's setting names, so a key an owner asks for must not change through the settings verb: any caller could have set puid to 0 or to its own uid. TerminalKeys now takes every ${setting:…} in a directory's owner, whatever else asks for it, and the owner is written as the parsed number so "0001" never reaches the node-engine as text (review 60). --- cmd/mesh-controller/terminal_settings_test.go | 55 +++++++++++++++++++ internal/catalogue/setting_in_owner_test.go | 29 ++++++++++ internal/catalogue/setting_into.go | 15 +++-- internal/catalogue/terminal_keys.go | 14 ++++- testdata/store-tests | 1 + 5 files changed, 106 insertions(+), 8 deletions(-) diff --git a/cmd/mesh-controller/terminal_settings_test.go b/cmd/mesh-controller/terminal_settings_test.go index 6eb7e6a9..a70d4ead 100644 --- a/cmd/mesh-controller/terminal_settings_test.go +++ b/cmd/mesh-controller/terminal_settings_test.go @@ -344,3 +344,58 @@ func TestAnyKeyOfAModuleWithATrustedMergeableFileIsRefusedThroughAVerb(t *testin t.Fatalf("a file marked untrusted through the verb: %v", err) } } + +// Who owns a module's directory is said at the terminal alone (novox/hq issue 279): a directory's owner may name +// the settings the module runs as, root chowns the folder to them at the next push, and through the settings verb +// any caller could have set puid to 0 or to its own uid. The key is refused through every verb route even where a +// file nothing trusts also asks for it, and taken at the terminal; the layer a refused call meets is unchanged. +func TestADirectoryOwnersKeyIsRefusedThroughAVerb(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + register(t, open, catalogue.Manifest{Module: "sonarr", Version: "1", + Resources: []map[string]any{ + {"id": "recycle-bin", "type": "directory", "mode": "0755", "owner": "${setting:puid}:${setting:pgid}"}, + {"id": "identity", "type": "file", "path": "/etc/sonarr.env", "mode": "0644", "trusted": false, + "content": "PUID=${setting:puid}\nPGID=${setting:pgid}\nX=${setting:x}\n"}}}) + if err := atTheTerminal(t, "settings", "set", "sonarr", `{"puid":1001,"pgid":2000,"x":0}`, + "--node", "anchor"); err != nil { + t.Fatalf("the owner at the terminal: %v", err) + } + if _, err := assign(ctx, open, "anchor", "sonarr"); err != nil { + t.Fatal(err) + } + layer := func() string { + t.Helper() + values, _, err := open.inventory.Layer(ctx, "anchor", "sonarr") + if err != nil { + t.Fatal(err) + } + raw, _ := json.Marshal(values) + return string(raw) + } + kept := layer() + refused := func(what string, err error) { + t.Helper() + if err == nil || !strings.Contains(err.Error(), "controller's terminal") { + t.Fatalf("%s: %v", what, err) + } + } + refused("puid 0 through the verb", throughVerb(t, "settings", map[string]any{"module": "sonarr", + "node": "anchor", "values": `{"puid":0,"pgid":2000,"x":0}`})) + refused("pgid through the verb, mesh-wide", throughVerb(t, "settings", map[string]any{"module": "sonarr", + "values": `{"pgid":0}`})) + refused("cleared through the verb", throughVerb(t, "settings", map[string]any{"module": "sonarr", + "node": "anchor", "clear": "true"})) + if err := throughVerb(t, "command", map[string]any{ + "command": `settings set sonarr '{"puid":0,"pgid":2000,"x":0}' --node anchor`}); err == nil { + t.Fatal("the command verb set the owner") + } + if got := layer(); got != kept { + t.Fatalf("a refused call changed the layer: %s, was %s", got, kept) + } + // Another key, the owner kept, is still the verb's. + if err := throughVerb(t, "settings", map[string]any{"module": "sonarr", "node": "anchor", + "values": `{"puid":1001,"pgid":2000,"x":1}`}); err != nil { + t.Fatalf("another key through the verb, the owner kept: %v", err) + } +} diff --git a/internal/catalogue/setting_in_owner_test.go b/internal/catalogue/setting_in_owner_test.go index e7704639..00affaa6 100644 --- a/internal/catalogue/setting_in_owner_test.go +++ b/internal/catalogue/setting_in_owner_test.go @@ -150,3 +150,32 @@ func TestASettingAnOwnerAsksForIsNotStray(t *testing.T) { t.Errorf("called stray: %s", stray) } } + +// An owner is written as the number parsed: "0001" and "00" reach the node-engine as 1 and 0, never as text. +// Uid 0 is allowed, because a key an owner asks for is the terminal's alone. +func TestAnOwnerSettingIsWrittenAsTheNumber(t *testing.T) { + m := recycler() + r := copied(m.Resources[0]) + if err := settingInto(r, []Layer{{From: "ace", Values: map[string]any{"puid": "0001", "pgid": "00"}}}, m.Module); err != nil { + t.Fatal(err) + } + if r["owner"] != "1:0" { + t.Fatalf("owned by %q, not 1:0", r["owner"]) + } +} + +// Every key a directory's owner asks for is the controller's terminal's alone (novox/hq issue 279, issue 339): +// root chowns the folder to it, so it is terminal-only whatever else asks for it — a file nothing trusts, a +// default, or nothing at all. +func TestAnOwnersKeyIsTheTerminals(t *testing.T) { + m := recycler() + m.Resources[1] = copied(m.Resources[1]) + m.Resources[1]["trusted"] = false + m.Settings = map[string]SettingDeclaration{"pgid": {Kind: KindPreference, Default: "1000", Why: "a test"}} + keys := strings.Join(TerminalKeys(m), ",") + for _, k := range []string{"puid", "pgid"} { + if !strings.Contains(","+keys+",", ","+k+",") { + t.Errorf("%s is not the terminal's: %s", k, keys) + } + } +} diff --git a/internal/catalogue/setting_into.go b/internal/catalogue/setting_into.go index 2c91ec63..ee04ee18 100644 --- a/internal/catalogue/setting_into.go +++ b/internal/catalogue/setting_into.go @@ -172,13 +172,14 @@ var ownerSide = `(?:\$\{setting:[a-z0-9][a-z0-9_.-]*\}|[0-9]+)` // ownerWithSetting is the only shape an owner holding a setting may have: user, or user:group. var ownerWithSetting = regexp.MustCompile(`^` + ownerSide + `(?::` + ownerSide + `)?$`) -// wholeID is whether a value is a uid or a gid: a whole number below the kernel's "no id" (2^32-1). -func wholeID(v string) bool { +// wholeID is a value as a uid or a gid, and whether it is one: a whole number below the kernel's "no id" +// (2^32-1). Uid 0 is one: a key an owner asks for is the controller's terminal's alone (TerminalKeys). +func wholeID(v string) (uint64, bool) { if v == "" || strings.TrimLeft(v, "0123456789") != "" { - return false + return 0, false } n, err := strconv.ParseUint(v, 10, 32) - return err == nil && n < 1<<32-1 + return n, err == nil && n < 1<<32-1 } // OwnerSettingProblems refuses, where a manifest is read, a directory whose owner carries a setting @@ -235,12 +236,14 @@ func settingIntoOwner(resource map[string]any, layers []Layer, module string) er module, resource["id"], key, key, module, key, orNoSettings(layers))} } v := plainly(value) - if !wholeID(v) { + id, whole := wholeID(v) + if !whole { return fmt.Errorf("%s: the setting %q is %q, which cannot own the directory %v: an owner a setting "+ "names is a uid or a gid, a whole number, never an account's name or other text (novox/hq issue 279)", module, key, v, resource["id"]) } - owner = strings.ReplaceAll(owner, "${setting:"+key+"}", v) + // Written as the number parsed, so "0001" reaches the node-engine as 1, never as text. + owner = strings.ReplaceAll(owner, "${setting:"+key+"}", strconv.FormatUint(id, 10)) } resource["owner"] = owner return nil diff --git a/internal/catalogue/terminal_keys.go b/internal/catalogue/terminal_keys.go index cd5bb8e1..d3cf47aa 100644 --- a/internal/catalogue/terminal_keys.go +++ b/internal/catalogue/terminal_keys.go @@ -33,6 +33,9 @@ import ( // 4. **A module's whole layer, when it has a mergeable file not marked `"trusted": false`** (novox/hq issue 340, // TrustedMergeable). A mergeable file takes any key a layer sets, not only those its content names, so no list // of keys covers it: an empty runtime configuration a provider reads would take a `url` of the caller's. +// 5. **Every setting a directory's owner asks for** (novox/hq issue 279): root chowns the directory to the number +// at the next push, so through a verb any caller could hand itself — or uid 0 — a module's folder. Whatever +// else asks for the key, a file marked `"trusted": false` included, the owner makes it the terminal's. // // Derived from the manifest, never listed by hand, so a provider or a trusted file added tomorrow is covered. @@ -42,8 +45,8 @@ import ( const TrustedField = "trusted" // TerminalKeys are the settings keys of a module that are set at the controller's terminal alone: places and -// accesses, every key its provisions serve and every setting a served value asks for, and every setting a file -// asks for unless it says `"trusted": false`. Places and accesses first, then the rest sorted. +// accesses, every key its provisions serve and every setting a served value asks for, every setting a file +// asks for unless it says `"trusted": false`, and every setting a directory's owner asks for. Places and accesses first, then the rest sorted. func TerminalKeys(m Manifest) []string { keys := map[string]bool{} for _, served := range m.Serves { @@ -57,6 +60,13 @@ func TerminalKeys(m Manifest) []string { } } for _, r := range m.Resources { + if fmt.Sprint(r["type"]) == "directory" { + owner, _ := r["owner"].(string) + for _, asked := range settingsUsed(owner) { + keys[asked] = true + } + continue + } if fmt.Sprint(r["type"]) != "file" { continue } diff --git a/testdata/store-tests b/testdata/store-tests index 67d064c8..591ba44c 100644 --- a/testdata/store-tests +++ b/testdata/store-tests @@ -26,3 +26,4 @@ ./cmd/mesh-controller TestAnUnassignThatLeavesItsNodeUnresolvedSaysSoOnceInItsAnswer ./cmd/mesh-controller TestAnUnassignThatLeavesItsNodeResolvedSaysNothingOfIt ./cmd/mesh-controller TestAnAssignThatLeavesItsNodeUnresolvedSaysOnlyThePlansRefusal +./cmd/mesh-controller TestADirectoryOwnersKeyIsRefusedThroughAVerb From d4de4543ccbf803ca10d3c706c5640a95b485b42 Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 12 Oct 2026 15:39:08 +0200 Subject: [PATCH 08/26] Judge S21 by time since the first failing report while the machine is heard, not by how many reports said it (issue 233 review) --- cmd/mesh-controller/apply_failing.go | 22 ++++----- cmd/mesh-controller/apply_failing_test.go | 58 +++++++++++++++++++---- cmd/mesh-controller/signals.go | 7 +-- cmd/mesh-controller/signals_test.go | 8 ++-- 4 files changed, 68 insertions(+), 27 deletions(-) diff --git a/cmd/mesh-controller/apply_failing.go b/cmd/mesh-controller/apply_failing.go index 5e9230e9..6daf960d 100644 --- a/cmd/mesh-controller/apply_failing.go +++ b/cmd/mesh-controller/apply_failing.go @@ -25,20 +25,21 @@ const packageManagerKey = "the package manager (" // watchApplyFailing is S21: **a machine that does not apply what it was sent is said by the mesh, not only // to someone who asks for `status`** (novox/hq issue 233, issue 187's family). A workstation refused every // declaration for an hour and a half because its package manager did not answer, and the mesh's only word -// on it was a line in `status`. Raised once its reports have said the same failure for longer than one -// apply interval, over at least two reports — a laptop shut for a week has had one attempt, not a week of -// them — naming the refusal's or the failure's words; cleared by a clean apply. Not while the machine is -// silent or asleep: S1 says silent, and asleep is not lost. +// on it was a line in `status`. Raised once the machine's first report of this failure is older than one apply +// interval **while the machine is heard** — judged by time, not by how many reports said it: a node-engine +// says a re-apply only when how it went changed, so the same failure is one report however long it lasts. +// Names the refusal's or the failure's words; cleared by the clean apply the node-engine then says. Not while +// the machine is silent, asleep or not known: S1 says silent, and asleep is not lost. func watchApplyFailing(f *signalFacts) []conditions.Observation { - quiet := map[string]bool{} + heard := map[string]bool{} for _, m := range f.machines { - if m.asleep() || (!m.lastHeard.IsZero() && f.now.Sub(m.lastHeard) > heartbeatBound(m.every)) { - quiet[m.name] = true + if !m.asleep() && !m.lastHeard.IsZero() && f.now.Sub(m.lastHeard) <= heartbeatBound(m.every) { + heard[m.name] = true } } var out []conditions.Observation for _, d := range f.doing { - if !d.Wrong() || d.Since == nil || d.Times < 2 || f.now.Sub(*d.Since) <= applyEvery || quiet[d.Node] { + if !d.Wrong() || d.Since == nil || f.now.Sub(*d.Since) <= applyEvery || !heard[d.Node] { continue } out = append(out, applyFailing(d, f.now)) @@ -57,14 +58,13 @@ func applyFailing(d inventory.Doing, now time.Time) conditions.Observation { if d.Outcome != inventory.OutcomeRefused { how = "applied its declaration only in part" } - o.Summary = fmt.Sprintf("%s has %s on %d reports in a row since %s (bound %s): %s", m, how, d.Times, since, - ago(applyEvery), failedNames(d)) + o.Summary = fmt.Sprintf("%s has %s since %s (bound %s): %s", m, how, since, ago(applyEvery), failedNames(d)) o.Headline = m + " is not applying its update" o.Explanation = fmt.Sprintf("%s has %s for %s, and trying again has not helped. The details say what fails.", m, plainHow(d), humanDuration(now.Sub(*d.Since))) if tool := unansweringTool(d); tool != "" { o.Summary = fmt.Sprintf("%s: %s does not answer, so no package reaches it; everything else it was sent "+ - "is applied, on %d reports in a row since %s (bound %s)", m, tool, d.Times, since, ago(applyEvery)) + "is applied, since %s (bound %s)", m, tool, since, ago(applyEvery)) o.Headline = fmt.Sprintf("%s cannot install packages", m) o.Explanation = fmt.Sprintf("The package manager on %s (%s) has not answered for %s, so no package reaches "+ "it. Everything else is still applied, so an update that repairs it can land; if none does, it needs a "+ diff --git a/cmd/mesh-controller/apply_failing_test.go b/cmd/mesh-controller/apply_failing_test.go index 36248b3f..bb76f910 100644 --- a/cmd/mesh-controller/apply_failing_test.go +++ b/cmd/mesh-controller/apply_failing_test.go @@ -1,10 +1,12 @@ package main import ( + "context" "strings" "testing" "time" + "github.com/novox/mesh-controller/internal/conditions" "github.com/novox/mesh-controller/internal/inventory" ) @@ -37,8 +39,8 @@ func TestAPackageManagerThatDoesNotAnswerIsRaisedNamingItsWords(t *testing.T) { if o.Key() != "machine.anchor.apply-failing" || o.Machine != "anchor" { t.Errorf("the condition is not the machine's: %s", o.Key()) } - if !strings.Contains(o.Summary, "pacman does not answer") || !strings.Contains(o.Summary, "18 reports") { - t.Errorf("the summary does not name the tool and how long: %s", o.Summary) + if !strings.Contains(o.Summary, "pacman does not answer") || !strings.Contains(o.Summary, "since 2026-10-04 12:30") { + t.Errorf("the summary does not name the tool and since when: %s", o.Summary) } if strings.Contains(o.Summary, "/etc/") { t.Errorf("the summary carries a path, which belongs in the evidence: %s", o.Summary) @@ -66,18 +68,56 @@ func TestARefusalThatGoesOnIsRaised(t *testing.T) { } } -// **One attempt is not a fault that goes on, and a silent or sleeping machine is S1's**: a laptop shut for a -// week comes back with one report of an old failure; a machine not heard from says silent, not this. -func TestOneReportOrAQuietMachineIsNotRaised(t *testing.T) { +// **Judged by time while the machine is heard, not by how many reports said it** (review of novox/hq issue 233): +// a node-engine says the same failure once, so one report an apply interval old is a fault that goes on; one +// younger is not yet; and a silent or sleeping machine is S1's. +func TestOneReportOlderThanAnApplyIsRaisedAndAQuietMachineIsNot(t *testing.T) { now := time.Date(2026, 10, 4, 14, 0, 0, 0, time.UTC) f := calm(now) - f.doing = []inventory.Doing{unansweringPackageManager(now, 7*24*time.Hour, 1)} - if got := watchApplyFailing(f); len(got) != 0 { - t.Errorf("one report was raised as a failure that goes on: %+v", got) + f.doing = []inventory.Doing{unansweringPackageManager(now, 6*time.Minute, 1)} + if got := watchApplyFailing(f); len(got) != 1 { + t.Errorf("one report of a failure six minutes old was not raised: %+v", got) } - f.doing = []inventory.Doing{unansweringPackageManager(now, time.Hour, 12)} + f.doing = []inventory.Doing{unansweringPackageManager(now, 4*time.Minute, 1)} + if got := watchApplyFailing(f); len(got) != 0 { + t.Errorf("a failure four minutes old was raised before one apply could clear it: %+v", got) + } + f.doing = []inventory.Doing{unansweringPackageManager(now, time.Hour, 1)} f.machines[0].lastHeard = now.Add(-time.Hour) if got := watchApplyFailing(f); len(got) != 0 { t.Errorf("a silent machine was raised for its last report: %+v", got) } + f = calm(now) + f.doing = []inventory.Doing{unansweringPackageManager(now, time.Hour, 1)} + f.doing[0].Node = "a-machine-the-mesh-does-not-know" + if got := watchApplyFailing(f); len(got) != 0 { + t.Errorf("a machine not heard of was raised: %+v", got) + } +} + +// **A failed re-apply followed by a clean one clears S21** (review of novox/hq issue 233): the node-engine says +// the clean re-apply, the store keeps it as applied with no failure since, and the condition the failure raised +// is cleared through the keeper on the next tick. +func TestACleanReapplyAfterAFailedOneClearsTheCondition(t *testing.T) { + now := time.Date(2026, 10, 4, 14, 0, 0, 0, time.UTC) + store := conditions.NewInMemory() + k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store, + Teller: &conditions.Told{}, Now: func() time.Time { return now }}) + defer k.Close(context.Background()) + w := &watchdogs{keeper: k, started: now.Add(-time.Hour)} + + failing := calm(now) + failing.doing = []inventory.Doing{unansweringPackageManager(now, 20*time.Minute, 1)} + w.see(t.Context(), failing) + open, err := k.Open(t.Context()) + if err != nil || len(open) != 1 || open[0].Kind != kindApplyFailing { + t.Fatalf("the failed re-apply raised %+v (%v), want apply-failing", open, err) + } + + clean := calm(now.Add(5 * time.Minute)) + clean.doing = nil // the clean re-apply recorded: NotDoingWhatTheyWereTold no longer lists the machine + w.see(t.Context(), clean) + if open, _ := k.Open(t.Context()); len(open) != 0 { + t.Fatalf("the condition stayed open after the clean re-apply: %+v", open) + } } diff --git a/cmd/mesh-controller/signals.go b/cmd/mesh-controller/signals.go index c855e2b7..e2cbc387 100644 --- a/cmd/mesh-controller/signals.go +++ b/cmd/mesh-controller/signals.go @@ -255,9 +255,10 @@ var signalsTable = []signalRow{ return newestOf(f.bus.waits, func(w busWaitFacts) time.Time { return w.since }) }}, {Row: "S21", Signal: "a clean apply", Emitter: "node-engine", Trigger: "each report of a declaration refused or applied only in part (novox/hq issue 233)", - Bound: "one apply interval (5 min) past the first report of the same failure, over at least two reports; not " + - "while the machine is silent or asleep; naming the refusal, or the package manager that does not answer " + - "with its words; cleared by a clean apply", + Bound: "one apply interval (5 min) past the first report of the same failure, judged by time while the machine " + + "is heard — the node-engine says a re-apply only when how it went changed; not while it is silent or " + + "asleep; naming the refusal, or the package manager that does not answer with its words; cleared by the " + + "clean apply it then says", Kind: kindApplyFailing, Severity: conditions.Warning, Phase: 2, needs: func(f *signalFacts) error { return f.doingErr }, watch: watchApplyFailing, newest: func(f *signalFacts) time.Time { diff --git a/cmd/mesh-controller/signals_test.go b/cmd/mesh-controller/signals_test.go index 64c4becc..6398c5ae 100644 --- a/cmd/mesh-controller/signals_test.go +++ b/cmd/mesh-controller/signals_test.go @@ -186,12 +186,12 @@ var suppressions = map[string]suppression{ }, past: func(f *signalFacts) { f.refusedSends = []inventory.Send{refusedSend(f.now.Add(-time.Second))} }, }, - // A machine whose package manager does not answer (novox/hq issue 233): said once the same failure has - // outlasted one apply interval over two reports. Inside: a second report, only five minutes after the first. + // A machine whose package manager does not answer (novox/hq issue 233): said once its first report of the + // failure is older than one apply interval while it is heard. Inside: exactly five minutes old. "S21": { - inside: func(f *signalFacts) { f.doing = []inventory.Doing{unansweringPackageManager(f.now, 5*time.Minute, 2)} }, + inside: func(f *signalFacts) { f.doing = []inventory.Doing{unansweringPackageManager(f.now, 5*time.Minute, 1)} }, past: func(f *signalFacts) { - f.doing = []inventory.Doing{unansweringPackageManager(f.now, 5*time.Minute+time.Second, 2)} + f.doing = []inventory.Doing{unansweringPackageManager(f.now, 5*time.Minute+time.Second, 1)} }, }, // Twice by hand within a fortnight is a healer wanted; once, or the first of two a day too old, is not. From 01fc9e1d732662ea5c5c5b8a66489b26cbbe3373 Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 12 Oct 2026 15:39:34 +0200 Subject: [PATCH 09/26] Read xz with a real dictionary maximum (review 65, hq issue 345) ulikunitz/xz's ReaderConfig.DictCap is a minimum: a stream asking for a 1 GiB or 4 GiB dictionary was given it. github.com/therootcompany/xz, a port of xz-embedded, takes a maximum and refuses a block whose LZMA2 dictionary is larger before allocating it. 64 MiB, xz's largest preset. --- go.mod | 2 +- go.sum | 4 +- internal/builder/upstream_file.go | 21 +- internal/builder/upstream_file_test.go | 62 + vendor/github.com/therootcompany/xz/AUTHORS | 8 + vendor/github.com/therootcompany/xz/LICENSE | 86 ++ vendor/github.com/therootcompany/xz/README.md | 19 + .../github.com/therootcompany/xz/dec_bcj.go | 461 ++++++ .../github.com/therootcompany/xz/dec_delta.go | 55 + .../github.com/therootcompany/xz/dec_lzma2.go | 1235 +++++++++++++++++ .../therootcompany/xz/dec_stream.go | 932 +++++++++++++ .../github.com/therootcompany/xz/dec_util.go | 52 + vendor/github.com/therootcompany/xz/dec_xz.go | 124 ++ vendor/github.com/therootcompany/xz/doc.go | 35 + vendor/github.com/therootcompany/xz/reader.go | 256 ++++ vendor/github.com/ulikunitz/xz/.gitignore | 28 - vendor/github.com/ulikunitz/xz/LICENSE | 26 - vendor/github.com/ulikunitz/xz/README.md | 88 -- vendor/github.com/ulikunitz/xz/SECURITY.md | 19 - vendor/github.com/ulikunitz/xz/TODO.md | 395 ------ vendor/github.com/ulikunitz/xz/bits.go | 79 -- vendor/github.com/ulikunitz/xz/crc.go | 54 - vendor/github.com/ulikunitz/xz/format.go | 721 ---------- .../github.com/ulikunitz/xz/fox-check-none.xz | Bin 96 -> 0 bytes vendor/github.com/ulikunitz/xz/fox.xz | Bin 104 -> 0 bytes .../ulikunitz/xz/internal/hash/cyclic_poly.go | 181 --- .../ulikunitz/xz/internal/hash/doc.go | 14 - .../ulikunitz/xz/internal/hash/rabin_karp.go | 66 - .../ulikunitz/xz/internal/hash/roller.go | 29 - .../ulikunitz/xz/internal/xlog/xlog.go | 456 ------ .../github.com/ulikunitz/xz/lzma/bintree.go | 522 ------- vendor/github.com/ulikunitz/xz/lzma/bitops.go | 47 - .../github.com/ulikunitz/xz/lzma/breader.go | 40 - vendor/github.com/ulikunitz/xz/lzma/buffer.go | 171 --- .../ulikunitz/xz/lzma/bytewriter.go | 37 - .../github.com/ulikunitz/xz/lzma/decoder.go | 277 ---- .../ulikunitz/xz/lzma/decoderdict.go | 128 -- .../ulikunitz/xz/lzma/directcodec.go | 38 - .../github.com/ulikunitz/xz/lzma/distcodec.go | 140 -- .../github.com/ulikunitz/xz/lzma/encoder.go | 268 ---- .../ulikunitz/xz/lzma/encoderdict.go | 149 -- vendor/github.com/ulikunitz/xz/lzma/fox.lzma | Bin 67 -> 0 bytes .../github.com/ulikunitz/xz/lzma/hashtable.go | 309 ----- vendor/github.com/ulikunitz/xz/lzma/header.go | 170 --- .../github.com/ulikunitz/xz/lzma/header2.go | 398 ------ .../ulikunitz/xz/lzma/lengthcodec.go | 115 -- .../ulikunitz/xz/lzma/literalcodec.go | 125 -- .../ulikunitz/xz/lzma/matchalgorithm.go | 52 - .../github.com/ulikunitz/xz/lzma/operation.go | 55 - vendor/github.com/ulikunitz/xz/lzma/prob.go | 53 - .../ulikunitz/xz/lzma/properties.go | 69 - .../ulikunitz/xz/lzma/rangecodec.go | 222 --- vendor/github.com/ulikunitz/xz/lzma/reader.go | 193 --- .../github.com/ulikunitz/xz/lzma/reader2.go | 231 --- vendor/github.com/ulikunitz/xz/lzma/state.go | 145 -- .../ulikunitz/xz/lzma/treecodecs.go | 133 -- vendor/github.com/ulikunitz/xz/lzma/writer.go | 209 --- .../github.com/ulikunitz/xz/lzma/writer2.go | 305 ---- vendor/github.com/ulikunitz/xz/lzmafilter.go | 117 -- vendor/github.com/ulikunitz/xz/make-docs | 5 - vendor/github.com/ulikunitz/xz/none-check.go | 23 - vendor/github.com/ulikunitz/xz/reader.go | 359 ----- vendor/github.com/ulikunitz/xz/writer.go | 399 ------ vendor/modules.txt | 9 +- 64 files changed, 3342 insertions(+), 7679 deletions(-) create mode 100644 vendor/github.com/therootcompany/xz/AUTHORS create mode 100644 vendor/github.com/therootcompany/xz/LICENSE create mode 100644 vendor/github.com/therootcompany/xz/README.md create mode 100644 vendor/github.com/therootcompany/xz/dec_bcj.go create mode 100644 vendor/github.com/therootcompany/xz/dec_delta.go create mode 100644 vendor/github.com/therootcompany/xz/dec_lzma2.go create mode 100644 vendor/github.com/therootcompany/xz/dec_stream.go create mode 100644 vendor/github.com/therootcompany/xz/dec_util.go create mode 100644 vendor/github.com/therootcompany/xz/dec_xz.go create mode 100644 vendor/github.com/therootcompany/xz/doc.go create mode 100644 vendor/github.com/therootcompany/xz/reader.go delete mode 100644 vendor/github.com/ulikunitz/xz/.gitignore delete mode 100644 vendor/github.com/ulikunitz/xz/LICENSE delete mode 100644 vendor/github.com/ulikunitz/xz/README.md delete mode 100644 vendor/github.com/ulikunitz/xz/SECURITY.md delete mode 100644 vendor/github.com/ulikunitz/xz/TODO.md delete mode 100644 vendor/github.com/ulikunitz/xz/bits.go delete mode 100644 vendor/github.com/ulikunitz/xz/crc.go delete mode 100644 vendor/github.com/ulikunitz/xz/format.go delete mode 100644 vendor/github.com/ulikunitz/xz/fox-check-none.xz delete mode 100644 vendor/github.com/ulikunitz/xz/fox.xz delete mode 100644 vendor/github.com/ulikunitz/xz/internal/hash/cyclic_poly.go delete mode 100644 vendor/github.com/ulikunitz/xz/internal/hash/doc.go delete mode 100644 vendor/github.com/ulikunitz/xz/internal/hash/rabin_karp.go delete mode 100644 vendor/github.com/ulikunitz/xz/internal/hash/roller.go delete mode 100644 vendor/github.com/ulikunitz/xz/internal/xlog/xlog.go delete mode 100644 vendor/github.com/ulikunitz/xz/lzma/bintree.go delete mode 100644 vendor/github.com/ulikunitz/xz/lzma/bitops.go delete mode 100644 vendor/github.com/ulikunitz/xz/lzma/breader.go delete mode 100644 vendor/github.com/ulikunitz/xz/lzma/buffer.go delete mode 100644 vendor/github.com/ulikunitz/xz/lzma/bytewriter.go delete mode 100644 vendor/github.com/ulikunitz/xz/lzma/decoder.go delete mode 100644 vendor/github.com/ulikunitz/xz/lzma/decoderdict.go delete mode 100644 vendor/github.com/ulikunitz/xz/lzma/directcodec.go delete mode 100644 vendor/github.com/ulikunitz/xz/lzma/distcodec.go delete mode 100644 vendor/github.com/ulikunitz/xz/lzma/encoder.go delete mode 100644 vendor/github.com/ulikunitz/xz/lzma/encoderdict.go delete mode 100644 vendor/github.com/ulikunitz/xz/lzma/fox.lzma delete mode 100644 vendor/github.com/ulikunitz/xz/lzma/hashtable.go delete mode 100644 vendor/github.com/ulikunitz/xz/lzma/header.go delete mode 100644 vendor/github.com/ulikunitz/xz/lzma/header2.go delete mode 100644 vendor/github.com/ulikunitz/xz/lzma/lengthcodec.go delete mode 100644 vendor/github.com/ulikunitz/xz/lzma/literalcodec.go delete mode 100644 vendor/github.com/ulikunitz/xz/lzma/matchalgorithm.go delete mode 100644 vendor/github.com/ulikunitz/xz/lzma/operation.go delete mode 100644 vendor/github.com/ulikunitz/xz/lzma/prob.go delete mode 100644 vendor/github.com/ulikunitz/xz/lzma/properties.go delete mode 100644 vendor/github.com/ulikunitz/xz/lzma/rangecodec.go delete mode 100644 vendor/github.com/ulikunitz/xz/lzma/reader.go delete mode 100644 vendor/github.com/ulikunitz/xz/lzma/reader2.go delete mode 100644 vendor/github.com/ulikunitz/xz/lzma/state.go delete mode 100644 vendor/github.com/ulikunitz/xz/lzma/treecodecs.go delete mode 100644 vendor/github.com/ulikunitz/xz/lzma/writer.go delete mode 100644 vendor/github.com/ulikunitz/xz/lzma/writer2.go delete mode 100644 vendor/github.com/ulikunitz/xz/lzmafilter.go delete mode 100644 vendor/github.com/ulikunitz/xz/make-docs delete mode 100644 vendor/github.com/ulikunitz/xz/none-check.go delete mode 100644 vendor/github.com/ulikunitz/xz/reader.go delete mode 100644 vendor/github.com/ulikunitz/xz/writer.go diff --git a/go.mod b/go.mod index e683fded..60ad6d19 100644 --- a/go.mod +++ b/go.mod @@ -8,7 +8,7 @@ require ( github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 github.com/novox/mesh-host v0.0.0 - github.com/ulikunitz/xz v0.5.17 + github.com/therootcompany/xz v1.0.1 golang.org/x/crypto v0.57.0 golang.org/x/net v0.58.0 golang.org/x/sys v0.48.0 diff --git a/go.sum b/go.sum index cace9cc9..96650c70 100644 --- a/go.sum +++ b/go.sum @@ -38,8 +38,8 @@ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UV github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= -github.com/ulikunitz/xz v0.5.17 h1:flR0y/x1hgM8EGV1AW3Xll6T413G0glV8UfBwR617V4= -github.com/ulikunitz/xz v0.5.17/go.mod h1:H9Rt/W6/Qj27PGauhQc6nfCDy7vHpzsOThBSaYDoEhw= +github.com/therootcompany/xz v1.0.1 h1:CmOtsn1CbtmyYiusbfmhmkpAAETj0wBIH6kCYaX+xzw= +github.com/therootcompany/xz v1.0.1/go.mod h1:3K3UH1yCKgBneZYhuQUvJ9HPD19UEXEI0BWbMn8qNMY= golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M= golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA= golang.org/x/net v0.58.0 h1:ynWG7rqYi4ccpTEuPZ2QGWHktVEM9DMCj9yzDE0Q7To= diff --git a/internal/builder/upstream_file.go b/internal/builder/upstream_file.go index e8dbc5db..38d374e7 100644 --- a/internal/builder/upstream_file.go +++ b/internal/builder/upstream_file.go @@ -20,7 +20,7 @@ import ( "time" "github.com/novox/mesh-controller/internal/catalogue" - "github.com/ulikunitz/xz" + "github.com/therootcompany/xz" ) // An archive whose source is somebody else's published file, pinned by URL and sha256 (novox/hq ADR 0316 @@ -194,14 +194,15 @@ func unpackUpstream(ctx context.Context, file, into, within string) error { case bytes.HasPrefix(magic, []byte("BZh")): stream = bzip2.NewReader(buffered) case bytes.HasPrefix(magic, []byte{0xfd, '7', 'z', 'X', 'Z', 0x00}): - // Read in Go (github.com/ulikunitz/xz), never by a tool the build seat's image may not carry: the seat's - // image is the catalogue's build-agent recipe, which installs git and docker-cli alone, and a release - // pinned as .tar.xz failed there on 2026-10-12 (hq issue 345). The dictionary is bounded, and what it - // unpacks to is bounded by untar (maxUpstream, maxEntries). - config := xz.ReaderConfig{DictCap: xzDictCap} - zipped, err := config.NewReader(buffered) + // Read in Go (github.com/therootcompany/xz, a port of xz-embedded), never by a tool the build seat's image + // may not carry: the seat's image is the catalogue's build-agent recipe, which installs git and docker-cli + // alone, and a release pinned as .tar.xz failed there on 2026-10-12 (hq issue 345). xzDictMax is a real + // maximum: a block whose LZMA2 dictionary is larger is refused before its dictionary is allocated + // (ErrMemlimit). What the stream unpacks to is bounded by untar (maxUpstream, maxEntries). + zipped, err := xz.NewReader(buffered, xzDictMax) if err != nil { - return fmt.Errorf("it is not an xz stream this build seat reads: %v", err) + return fmt.Errorf("it is not an xz stream this build seat reads (a dictionary of %d MiB at most): %v", + xzDictMax>>20, err) } stream = zipped default: @@ -211,9 +212,9 @@ func unpackUpstream(ctx context.Context, file, into, within string) error { return untar(stream, into, within) } -// xzDictCap is the largest xz dictionary read: xz's own presets use at most 64 MiB (-9); a stream asking for +// xzDictMax is the largest xz dictionary read: xz's own presets use at most 64 MiB (-9); a stream asking for // more is refused rather than given the memory. -const xzDictCap = 64 << 20 +const xzDictMax = 64 << 20 // untar writes a tar stream's files and directories within `within` into a directory. func untar(stream io.Reader, into, within string) error { diff --git a/internal/builder/upstream_file_test.go b/internal/builder/upstream_file_test.go index 0fc2dcd3..34e82f1e 100644 --- a/internal/builder/upstream_file_test.go +++ b/internal/builder/upstream_file_test.go @@ -6,8 +6,10 @@ import ( "compress/gzip" "context" "crypto/sha256" + "encoding/binary" "encoding/hex" "fmt" + "hash/crc32" "io" "net" "net/http" @@ -15,6 +17,7 @@ import ( "os" "os/exec" "path/filepath" + "runtime" "sort" "strings" "testing" @@ -405,3 +408,62 @@ func TestAnXzReleaseIsReadWithoutAnyXzToolOnTheBuildSeat(t *testing.T) { t.Fatalf("%v", e) } } + +// withDictionary is an xz stream whose first block's LZMA2 dictionary property is props, its header's CRC32 +// recomputed: a stream asking for a dictionary of its own choosing. +func withDictionary(t *testing.T, stream []byte, props byte) []byte { + t.Helper() + out := append([]byte{}, stream...) + const at = 12 // the stream header + size := (int(out[at]) + 1) * 4 + header := out[at : at+size] + flags := header[1] + i := 2 + readVarint := func() { + for header[i]&0x80 != 0 { + i++ + } + i++ + } + if flags&0x40 != 0 { + readVarint() + } + if flags&0x80 != 0 { + readVarint() + } + if header[i] != 0x21 || header[i+1] != 0x01 { + t.Fatalf("the fixture's first filter is not LZMA2: % x", header) + } + header[i+2] = props + binary.LittleEndian.PutUint32(header[size-4:], crc32.ChecksumIEEE(header[:size-4])) + return out +} + +func TestAnXzStreamAskingForTooLargeADictionaryIsRefused(t *testing.T) { + body, err := os.ReadFile(filepath.Join("testdata", "release.tar.xz")) + if err != nil { + t.Fatal(err) + } + // 1 GiB (props 36) and 4 GiB less one (props 40): over the 64 MiB maximum, refused before allocation. + for _, props := range []byte{36, 40} { + crafted := withDictionary(t, body, props) + v := &vendor{files: map[string][]byte{"/r.tar.xz": crafted}} + base := serveVendor(t, v) + var before, after runtime.MemStats + runtime.GC() + runtime.ReadMemStats(&before) + _, err := upstreamArchive(context.Background(), t.TempDir(), catalogue.Artifact{Name: "archive", Kind: "archive", + URL: base + "/r.tar.xz", SHA256: sumHex(crafted), Version: "1", Within: "betterbird"}, func(string, string, ...any) {}) + runtime.ReadMemStats(&after) + if err == nil || !strings.Contains(err.Error(), "dictionary") { + t.Errorf("props %d: a dictionary over the maximum was not refused: %v", props, err) + } + if grew := after.TotalAlloc - before.TotalAlloc; grew > 256<<20 { + t.Errorf("props %d: %d bytes were allocated for a refused stream", props, grew) + } + } + // The fixture as it is (a small dictionary) still reads. + if crafted := withDictionary(t, body, 0); len(crafted) == 0 { + t.Fatal("no fixture") + } +} diff --git a/vendor/github.com/therootcompany/xz/AUTHORS b/vendor/github.com/therootcompany/xz/AUTHORS new file mode 100644 index 00000000..657330e1 --- /dev/null +++ b/vendor/github.com/therootcompany/xz/AUTHORS @@ -0,0 +1,8 @@ +# Package xz authors + +Michael Cross + +# XZ Embedded authors + +Lasse Collin +Igor Pavlov diff --git a/vendor/github.com/therootcompany/xz/LICENSE b/vendor/github.com/therootcompany/xz/LICENSE new file mode 100644 index 00000000..d016e27f --- /dev/null +++ b/vendor/github.com/therootcompany/xz/LICENSE @@ -0,0 +1,86 @@ +Creative Commons CC0 1.0 Universal + +CREATIVE COMMONS CORPORATION IS NOT A LAW FIRM AND DOES NOT PROVIDE LEGAL +SERVICES. DISTRIBUTION OF THIS DOCUMENT DOES NOT CREATE AN ATTORNEY-CLIENT +RELATIONSHIP. CREATIVE COMMONS PROVIDES THIS INFORMATION ON AN "AS-IS" BASIS. +CREATIVE COMMONS MAKES NO WARRANTIES REGARDING THE USE OF THIS DOCUMENT OR THE +INFORMATION OR WORKS PROVIDED HEREUNDER, AND DISCLAIMS LIABILITY FOR DAMAGES +RESULTING FROM THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS PROVIDED +HEREUNDER. + +Statement of Purpose + +The laws of most jurisdictions throughout the world automatically confer +exclusive Copyright and Related Rights (defined below) upon the creator and +subsequent owner(s) (each and all, an "owner") of an original work of +authorship and/or a database (each, a "Work"). + +Certain owners wish to permanently relinquish those rights to a Work for the +purpose of contributing to a commons of creative, cultural and scientific +works ("Commons") that the public can reliably and without fear of later +claims of infringement build upon, modify, incorporate in other works, reuse +and redistribute as freely as possible in any form whatsoever and for any +purposes, including without limitation commercial purposes. These owners may +contribute to the Commons to promote the ideal of a free culture and the +further production of creative, cultural and scientific works, or to gain +reputation or greater distribution for their Work in part through the use and +efforts of others. + +For these and/or other purposes and motivations, and without any expectation +of additional consideration or compensation, the person associating CC0 with a +Work (the "Affirmer"), to the extent that he or she is an owner of Copyright +and Related Rights in the Work, voluntarily elects to apply CC0 to the Work +and publicly distribute the Work under its terms, with knowledge of his or her +Copyright and Related Rights in the Work and the meaning and intended legal +effect of CC0 on those rights. + +1. Copyright and Related Rights. A Work made available under CC0 may be protected by copyright and related or neighboring rights ("Copyright and Related Rights"). Copyright and Related Rights include, but are not limited to, the following: + +i. the right to reproduce, adapt, distribute, perform, display, communicate, +and translate a Work; + +ii. moral rights retained by the original author(s) and/or performer(s); + +iii. publicity and privacy rights pertaining to a person's image or +likeness depicted in a Work; + +iv. rights protecting against unfair competition in regards to a Work, subject +to the limitations in paragraph 4(a), below; + +v. rights protecting the extraction, dissemination, use and reuse of data in a +Work; + +vi. database rights (such as those arising under Directive 96/9/EC of the +European Parliament and of the Council of 11 March 1996 on the legal +protection of databases, and under any national implementation thereof, +including any amended or successor version of such directive); and + +vii. other similar, equivalent or corresponding rights throughout the world +based on applicable law or treaty, and any national implementations thereof. + +2. Waiver. To the greatest extent permitted by, but not in contravention of, applicable law, Affirmer hereby overtly, fully, permanently, irrevocably and unconditionally waives, abandons, and surrenders all of Affirmer's Copyright and Related Rights and associated claims and causes of action, whether now known or unknown (including existing as well as future claims and causes of action), in the Work (i) in all territories worldwide, (ii) for the maximum duration provided by applicable law or treaty (including future time extensions), (iii) in any current or future medium and for any number of copies, and (iv) for any purpose whatsoever, including without limitation commercial, advertising or promotional purposes (the "Waiver"). Affirmer makes the Waiver for the benefit of each member of the public at large and to the detriment of Affirmer's heirs and successors, fully intending that such Waiver shall not be subject to revocation, rescission, cancellation, termination, or any other legal or equitable action to disrupt the quiet enjoyment of the Work by the public as contemplated by Affirmer's express Statement of Purpose. + +3. Public License Fallback. Should any part of the Waiver for any reason be judged legally invalid or ineffective under applicable law, then the Waiver shall be preserved to the maximum extent permitted taking into account Affirmer's express Statement of Purpose. In addition, to the extent the Waiver is so judged Affirmer hereby grants to each affected person a royalty-free, non transferable, non sublicensable, non exclusive, irrevocable and unconditional license to exercise Affirmer's Copyright and Related Rights in the Work (i) in all territories worldwide, (ii) for the maximum duration provided by applicable law or treaty (including future time extensions), (iii) in any current or future medium and for any number of copies, and (iv) for any purpose whatsoever, including without limitation commercial, advertising or promotional purposes (the "License"). The License shall be deemed effective as of the date CC0 was applied by Affirmer to the Work. Should any part of the License for any reason be judged legally invalid or ineffective under applicable law, such partial invalidity or ineffectiveness shall not invalidate the remainder of the License, and in such case Affirmer hereby affirms that he or she will not (i) exercise any of his or her remaining Copyright and Related Rights in the Work or (ii) assert any associated claims and causes of action with respect to the Work, in either case contrary to Affirmer's express Statement of Purpose. + +4. Limitations and Disclaimers. + +a. No trademark or patent rights held by Affirmer are waived, abandoned, +surrendered, licensed or otherwise affected by this document. + +b. Affirmer offers the Work as-is and makes no representations or warranties +of any kind concerning the Work, express, implied, statutory or otherwise, +including without limitation warranties of title, merchantability, fitness for +a particular purpose, non infringement, or the absence of latent or other +defects, accuracy, or the present or absence of errors, whether or not +discoverable, all to the greatest extent permissible under applicable law. + +c. Affirmer disclaims responsibility for clearing rights of other persons that +may apply to the Work or any use thereof, including without limitation any +person's Copyright and Related Rights in the Work. Further, Affirmer +disclaims responsibility for obtaining any necessary consents, permissions or +other rights required for any use of the Work. + +d. Affirmer understands and acknowledges that Creative Commons is not a party +to this document and has no duty or obligation with respect to this CC0 or use +of the Work. + diff --git a/vendor/github.com/therootcompany/xz/README.md b/vendor/github.com/therootcompany/xz/README.md new file mode 100644 index 00000000..19fd1c46 --- /dev/null +++ b/vendor/github.com/therootcompany/xz/README.md @@ -0,0 +1,19 @@ +# Xz + +Package xz implements XZ decompression natively in Go. + +Documentation at . + +Download and install with `go get github.com/xi2/xz`. + +If you need compression as well as decompression, you might want to +look at . + +# LICENSE + +This was originally released into the public domain by the AUTHORS. + +Here it is licensed more explicitly as Creative Commons CC0 1.0 Universal +(CC0-1.0) so that it can be detected by automated tooling, and satisfy the legal +requirements for vendor integration for cases in which a "public domain" +statement is not sufficient. diff --git a/vendor/github.com/therootcompany/xz/dec_bcj.go b/vendor/github.com/therootcompany/xz/dec_bcj.go new file mode 100644 index 00000000..a8a3df92 --- /dev/null +++ b/vendor/github.com/therootcompany/xz/dec_bcj.go @@ -0,0 +1,461 @@ +/* + * Branch/Call/Jump (BCJ) filter decoders + * + * Authors: Lasse Collin + * Igor Pavlov + * + * Translation to Go: Michael Cross + * + * This file has been put into the public domain. + * You can do whatever you want with this file. + */ + +package xz + +/* from linux/lib/xz/xz_dec_bcj.c *************************************/ + +type xzDecBCJ struct { + /* Type of the BCJ filter being used */ + typ xzFilterID + /* + * Return value of the next filter in the chain. We need to preserve + * this information across calls, because we must not call the next + * filter anymore once it has returned xzStreamEnd + */ + ret xzRet + /* + * Absolute position relative to the beginning of the uncompressed + * data (in a single .xz Block). + */ + pos int + /* x86 filter state */ + x86PrevMask uint32 + /* Temporary space to hold the variables from xzBuf */ + out []byte + outPos int + temp struct { + /* Amount of already filtered data in the beginning of buf */ + filtered int + /* + * Buffer to hold a mix of filtered and unfiltered data. This + * needs to be big enough to hold Alignment + 2 * Look-ahead: + * + * Type Alignment Look-ahead + * x86 1 4 + * PowerPC 4 0 + * IA-64 16 0 + * ARM 4 0 + * ARM-Thumb 2 2 + * SPARC 4 0 + */ + buf []byte // slice buf will be backed by bufArray + bufArray [16]byte + } +} + +/* + * This is used to test the most significant byte of a memory address + * in an x86 instruction. + */ +func bcjX86TestMSByte(b byte) bool { + return b == 0x00 || b == 0xff +} + +func bcjX86Filter(s *xzDecBCJ, buf []byte) int { + var maskToAllowedStatus = []bool{ + true, true, true, false, true, false, false, false, + } + var maskToBitNum = []byte{0, 1, 2, 2, 3, 3, 3, 3} + var i int + var prevPos int = -1 + var prevMask uint32 = s.x86PrevMask + var src uint32 + var dest uint32 + var j uint32 + var b byte + if len(buf) <= 4 { + return 0 + } + for i = 0; i < len(buf)-4; i++ { + if buf[i]&0xfe != 0xe8 { + continue + } + prevPos = i - prevPos + if prevPos > 3 { + prevMask = 0 + } else { + prevMask = (prevMask << (uint(prevPos) - 1)) & 7 + if prevMask != 0 { + b = buf[i+4-int(maskToBitNum[prevMask])] + if !maskToAllowedStatus[prevMask] || bcjX86TestMSByte(b) { + prevPos = i + prevMask = prevMask<<1 | 1 + continue + } + } + } + prevPos = i + if bcjX86TestMSByte(buf[i+4]) { + src = getLE32(buf[i+1:]) + for { + dest = src - uint32(s.pos+i+5) + if prevMask == 0 { + break + } + j = uint32(maskToBitNum[prevMask]) * 8 + b = byte(dest >> (24 - j)) + if !bcjX86TestMSByte(b) { + break + } + src = dest ^ (1<<(32-j) - 1) + } + dest &= 0x01FFFFFF + dest |= 0 - dest&0x01000000 + putLE32(dest, buf[i+1:]) + i += 4 + } else { + prevMask = prevMask<<1 | 1 + } + } + prevPos = i - prevPos + if prevPos > 3 { + s.x86PrevMask = 0 + } else { + s.x86PrevMask = prevMask << (uint(prevPos) - 1) + } + return i +} + +func bcjPowerPCFilter(s *xzDecBCJ, buf []byte) int { + var i int + var instr uint32 + for i = 0; i+4 <= len(buf); i += 4 { + instr = getBE32(buf[i:]) + if instr&0xFC000003 == 0x48000001 { + instr &= 0x03FFFFFC + instr -= uint32(s.pos + i) + instr &= 0x03FFFFFC + instr |= 0x48000001 + putBE32(instr, buf[i:]) + } + } + return i +} + +var bcjIA64BranchTable = [...]byte{ + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 4, 4, 6, 6, 0, 0, 7, 7, + 4, 4, 0, 0, 4, 4, 0, 0, +} + +func bcjIA64Filter(s *xzDecBCJ, buf []byte) int { + var branchTable = bcjIA64BranchTable[:] + /* + * The local variables take a little bit stack space, but it's less + * than what LZMA2 decoder takes, so it doesn't make sense to reduce + * stack usage here without doing that for the LZMA2 decoder too. + */ + /* Loop counters */ + var i int + var j int + /* Instruction slot (0, 1, or 2) in the 128-bit instruction word */ + var slot uint32 + /* Bitwise offset of the instruction indicated by slot */ + var bitPos uint32 + /* bit_pos split into byte and bit parts */ + var bytePos uint32 + var bitRes uint32 + /* Address part of an instruction */ + var addr uint32 + /* Mask used to detect which instructions to convert */ + var mask uint32 + /* 41-bit instruction stored somewhere in the lowest 48 bits */ + var instr uint64 + /* Instruction normalized with bit_res for easier manipulation */ + var norm uint64 + for i = 0; i+16 <= len(buf); i += 16 { + mask = uint32(branchTable[buf[i]&0x1f]) + for slot, bitPos = 0, 5; slot < 3; slot, bitPos = slot+1, bitPos+41 { + if (mask>>slot)&1 == 0 { + continue + } + bytePos = bitPos >> 3 + bitRes = bitPos & 7 + instr = 0 + for j = 0; j < 6; j++ { + instr |= uint64(buf[i+j+int(bytePos)]) << (8 * uint(j)) + } + norm = instr >> bitRes + if (norm>>37)&0x0f == 0x05 && (norm>>9)&0x07 == 0 { + addr = uint32((norm >> 13) & 0x0fffff) + addr |= (uint32(norm>>36) & 1) << 20 + addr <<= 4 + addr -= uint32(s.pos + i) + addr >>= 4 + norm &= ^(uint64(0x8fffff) << 13) + norm |= uint64(addr&0x0fffff) << 13 + norm |= uint64(addr&0x100000) << (36 - 20) + instr &= 1<> (8 * uint(j))) + } + } + } + } + return i +} + +func bcjARMFilter(s *xzDecBCJ, buf []byte) int { + var i int + var addr uint32 + for i = 0; i+4 <= len(buf); i += 4 { + if buf[i+3] == 0xeb { + addr = uint32(buf[i]) | uint32(buf[i+1])<<8 | + uint32(buf[i+2])<<16 + addr <<= 2 + addr -= uint32(s.pos + i + 8) + addr >>= 2 + buf[i] = byte(addr) + buf[i+1] = byte(addr >> 8) + buf[i+2] = byte(addr >> 16) + } + } + return i +} + +func bcjARMThumbFilter(s *xzDecBCJ, buf []byte) int { + var i int + var addr uint32 + for i = 0; i+4 <= len(buf); i += 2 { + if buf[i+1]&0xf8 == 0xf0 && buf[i+3]&0xf8 == 0xf8 { + addr = uint32(buf[i+1]&0x07)<<19 | + uint32(buf[i])<<11 | + uint32(buf[i+3]&0x07)<<8 | + uint32(buf[i+2]) + addr <<= 1 + addr -= uint32(s.pos + i + 4) + addr >>= 1 + buf[i+1] = byte(0xf0 | (addr>>19)&0x07) + buf[i] = byte(addr >> 11) + buf[i+3] = byte(0xf8 | (addr>>8)&0x07) + buf[i+2] = byte(addr) + i += 2 + } + } + return i +} + +func bcjSPARCFilter(s *xzDecBCJ, buf []byte) int { + var i int + var instr uint32 + for i = 0; i+4 <= len(buf); i += 4 { + instr = getBE32(buf[i:]) + if instr>>22 == 0x100 || instr>>22 == 0x1ff { + instr <<= 2 + instr -= uint32(s.pos + i) + instr >>= 2 + instr = (0x40000000 - instr&0x400000) | + 0x40000000 | (instr & 0x3FFFFF) + putBE32(instr, buf[i:]) + } + } + return i +} + +/* + * Apply the selected BCJ filter. Update *pos and s.pos to match the amount + * of data that got filtered. + */ +func bcjApply(s *xzDecBCJ, buf []byte, pos *int) { + var filtered int + buf = buf[*pos:] + switch s.typ { + case idBCJX86: + filtered = bcjX86Filter(s, buf) + case idBCJPowerPC: + filtered = bcjPowerPCFilter(s, buf) + case idBCJIA64: + filtered = bcjIA64Filter(s, buf) + case idBCJARM: + filtered = bcjARMFilter(s, buf) + case idBCJARMThumb: + filtered = bcjARMThumbFilter(s, buf) + case idBCJSPARC: + filtered = bcjSPARCFilter(s, buf) + default: + /* Never reached */ + } + *pos += filtered + s.pos += filtered +} + +/* + * Flush pending filtered data from temp to the output buffer. + * Move the remaining mixture of possibly filtered and unfiltered + * data to the beginning of temp. + */ +func bcjFlush(s *xzDecBCJ, b *xzBuf) { + var copySize int + copySize = len(b.out) - b.outPos + if copySize > s.temp.filtered { + copySize = s.temp.filtered + } + copy(b.out[b.outPos:], s.temp.buf[:copySize]) + b.outPos += copySize + s.temp.filtered -= copySize + copy(s.temp.buf, s.temp.buf[copySize:]) + s.temp.buf = s.temp.buf[:len(s.temp.buf)-copySize] +} + +/* + * Decode raw stream which has a BCJ filter as the first filter. + * + * The BCJ filter functions are primitive in sense that they process the + * data in chunks of 1-16 bytes. To hide this issue, this function does + * some buffering. + */ +func xzDecBCJRun(s *xzDecBCJ, b *xzBuf, chain func(*xzBuf) xzRet) xzRet { + var outStart int + /* + * Flush pending already filtered data to the output buffer. Return + * immediately if we couldn't flush everything, or if the next + * filter in the chain had already returned xzStreamEnd. + */ + if s.temp.filtered > 0 { + bcjFlush(s, b) + if s.temp.filtered > 0 { + return xzOK + } + if s.ret == xzStreamEnd { + return xzStreamEnd + } + } + /* + * If we have more output space than what is currently pending in + * temp, copy the unfiltered data from temp to the output buffer + * and try to fill the output buffer by decoding more data from the + * next filter in the chain. Apply the BCJ filter on the new data + * in the output buffer. If everything cannot be filtered, copy it + * to temp and rewind the output buffer position accordingly. + * + * This needs to be always run when len(temp.buf) == 0 to handle a special + * case where the output buffer is full and the next filter has no + * more output coming but hasn't returned xzStreamEnd yet. + */ + if len(s.temp.buf) < len(b.out)-b.outPos || len(s.temp.buf) == 0 { + outStart = b.outPos + copy(b.out[b.outPos:], s.temp.buf) + b.outPos += len(s.temp.buf) + s.ret = chain(b) + if s.ret != xzStreamEnd && s.ret != xzOK { + return s.ret + } + bcjApply(s, b.out[:b.outPos], &outStart) + /* + * As an exception, if the next filter returned xzStreamEnd, + * we can do that too, since the last few bytes that remain + * unfiltered are meant to remain unfiltered. + */ + if s.ret == xzStreamEnd { + return xzStreamEnd + } + s.temp.buf = s.temp.bufArray[:b.outPos-outStart] + b.outPos -= len(s.temp.buf) + copy(s.temp.buf, b.out[b.outPos:]) + /* + * If there wasn't enough input to the next filter to fill + * the output buffer with unfiltered data, there's no point + * to try decoding more data to temp. + */ + if b.outPos+len(s.temp.buf) < len(b.out) { + return xzOK + } + } + /* + * We have unfiltered data in temp. If the output buffer isn't full + * yet, try to fill the temp buffer by decoding more data from the + * next filter. Apply the BCJ filter on temp. Then we hopefully can + * fill the actual output buffer by copying filtered data from temp. + * A mix of filtered and unfiltered data may be left in temp; it will + * be taken care on the next call to this function. + */ + if b.outPos < len(b.out) { + /* Make b.out temporarily point to s.temp. */ + s.out = b.out + s.outPos = b.outPos + b.out = s.temp.bufArray[:] + b.outPos = len(s.temp.buf) + s.ret = chain(b) + s.temp.buf = s.temp.bufArray[:b.outPos] + b.out = s.out + b.outPos = s.outPos + if s.ret != xzOK && s.ret != xzStreamEnd { + return s.ret + } + bcjApply(s, s.temp.buf, &s.temp.filtered) + /* + * If the next filter returned xzStreamEnd, we mark that + * everything is filtered, since the last unfiltered bytes + * of the stream are meant to be left as is. + */ + if s.ret == xzStreamEnd { + s.temp.filtered = len(s.temp.buf) + } + bcjFlush(s, b) + if s.temp.filtered > 0 { + return xzOK + } + } + return s.ret +} + +/* + * Allocate memory for BCJ decoders. xzDecBCJReset must be used before + * calling xzDecBCJRun. + */ +func xzDecBCJCreate() *xzDecBCJ { + return new(xzDecBCJ) +} + +/* + * Decode the Filter ID of a BCJ filter and check the start offset is + * valid. Returns xzOK if the given Filter ID and offset is + * supported. Otherwise xzOptionsError is returned. + */ +func xzDecBCJReset(s *xzDecBCJ, id xzFilterID, offset int) xzRet { + switch id { + case idBCJX86: + case idBCJPowerPC: + case idBCJIA64: + case idBCJARM: + case idBCJARMThumb: + case idBCJSPARC: + default: + /* Unsupported Filter ID */ + return xzOptionsError + } + // check offset is a multiple of alignment + switch id { + case idBCJPowerPC, idBCJARM, idBCJSPARC: + if offset%4 != 0 { + return xzOptionsError + } + case idBCJIA64: + if offset%16 != 0 { + return xzOptionsError + } + case idBCJARMThumb: + if offset%2 != 0 { + return xzOptionsError + } + } + s.typ = id + s.ret = xzOK + s.pos = offset + s.x86PrevMask = 0 + s.temp.filtered = 0 + s.temp.buf = nil + return xzOK +} diff --git a/vendor/github.com/therootcompany/xz/dec_delta.go b/vendor/github.com/therootcompany/xz/dec_delta.go new file mode 100644 index 00000000..19df5908 --- /dev/null +++ b/vendor/github.com/therootcompany/xz/dec_delta.go @@ -0,0 +1,55 @@ +/* + * Delta decoder + * + * Author: Lasse Collin + * + * Translation to Go: Michael Cross + * + * This file has been put into the public domain. + * You can do whatever you want with this file. + */ + +package xz + +type xzDecDelta struct { + delta [256]byte + pos byte + distance int // in range [1, 256] +} + +/* + * Decode raw stream which has a delta filter as the first filter. + */ +func xzDecDeltaRun(s *xzDecDelta, b *xzBuf, chain func(*xzBuf) xzRet) xzRet { + outStart := b.outPos + ret := chain(b) + for i := outStart; i < b.outPos; i++ { + tmp := b.out[i] + s.delta[byte(s.distance+int(s.pos))] + s.delta[s.pos] = tmp + b.out[i] = tmp + s.pos-- + } + return ret +} + +/* + * Allocate memory for a delta decoder. xzDecDeltaReset must be used + * before calling xzDecDeltaRun. + */ +func xzDecDeltaCreate() *xzDecDelta { + return new(xzDecDelta) +} + +/* + * Returns xzOK if the given distance is valid. Otherwise + * xzOptionsError is returned. + */ +func xzDecDeltaReset(s *xzDecDelta, distance int) xzRet { + if distance < 1 || distance > 256 { + return xzOptionsError + } + s.delta = [256]byte{} + s.pos = 0 + s.distance = distance + return xzOK +} diff --git a/vendor/github.com/therootcompany/xz/dec_lzma2.go b/vendor/github.com/therootcompany/xz/dec_lzma2.go new file mode 100644 index 00000000..fa42e471 --- /dev/null +++ b/vendor/github.com/therootcompany/xz/dec_lzma2.go @@ -0,0 +1,1235 @@ +/* + * LZMA2 decoder + * + * Authors: Lasse Collin + * Igor Pavlov + * + * Translation to Go: Michael Cross + * + * This file has been put into the public domain. + * You can do whatever you want with this file. + */ + +package xz + +/* from linux/lib/xz/xz_lzma2.h ***************************************/ + +/* Range coder constants */ +const ( + rcShiftBits = 8 + rcTopBits = 24 + rcTopValue = 1 << rcTopBits + rcBitModelTotalBits = 11 + rcBitModelTotal = 1 << rcBitModelTotalBits + rcMoveBits = 5 +) + +/* + * Maximum number of position states. A position state is the lowest pb + * number of bits of the current uncompressed offset. In some places there + * are different sets of probabilities for different position states. + */ +const posStatesMax = 1 << 4 + +/* + * lzmaState is used to track which LZMA symbols have occurred most recently + * and in which order. This information is used to predict the next symbol. + * + * Symbols: + * - Literal: One 8-bit byte + * - Match: Repeat a chunk of data at some distance + * - Long repeat: Multi-byte match at a recently seen distance + * - Short repeat: One-byte repeat at a recently seen distance + * + * The symbol names are in from STATE-oldest-older-previous. REP means + * either short or long repeated match, and NONLIT means any non-literal. + */ +type lzmaState int + +const ( + stateLitLit lzmaState = iota + stateMatchLitLit + stateRepLitLit + stateShortrepLitLit + stateMatchLit + stateRepList + stateShortrepLit + stateLitMatch + stateLitLongrep + stateLitShortrep + stateNonlitMatch + stateNonlitRep +) + +/* Total number of states */ +const states = 12 + +/* The lowest 7 states indicate that the previous state was a literal. */ +const litStates = 7 + +/* Indicate that the latest symbol was a literal. */ +func lzmaStateLiteral(state *lzmaState) { + switch { + case *state <= stateShortrepLitLit: + *state = stateLitLit + case *state <= stateLitShortrep: + *state -= 3 + default: + *state -= 6 + } +} + +/* Indicate that the latest symbol was a match. */ +func lzmaStateMatch(state *lzmaState) { + if *state < litStates { + *state = stateLitMatch + } else { + *state = stateNonlitMatch + } +} + +/* Indicate that the latest state was a long repeated match. */ +func lzmaStateLongRep(state *lzmaState) { + if *state < litStates { + *state = stateLitLongrep + } else { + *state = stateNonlitRep + } +} + +/* Indicate that the latest symbol was a short match. */ +func lzmaStateShortRep(state *lzmaState) { + if *state < litStates { + *state = stateLitShortrep + } else { + *state = stateNonlitRep + } +} + +/* Test if the previous symbol was a literal. */ +func lzmaStateIsLiteral(state lzmaState) bool { + return state < litStates +} + +/* Each literal coder is divided in three sections: + * - 0x001-0x0FF: Without match byte + * - 0x101-0x1FF: With match byte; match bit is 0 + * - 0x201-0x2FF: With match byte; match bit is 1 + * + * Match byte is used when the previous LZMA symbol was something else than + * a literal (that is, it was some kind of match). + */ +const literalCoderSize = 0x300 + +/* Maximum number of literal coders */ +const literalCodersMax = 1 << 4 + +/* Minimum length of a match is two bytes. */ +const matchLenMin = 2 + +/* Match length is encoded with 4, 5, or 10 bits. + * + * Length Bits + * 2-9 4 = Choice=0 + 3 bits + * 10-17 5 = Choice=1 + Choice2=0 + 3 bits + * 18-273 10 = Choice=1 + Choice2=1 + 8 bits + */ +const ( + lenLowBits = 3 + lenLowSymbols = 1 << lenLowBits + lenMidBits = 3 + lenMidSymbols = 1 << lenMidBits + lenHighBits = 8 + lenHighSymbols = 1 << lenHighBits +) + +/* + * Different sets of probabilities are used for match distances that have + * very short match length: Lengths of 2, 3, and 4 bytes have a separate + * set of probabilities for each length. The matches with longer length + * use a shared set of probabilities. + */ +const distStates = 4 + +/* + * Get the index of the appropriate probability array for decoding + * the distance slot. + */ +func lzmaGetDistState(len uint32) uint32 { + if len < distStates+matchLenMin { + return len - matchLenMin + } else { + return distStates - 1 + } +} + +/* + * The highest two bits of a 32-bit match distance are encoded using six bits. + * This six-bit value is called a distance slot. This way encoding a 32-bit + * value takes 6-36 bits, larger values taking more bits. + */ +const ( + distSlotBits = 6 + distSlots = 1 << distSlotBits +) + +/* Match distances up to 127 are fully encoded using probabilities. Since + * the highest two bits (distance slot) are always encoded using six bits, + * the distances 0-3 don't need any additional bits to encode, since the + * distance slot itself is the same as the actual distance. distModelStart + * indicates the first distance slot where at least one additional bit is + * needed. + */ +const distModelStart = 4 + +/* + * Match distances greater than 127 are encoded in three pieces: + * - distance slot: the highest two bits + * - direct bits: 2-26 bits below the highest two bits + * - alignment bits: four lowest bits + * + * Direct bits don't use any probabilities. + * + * The distance slot value of 14 is for distances 128-191. + */ +const distModelEnd = 14 + +/* Distance slots that indicate a distance <= 127. */ +const ( + fullDistancesBits = distModelEnd / 2 + fullDistances = 1 << fullDistancesBits +) + +/* + * For match distances greater than 127, only the highest two bits and the + * lowest four bits (alignment) is encoded using probabilities. + */ +const ( + alignBits = 4 + alignSize = 1 << alignBits +) + +/* from linux/lib/xz/xz_dec_lzma2.c ***********************************/ + +/* + * Range decoder initialization eats the first five bytes of each LZMA chunk. + */ +const rcInitBytes = 5 + +/* + * Minimum number of usable input buffer to safely decode one LZMA symbol. + * The worst case is that we decode 22 bits using probabilities and 26 + * direct bits. This may decode at maximum of 20 bytes of input. However, + * lzmaMain does an extra normalization before returning, thus we + * need to put 21 here. + */ +const lzmaInRequired = 21 + +/* + * Dictionary (history buffer) + * + * These are always true: + * start <= pos <= full <= end + * pos <= limit <= end + * end == size + * size <= sizeMax + * len(buf) <= size + */ +type dictionary struct { + /* The history buffer */ + buf []byte + /* Old position in buf (before decoding more data) */ + start uint32 + /* Position in buf */ + pos uint32 + /* + * How full dictionary is. This is used to detect corrupt input that + * would read beyond the beginning of the uncompressed stream. + */ + full uint32 + /* Write limit; we don't write to buf[limit] or later bytes. */ + limit uint32 + /* + * End of the dictionary buffer. This is the same as the + * dictionary size. + */ + end uint32 + /* + * Size of the dictionary as specified in Block Header. This is used + * together with "full" to detect corrupt input that would make us + * read beyond the beginning of the uncompressed stream. + */ + size uint32 + /* Maximum allowed dictionary size. */ + sizeMax uint32 +} + +/* Range decoder */ +type rcDec struct { + rnge uint32 + code uint32 + /* + * Number of initializing bytes remaining to be read + * by rcReadInit. + */ + initBytesLeft uint32 + /* + * Buffer from which we read our input. It can be either + * temp.buf or the caller-provided input buffer. + */ + in []byte + inPos int + inLimit int +} + +/* Probabilities for a length decoder. */ +type lzmaLenDec struct { + /* Probability of match length being at least 10 */ + choice uint16 + /* Probability of match length being at least 18 */ + choice2 uint16 + /* Probabilities for match lengths 2-9 */ + low [posStatesMax][lenLowSymbols]uint16 + /* Probabilities for match lengths 10-17 */ + mid [posStatesMax][lenMidSymbols]uint16 + /* Probabilities for match lengths 18-273 */ + high [lenHighSymbols]uint16 +} + +type lzmaDec struct { + /* Distances of latest four matches */ + rep0 uint32 + rep1 uint32 + rep2 uint32 + rep3 uint32 + /* Types of the most recently seen LZMA symbols */ + state lzmaState + /* + * Length of a match. This is updated so that dictRepeat can + * be called again to finish repeating the whole match. + */ + len uint32 + /* + * LZMA properties or related bit masks (number of literal + * context bits, a mask derived from the number of literal + * position bits, and a mask derived from the number + * position bits) + */ + lc uint32 + literalPosMask uint32 + posMask uint32 + /* If 1, it's a match. Otherwise it's a single 8-bit literal. */ + isMatch [states][posStatesMax]uint16 + /* If 1, it's a repeated match. The distance is one of rep0 .. rep3. */ + isRep [states]uint16 + /* + * If 0, distance of a repeated match is rep0. + * Otherwise check is_rep1. + */ + isRep0 [states]uint16 + /* + * If 0, distance of a repeated match is rep1. + * Otherwise check is_rep2. + */ + isRep1 [states]uint16 + /* If 0, distance of a repeated match is rep2. Otherwise it is rep3. */ + isRep2 [states]uint16 + /* + * If 1, the repeated match has length of one byte. Otherwise + * the length is decoded from rep_len_decoder. + */ + isRep0Long [states][posStatesMax]uint16 + /* + * Probability tree for the highest two bits of the match + * distance. There is a separate probability tree for match + * lengths of 2 (i.e. MATCH_LEN_MIN), 3, 4, and [5, 273]. + */ + distSlot [distStates][distSlots]uint16 + /* + * Probility trees for additional bits for match distance + * when the distance is in the range [4, 127]. + */ + distSpecial [fullDistances - distModelEnd]uint16 + /* + * Probability tree for the lowest four bits of a match + * distance that is equal to or greater than 128. + */ + distAlign [alignSize]uint16 + /* Length of a normal match */ + matchLenDec lzmaLenDec + /* Length of a repeated match */ + repLenDec lzmaLenDec + /* Probabilities of literals */ + literal [literalCodersMax][literalCoderSize]uint16 +} + +// type of lzma2Dec.sequence +type lzma2Seq int + +const ( + seqControl lzma2Seq = iota + seqUncompressed1 + seqUncompressed2 + seqCompressed0 + seqCompressed1 + seqProperties + seqLZMAPrepare + seqLZMARun + seqCopy +) + +type lzma2Dec struct { + /* Position in xzDecLZMA2Run. */ + sequence lzma2Seq + /* Next position after decoding the compressed size of the chunk. */ + nextSequence lzma2Seq + /* Uncompressed size of LZMA chunk (2 MiB at maximum) */ + uncompressed int + /* + * Compressed size of LZMA chunk or compressed/uncompressed + * size of uncompressed chunk (64 KiB at maximum) + */ + compressed int + /* + * True if dictionary reset is needed. This is false before + * the first chunk (LZMA or uncompressed). + */ + needDictReset bool + /* + * True if new LZMA properties are needed. This is false + * before the first LZMA chunk. + */ + needProps bool +} + +type xzDecLZMA2 struct { + /* + * The order below is important on x86 to reduce code size and + * it shouldn't hurt on other platforms. Everything up to and + * including lzma.pos_mask are in the first 128 bytes on x86-32, + * which allows using smaller instructions to access those + * variables. On x86-64, fewer variables fit into the first 128 + * bytes, but this is still the best order without sacrificing + * the readability by splitting the structures. + */ + rc rcDec + dict dictionary + lzma2 lzma2Dec + lzma lzmaDec + /* + * Temporary buffer which holds small number of input bytes between + * decoder calls. See lzma2LZMA for details. + */ + temp struct { + buf []byte // slice buf will be backed by bufArray + bufArray [3 * lzmaInRequired]byte + } +} + +/************** + * Dictionary * + **************/ + +/* + * Reset the dictionary state. When in single-call mode, set up the beginning + * of the dictionary to point to the actual output buffer. + */ +func dictReset(dict *dictionary, b *xzBuf) { + dict.start = 0 + dict.pos = 0 + dict.limit = 0 + dict.full = 0 +} + +/* Set dictionary write limit */ +func dictLimit(dict *dictionary, outMax int) { + if dict.end-dict.pos <= uint32(outMax) { + dict.limit = dict.end + } else { + dict.limit = dict.pos + uint32(outMax) + } +} + +/* Return true if at least one byte can be written into the dictionary. */ +func dictHasSpace(dict *dictionary) bool { + return dict.pos < dict.limit +} + +/* + * Get a byte from the dictionary at the given distance. The distance is + * assumed to valid, or as a special case, zero when the dictionary is + * still empty. This special case is needed for single-call decoding to + * avoid writing a '\x00' to the end of the destination buffer. + */ +func dictGet(dict *dictionary, dist uint32) uint32 { + var offset uint32 = dict.pos - dist - 1 + if dist >= dict.pos { + offset += dict.end + } + if dict.full > 0 { + return uint32(dict.buf[offset]) + } + return 0 +} + +/* + * Put one byte into the dictionary. It is assumed that there is space for it. + */ +func dictPut(dict *dictionary, byte byte) { + dict.buf[dict.pos] = byte + dict.pos++ + if dict.full < dict.pos { + dict.full = dict.pos + } +} + +/* + * Repeat given number of bytes from the given distance. If the distance is + * invalid, false is returned. On success, true is returned and *len is + * updated to indicate how many bytes were left to be repeated. + */ +func dictRepeat(dict *dictionary, len *uint32, dist uint32) bool { + var back uint32 + var left uint32 + if dist >= dict.full || dist >= dict.size { + return false + } + left = dict.limit - dict.pos + if left > *len { + left = *len + } + *len -= left + back = dict.pos - dist - 1 + if dist >= dict.pos { + back += dict.end + } + for { + dict.buf[dict.pos] = dict.buf[back] + dict.pos++ + back++ + if back == dict.end { + back = 0 + } + left-- + if !(left > 0) { + break + } + } + if dict.full < dict.pos { + dict.full = dict.pos + } + return true +} + +/* Copy uncompressed data as is from input to dictionary and output buffers. */ +func dictUncompressed(dict *dictionary, b *xzBuf, left *int) { + var copySize int + for *left > 0 && b.inPos < len(b.in) && b.outPos < len(b.out) { + copySize = len(b.in) - b.inPos + if copySize > len(b.out)-b.outPos { + copySize = len(b.out) - b.outPos + } + if copySize > int(dict.end-dict.pos) { + copySize = int(dict.end - dict.pos) + } + if copySize > *left { + copySize = *left + } + *left -= copySize + copy(dict.buf[dict.pos:], b.in[b.inPos:b.inPos+copySize]) + dict.pos += uint32(copySize) + if dict.full < dict.pos { + dict.full = dict.pos + } + if dict.pos == dict.end { + dict.pos = 0 + } + copy(b.out[b.outPos:], b.in[b.inPos:b.inPos+copySize]) + dict.start = dict.pos + b.outPos += copySize + b.inPos += copySize + } +} + +/* + * Flush pending data from dictionary to b.out. It is assumed that there is + * enough space in b.out. This is guaranteed because caller uses dictLimit + * before decoding data into the dictionary. + */ +func dictFlush(dict *dictionary, b *xzBuf) int { + var copySize int = int(dict.pos - dict.start) + if dict.pos == dict.end { + dict.pos = 0 + } + copy(b.out[b.outPos:], dict.buf[dict.start:dict.start+uint32(copySize)]) + dict.start = dict.pos + b.outPos += copySize + return copySize +} + +/***************** + * Range decoder * + *****************/ + +/* Reset the range decoder. */ +func rcReset(rc *rcDec) { + rc.rnge = ^uint32(0) + rc.code = 0 + rc.initBytesLeft = rcInitBytes +} + +/* + * Read the first five initial bytes into rc->code if they haven't been + * read already. (Yes, the first byte gets completely ignored.) + */ +func rcReadInit(rc *rcDec, b *xzBuf) bool { + for rc.initBytesLeft > 0 { + if b.inPos == len(b.in) { + return false + } + rc.code = rc.code<<8 + uint32(b.in[b.inPos]) + b.inPos++ + rc.initBytesLeft-- + } + return true +} + +/* Return true if there may not be enough input for the next decoding loop. */ +func rcLimitExceeded(rc *rcDec) bool { + return rc.inPos > rc.inLimit +} + +/* + * Return true if it is possible (from point of view of range decoder) that + * we have reached the end of the LZMA chunk. + */ +func rcIsFinished(rc *rcDec) bool { + return rc.code == 0 +} + +/* Read the next input byte if needed. */ +func rcNormalize(rc *rcDec) { + if rc.rnge < rcTopValue { + rc.rnge <<= rcShiftBits + rc.code = rc.code<> rcBitModelTotalBits) * uint32(*prob) + if rc.code < bound { + rc.rnge = bound + *prob += (rcBitModelTotal - *prob) >> rcMoveBits + bit = false + } else { + rc.rnge -= bound + rc.code -= bound + *prob -= *prob >> rcMoveBits + bit = true + } + return bit +} + +/* Decode a bittree starting from the most significant bit. */ +func rcBittree(rc *rcDec, probs []uint16, limit uint32) uint32 { + var symbol uint32 = 1 + for { + if rcBit(rc, &probs[symbol-1]) { + symbol = symbol<<1 + 1 + } else { + symbol <<= 1 + } + if !(symbol < limit) { + break + } + } + return symbol +} + +/* Decode a bittree starting from the least significant bit. */ +func rcBittreeReverse(rc *rcDec, probs []uint16, dest *uint32, limit uint32) { + var symbol uint32 = 1 + var i uint32 = 0 + for { + if rcBit(rc, &probs[symbol-1]) { + symbol = symbol<<1 + 1 + *dest += 1 << i + } else { + symbol <<= 1 + } + i++ + if !(i < limit) { + break + } + } +} + +/* Decode direct bits (fixed fifty-fifty probability) */ +func rcDirect(rc *rcDec, dest *uint32, limit uint32) { + var mask uint32 + for { + rcNormalize(rc) + rc.rnge >>= 1 + rc.code -= rc.rnge + mask = 0 - rc.code>>31 + rc.code += rc.rnge & mask + *dest = *dest<<1 + mask + 1 + limit-- + if !(limit > 0) { + break + } + } +} + +/******** + * LZMA * + ********/ + +/* Get pointer to literal coder probability array. */ +func lzmaLiteralProbs(s *xzDecLZMA2) []uint16 { + var prevByte uint32 = dictGet(&s.dict, 0) + var low uint32 = prevByte >> (8 - s.lzma.lc) + var high uint32 = (s.dict.pos & s.lzma.literalPosMask) << s.lzma.lc + return s.lzma.literal[low+high][:] +} + +/* Decode a literal (one 8-bit byte) */ +func lzmaLiteral(s *xzDecLZMA2) { + var probs []uint16 + var symbol uint32 + var matchByte uint32 + var matchBit uint32 + var offset uint32 + var i uint32 + probs = lzmaLiteralProbs(s) + if lzmaStateIsLiteral(s.lzma.state) { + symbol = rcBittree(&s.rc, probs[1:], 0x100) + } else { + symbol = 1 + matchByte = dictGet(&s.dict, s.lzma.rep0) << 1 + offset = 0x100 + for { + matchBit = matchByte & offset + matchByte <<= 1 + i = offset + matchBit + symbol + if rcBit(&s.rc, &probs[i]) { + symbol = symbol<<1 + 1 + offset &= matchBit + } else { + symbol <<= 1 + offset &= ^matchBit + } + if !(symbol < 0x100) { + break + } + } + } + dictPut(&s.dict, byte(symbol)) + lzmaStateLiteral(&s.lzma.state) +} + +/* Decode the length of the match into s.lzma.len. */ +func lzmaLen(s *xzDecLZMA2, l *lzmaLenDec, posState uint32) { + var probs []uint16 + var limit uint32 + switch { + case !rcBit(&s.rc, &l.choice): + probs = l.low[posState][:] + limit = lenLowSymbols + s.lzma.len = matchLenMin + case !rcBit(&s.rc, &l.choice2): + probs = l.mid[posState][:] + limit = lenMidSymbols + s.lzma.len = matchLenMin + lenLowSymbols + default: + probs = l.high[:] + limit = lenHighSymbols + s.lzma.len = matchLenMin + lenLowSymbols + lenMidSymbols + } + s.lzma.len += rcBittree(&s.rc, probs[1:], limit) - limit +} + +/* Decode a match. The distance will be stored in s.lzma.rep0. */ +func lzmaMatch(s *xzDecLZMA2, posState uint32) { + var probs []uint16 + var distSlot uint32 + var limit uint32 + lzmaStateMatch(&s.lzma.state) + s.lzma.rep3 = s.lzma.rep2 + s.lzma.rep2 = s.lzma.rep1 + s.lzma.rep1 = s.lzma.rep0 + lzmaLen(s, &s.lzma.matchLenDec, posState) + probs = s.lzma.distSlot[lzmaGetDistState(s.lzma.len)][:] + distSlot = rcBittree(&s.rc, probs[1:], distSlots) - distSlots + if distSlot < distModelStart { + s.lzma.rep0 = distSlot + } else { + limit = distSlot>>1 - 1 + s.lzma.rep0 = 2 + distSlot&1 + if distSlot < distModelEnd { + s.lzma.rep0 <<= limit + probs = s.lzma.distSpecial[s.lzma.rep0-distSlot:] + rcBittreeReverse(&s.rc, probs, &s.lzma.rep0, limit) + } else { + rcDirect(&s.rc, &s.lzma.rep0, limit-alignBits) + s.lzma.rep0 <<= alignBits + rcBittreeReverse( + &s.rc, s.lzma.distAlign[1:], &s.lzma.rep0, alignBits) + } + } +} + +/* + * Decode a repeated match. The distance is one of the four most recently + * seen matches. The distance will be stored in s.lzma.rep0. + */ +func lzmaRepMatch(s *xzDecLZMA2, posState uint32) { + var tmp uint32 + if !rcBit(&s.rc, &s.lzma.isRep0[s.lzma.state]) { + if !rcBit(&s.rc, &s.lzma.isRep0Long[s.lzma.state][posState]) { + lzmaStateShortRep(&s.lzma.state) + s.lzma.len = 1 + return + } + } else { + if !rcBit(&s.rc, &s.lzma.isRep1[s.lzma.state]) { + tmp = s.lzma.rep1 + } else { + if !rcBit(&s.rc, &s.lzma.isRep2[s.lzma.state]) { + tmp = s.lzma.rep2 + } else { + tmp = s.lzma.rep3 + s.lzma.rep3 = s.lzma.rep2 + } + s.lzma.rep2 = s.lzma.rep1 + } + s.lzma.rep1 = s.lzma.rep0 + s.lzma.rep0 = tmp + } + lzmaStateLongRep(&s.lzma.state) + lzmaLen(s, &s.lzma.repLenDec, posState) +} + +/* LZMA decoder core */ +func lzmaMain(s *xzDecLZMA2) bool { + var posState uint32 + /* + * If the dictionary was reached during the previous call, try to + * finish the possibly pending repeat in the dictionary. + */ + if dictHasSpace(&s.dict) && s.lzma.len > 0 { + dictRepeat(&s.dict, &s.lzma.len, s.lzma.rep0) + } + /* + * Decode more LZMA symbols. One iteration may consume up to + * lzmaInRequired - 1 bytes. + */ + for dictHasSpace(&s.dict) && !rcLimitExceeded(&s.rc) { + posState = s.dict.pos & s.lzma.posMask + if !rcBit(&s.rc, &s.lzma.isMatch[s.lzma.state][posState]) { + lzmaLiteral(s) + } else { + if rcBit(&s.rc, &s.lzma.isRep[s.lzma.state]) { + lzmaRepMatch(s, posState) + } else { + lzmaMatch(s, posState) + } + if !dictRepeat(&s.dict, &s.lzma.len, s.lzma.rep0) { + return false + } + } + } + /* + * Having the range decoder always normalized when we are outside + * this function makes it easier to correctly handle end of the chunk. + */ + rcNormalize(&s.rc) + return true +} + +/* + * Reset the LZMA decoder and range decoder state. Dictionary is not reset + * here, because LZMA state may be reset without resetting the dictionary. + */ +func lzmaReset(s *xzDecLZMA2) { + s.lzma.state = stateLitLit + s.lzma.rep0 = 0 + s.lzma.rep1 = 0 + s.lzma.rep2 = 0 + s.lzma.rep3 = 0 + /* All probabilities are initialized to the same value, v */ + v := uint16(rcBitModelTotal / 2) + s.lzma.matchLenDec.choice = v + s.lzma.matchLenDec.choice2 = v + s.lzma.repLenDec.choice = v + s.lzma.repLenDec.choice2 = v + for _, m := range [][]uint16{ + s.lzma.isRep[:], s.lzma.isRep0[:], s.lzma.isRep1[:], + s.lzma.isRep2[:], s.lzma.distSpecial[:], s.lzma.distAlign[:], + s.lzma.matchLenDec.high[:], s.lzma.repLenDec.high[:], + } { + for j := range m { + m[j] = v + } + } + for i := range s.lzma.isMatch { + for j := range s.lzma.isMatch[i] { + s.lzma.isMatch[i][j] = v + } + } + for i := range s.lzma.isRep0Long { + for j := range s.lzma.isRep0Long[i] { + s.lzma.isRep0Long[i][j] = v + } + } + for i := range s.lzma.distSlot { + for j := range s.lzma.distSlot[i] { + s.lzma.distSlot[i][j] = v + } + } + for i := range s.lzma.literal { + for j := range s.lzma.literal[i] { + s.lzma.literal[i][j] = v + } + } + for i := range s.lzma.matchLenDec.low { + for j := range s.lzma.matchLenDec.low[i] { + s.lzma.matchLenDec.low[i][j] = v + } + } + for i := range s.lzma.matchLenDec.mid { + for j := range s.lzma.matchLenDec.mid[i] { + s.lzma.matchLenDec.mid[i][j] = v + } + } + for i := range s.lzma.repLenDec.low { + for j := range s.lzma.repLenDec.low[i] { + s.lzma.repLenDec.low[i][j] = v + } + } + for i := range s.lzma.repLenDec.mid { + for j := range s.lzma.repLenDec.mid[i] { + s.lzma.repLenDec.mid[i][j] = v + } + } + rcReset(&s.rc) +} + +/* + * Decode and validate LZMA properties (lc/lp/pb) and calculate the bit masks + * from the decoded lp and pb values. On success, the LZMA decoder state is + * reset and true is returned. + */ +func lzmaProps(s *xzDecLZMA2, props byte) bool { + if props > (4*5+4)*9+8 { + return false + } + s.lzma.posMask = 0 + for props >= 9*5 { + props -= 9 * 5 + s.lzma.posMask++ + } + s.lzma.posMask = 1<= 9 { + props -= 9 + s.lzma.literalPosMask++ + } + s.lzma.lc = uint32(props) + if s.lzma.lc+s.lzma.literalPosMask > 4 { + return false + } + s.lzma.literalPosMask = 1< 0 || s.lzma2.compressed == 0 { + tmp = 2*lzmaInRequired - len(s.temp.buf) + if tmp > s.lzma2.compressed-len(s.temp.buf) { + tmp = s.lzma2.compressed - len(s.temp.buf) + } + if tmp > inAvail { + tmp = inAvail + } + copy(s.temp.bufArray[len(s.temp.buf):], b.in[b.inPos:b.inPos+tmp]) + switch { + case len(s.temp.buf)+tmp == s.lzma2.compressed: + for i := len(s.temp.buf) + tmp; i < len(s.temp.bufArray); i++ { + s.temp.bufArray[i] = 0 + } + s.rc.inLimit = len(s.temp.buf) + tmp + case len(s.temp.buf)+tmp < lzmaInRequired: + s.temp.buf = s.temp.bufArray[:len(s.temp.buf)+tmp] + b.inPos += tmp + return true + default: + s.rc.inLimit = len(s.temp.buf) + tmp - lzmaInRequired + } + s.rc.in = s.temp.bufArray[:] + s.rc.inPos = 0 + if !lzmaMain(s) || s.rc.inPos > len(s.temp.buf)+tmp { + return false + } + s.lzma2.compressed -= s.rc.inPos + if s.rc.inPos < len(s.temp.buf) { + copy(s.temp.buf, s.temp.buf[s.rc.inPos:]) + s.temp.buf = s.temp.buf[:len(s.temp.buf)-s.rc.inPos] + return true + } + b.inPos += s.rc.inPos - len(s.temp.buf) + s.temp.buf = nil + } + inAvail = len(b.in) - b.inPos + if inAvail >= lzmaInRequired { + s.rc.in = b.in + s.rc.inPos = b.inPos + if inAvail >= s.lzma2.compressed+lzmaInRequired { + s.rc.inLimit = b.inPos + s.lzma2.compressed + } else { + s.rc.inLimit = len(b.in) - lzmaInRequired + } + if !lzmaMain(s) { + return false + } + inAvail = s.rc.inPos - b.inPos + if inAvail > s.lzma2.compressed { + return false + } + s.lzma2.compressed -= inAvail + b.inPos = s.rc.inPos + } + inAvail = len(b.in) - b.inPos + if inAvail < lzmaInRequired { + if inAvail > s.lzma2.compressed { + inAvail = s.lzma2.compressed + } + s.temp.buf = s.temp.bufArray[:inAvail] + copy(s.temp.buf, b.in[b.inPos:]) + b.inPos += inAvail + } + return true +} + +/* + * Take care of the LZMA2 control layer, and forward the job of actual LZMA + * decoding or copying of uncompressed chunks to other functions. + */ +func xzDecLZMA2Run(s *xzDecLZMA2, b *xzBuf) xzRet { + var tmp int + for b.inPos < len(b.in) || s.lzma2.sequence == seqLZMARun { + switch s.lzma2.sequence { + case seqControl: + /* + * LZMA2 control byte + * + * Exact values: + * 0x00 End marker + * 0x01 Dictionary reset followed by + * an uncompressed chunk + * 0x02 Uncompressed chunk (no dictionary reset) + * + * Highest three bits (s.control & 0xE0): + * 0xE0 Dictionary reset, new properties and state + * reset, followed by LZMA compressed chunk + * 0xC0 New properties and state reset, followed + * by LZMA compressed chunk (no dictionary + * reset) + * 0xA0 State reset using old properties, + * followed by LZMA compressed chunk (no + * dictionary reset) + * 0x80 LZMA chunk (no dictionary or state reset) + * + * For LZMA compressed chunks, the lowest five bits + * (s.control & 1F) are the highest bits of the + * uncompressed size (bits 16-20). + * + * A new LZMA2 stream must begin with a dictionary + * reset. The first LZMA chunk must set new + * properties and reset the LZMA state. + * + * Values that don't match anything described above + * are invalid and we return xzDataError. + */ + tmp = int(b.in[b.inPos]) + b.inPos++ + if tmp == 0x00 { + return xzStreamEnd + } + switch { + case tmp >= 0xe0 || tmp == 0x01: + s.lzma2.needProps = true + s.lzma2.needDictReset = false + dictReset(&s.dict, b) + case s.lzma2.needDictReset: + return xzDataError + } + if tmp >= 0x80 { + s.lzma2.uncompressed = (tmp & 0x1f) << 16 + s.lzma2.sequence = seqUncompressed1 + switch { + case tmp >= 0xc0: + /* + * When there are new properties, + * state reset is done at + * seqProperties. + */ + s.lzma2.needProps = false + s.lzma2.nextSequence = seqProperties + case s.lzma2.needProps: + return xzDataError + default: + s.lzma2.nextSequence = seqLZMAPrepare + if tmp >= 0xa0 { + lzmaReset(s) + } + } + } else { + if tmp > 0x02 { + return xzDataError + } + s.lzma2.sequence = seqCompressed0 + s.lzma2.nextSequence = seqCopy + } + case seqUncompressed1: + s.lzma2.uncompressed += int(b.in[b.inPos]) << 8 + b.inPos++ + s.lzma2.sequence = seqUncompressed2 + case seqUncompressed2: + s.lzma2.uncompressed += int(b.in[b.inPos]) + 1 + b.inPos++ + s.lzma2.sequence = seqCompressed0 + case seqCompressed0: + s.lzma2.compressed += int(b.in[b.inPos]) << 8 + b.inPos++ + s.lzma2.sequence = seqCompressed1 + case seqCompressed1: + s.lzma2.compressed += int(b.in[b.inPos]) + 1 + b.inPos++ + s.lzma2.sequence = s.lzma2.nextSequence + case seqProperties: + if !lzmaProps(s, b.in[b.inPos]) { + return xzDataError + } + b.inPos++ + s.lzma2.sequence = seqLZMAPrepare + fallthrough + case seqLZMAPrepare: + if s.lzma2.compressed < rcInitBytes { + return xzDataError + } + if !rcReadInit(&s.rc, b) { + return xzOK + } + s.lzma2.compressed -= rcInitBytes + s.lzma2.sequence = seqLZMARun + fallthrough + case seqLZMARun: + /* + * Set dictionary limit to indicate how much we want + * to be encoded at maximum. Decode new data into the + * dictionary. Flush the new data from dictionary to + * b.out. Check if we finished decoding this chunk. + * In case the dictionary got full but we didn't fill + * the output buffer yet, we may run this loop + * multiple times without changing s.lzma2.sequence. + */ + outMax := len(b.out) - b.outPos + if outMax > s.lzma2.uncompressed { + outMax = s.lzma2.uncompressed + } + dictLimit(&s.dict, outMax) + if !lzma2LZMA(s, b) { + return xzDataError + } + s.lzma2.uncompressed -= dictFlush(&s.dict, b) + switch { + case s.lzma2.uncompressed == 0: + if s.lzma2.compressed > 0 || s.lzma.len > 0 || + !rcIsFinished(&s.rc) { + return xzDataError + } + rcReset(&s.rc) + s.lzma2.sequence = seqControl + case b.outPos == len(b.out) || + b.inPos == len(b.in) && + len(s.temp.buf) < s.lzma2.compressed: + return xzOK + } + case seqCopy: + dictUncompressed(&s.dict, b, &s.lzma2.compressed) + if s.lzma2.compressed > 0 { + return xzOK + } + s.lzma2.sequence = seqControl + } + } + return xzOK +} + +/* + * Allocate memory for LZMA2 decoder. xzDecLZMA2Reset must be used + * before calling xzDecLZMA2Run. + */ +func xzDecLZMA2Create(dictMax uint32) *xzDecLZMA2 { + s := new(xzDecLZMA2) + s.dict.sizeMax = dictMax + return s +} + +/* + * Decode the LZMA2 properties (one byte) and reset the decoder. Return + * xzOK on success, xzMemlimitError if the preallocated dictionary is not + * big enough, and xzOptionsError if props indicates something that this + * decoder doesn't support. + */ +func xzDecLZMA2Reset(s *xzDecLZMA2, props byte) xzRet { + if props > 40 { + return xzOptionsError // Bigger than 4 GiB + } + if props == 40 { + s.dict.size = ^uint32(0) + } else { + s.dict.size = uint32(2 + props&1) + s.dict.size <<= props>>1 + 11 + } + if s.dict.size > s.dict.sizeMax { + return xzMemlimitError + } + s.dict.end = s.dict.size + if len(s.dict.buf) < int(s.dict.size) { + s.dict.buf = make([]byte, s.dict.size) + } + s.lzma.len = 0 + s.lzma2.sequence = seqControl + s.lzma2.compressed = 0 + s.lzma2.uncompressed = 0 + s.lzma2.needDictReset = true + s.temp.buf = nil + return xzOK +} diff --git a/vendor/github.com/therootcompany/xz/dec_stream.go b/vendor/github.com/therootcompany/xz/dec_stream.go new file mode 100644 index 00000000..9381a3c8 --- /dev/null +++ b/vendor/github.com/therootcompany/xz/dec_stream.go @@ -0,0 +1,932 @@ +/* + * .xz Stream decoder + * + * Author: Lasse Collin + * + * Translation to Go: Michael Cross + * + * This file has been put into the public domain. + * You can do whatever you want with this file. + */ + +package xz + +import ( + "bytes" + "crypto/sha256" + "hash" + "hash/crc32" + "hash/crc64" +) + +/* from linux/lib/xz/xz_stream.h **************************************/ + +/* + * See the .xz file format specification at + * http://tukaani.org/xz/xz-file-format.txt + * to understand the container format. + */ +const ( + streamHeaderSize = 12 + headerMagic = "\xfd7zXZ\x00" + footerMagic = "YZ" +) + +/* + * Variable-length integer can hold a 63-bit unsigned integer or a special + * value indicating that the value is unknown. + */ +type vliType uint64 + +const ( + vliUnknown vliType = ^vliType(0) + /* Maximum encoded size of a VLI */ + vliBytesMax = 8 * 8 / 7 // (Sizeof(vliType) * 8 / 7) +) + +/* from linux/lib/xz/xz_dec_stream.c **********************************/ + +/* Hash used to validate the Index field */ +type xzDecHash struct { + unpadded vliType + uncompressed vliType + sha256 hash.Hash +} + +// type of xzDec.sequence +type xzDecSeq int + +const ( + seqStreamHeader xzDecSeq = iota + seqBlockStart + seqBlockHeader + seqBlockUncompress + seqBlockPadding + seqBlockCheck + seqIndex + seqIndexPadding + seqIndexCRC32 + seqStreamFooter +) + +// type of xzDec.index.sequence +type xzDecIndexSeq int + +const ( + seqIndexCount xzDecIndexSeq = iota + seqIndexUnpadded + seqIndexUncompressed +) + +/** + * xzDec - Opaque type to hold the XZ decoder state + */ +type xzDec struct { + /* Position in decMain */ + sequence xzDecSeq + /* Position in variable-length integers and Check fields */ + pos int + /* Variable-length integer decoded by decVLI */ + vli vliType + /* Saved inPos and outPos */ + inStart int + outStart int + /* CRC32 checksum hash used in Index */ + crc32 hash.Hash + /* Hashes used in Blocks */ + checkCRC32 hash.Hash + checkCRC64 hash.Hash + checkSHA256 hash.Hash + /* for checkTypes CRC32/CRC64/SHA256, check is one of the above 3 hashes */ + check hash.Hash + /* Embedded stream header struct containing CheckType */ + *Header + /* + * True if the next call to xzDecRun is allowed to return + * xzBufError. + */ + allowBufError bool + /* Information stored in Block Header */ + blockHeader struct { + /* + * Value stored in the Compressed Size field, or + * vliUnknown if Compressed Size is not present. + */ + compressed vliType + /* + * Value stored in the Uncompressed Size field, or + * vliUnknown if Uncompressed Size is not present. + */ + uncompressed vliType + /* Size of the Block Header field */ + size int + } + /* Information collected when decoding Blocks */ + block struct { + /* Observed compressed size of the current Block */ + compressed vliType + /* Observed uncompressed size of the current Block */ + uncompressed vliType + /* Number of Blocks decoded so far */ + count vliType + /* + * Hash calculated from the Block sizes. This is used to + * validate the Index field. + */ + hash xzDecHash + } + /* Variables needed when verifying the Index field */ + index struct { + /* Position in decIndex */ + sequence xzDecIndexSeq + /* Size of the Index in bytes */ + size vliType + /* Number of Records (matches block.count in valid files) */ + count vliType + /* + * Hash calculated from the Records (matches block.hash in + * valid files). + */ + hash xzDecHash + } + /* + * Temporary buffer needed to hold Stream Header, Block Header, + * and Stream Footer. The Block Header is the biggest (1 KiB) + * so we reserve space according to that. bufArray has to be aligned + * to a multiple of four bytes; the variables before it + * should guarantee this. + */ + temp struct { + pos int + buf []byte // slice buf will be backed by bufArray + bufArray [1024]byte + } + // chain is the function (or to be more precise, closure) which + // does the decompression and will call into the lzma2 and other + // filter code as needed. It is constructed by decBlockHeader + chain func(b *xzBuf) xzRet + // lzma2 holds the state of the last filter (which must be LZMA2) + lzma2 *xzDecLZMA2 + // pointers to allocated BCJ/Delta filters + bcjs []*xzDecBCJ + deltas []*xzDecDelta + // number of currently in use BCJ/Delta filters from the above + bcjsUsed int + deltasUsed int +} + +/* Sizes of the Check field with different Check IDs */ +var checkSizes = [...]byte{ + 0, + 4, 4, 4, + 8, 8, 8, + 16, 16, 16, + 32, 32, 32, + 64, 64, 64, +} + +/* + * Fill s.temp by copying data starting from b.in[b.inPos]. Caller + * must have set s.temp.pos to indicate how much data we are supposed + * to copy into s.temp.buf. Return true once s.temp.pos has reached + * len(s.temp.buf). + */ +func fillTemp(s *xzDec, b *xzBuf) bool { + copySize := len(b.in) - b.inPos + tempRemaining := len(s.temp.buf) - s.temp.pos + if copySize > tempRemaining { + copySize = tempRemaining + } + copy(s.temp.buf[s.temp.pos:], b.in[b.inPos:]) + b.inPos += copySize + s.temp.pos += copySize + if s.temp.pos == len(s.temp.buf) { + s.temp.pos = 0 + return true + } + return false +} + +/* Decode a variable-length integer (little-endian base-128 encoding) */ +func decVLI(s *xzDec, in []byte, inPos *int) xzRet { + var byte byte + if s.pos == 0 { + s.vli = 0 + } + for *inPos < len(in) { + byte = in[*inPos] + *inPos++ + s.vli |= vliType(byte&0x7f) << uint(s.pos) + if byte&0x80 == 0 { + /* Don't allow non-minimal encodings. */ + if byte == 0 && s.pos != 0 { + return xzDataError + } + s.pos = 0 + return xzStreamEnd + } + s.pos += 7 + if s.pos == 7*vliBytesMax { + return xzDataError + } + } + return xzOK +} + +/* + * Decode the Compressed Data field from a Block. Update and validate + * the observed compressed and uncompressed sizes of the Block so that + * they don't exceed the values possibly stored in the Block Header + * (validation assumes that no integer overflow occurs, since vliType + * is uint64). Update s.check if presence of the CRC32/CRC64/SHA256 + * field was indicated in Stream Header. + * + * Once the decoding is finished, validate that the observed sizes match + * the sizes possibly stored in the Block Header. Update the hash and + * Block count, which are later used to validate the Index field. + */ +func decBlock(s *xzDec, b *xzBuf) xzRet { + var ret xzRet + s.inStart = b.inPos + s.outStart = b.outPos + ret = s.chain(b) + s.block.compressed += vliType(b.inPos - s.inStart) + s.block.uncompressed += vliType(b.outPos - s.outStart) + /* + * There is no need to separately check for vliUnknown since + * the observed sizes are always smaller than vliUnknown. + */ + if s.block.compressed > s.blockHeader.compressed || + s.block.uncompressed > s.blockHeader.uncompressed { + return xzDataError + } + switch s.CheckType { + case CheckCRC32, CheckCRC64, CheckSHA256: + _, _ = s.check.Write(b.out[s.outStart:b.outPos]) + } + if ret == xzStreamEnd { + if s.blockHeader.compressed != vliUnknown && + s.blockHeader.compressed != s.block.compressed { + return xzDataError + } + if s.blockHeader.uncompressed != vliUnknown && + s.blockHeader.uncompressed != s.block.uncompressed { + return xzDataError + } + s.block.hash.unpadded += + vliType(s.blockHeader.size) + s.block.compressed + s.block.hash.unpadded += vliType(checkSizes[s.CheckType]) + s.block.hash.uncompressed += s.block.uncompressed + var buf [2 * 8]byte // 2*Sizeof(vliType) + putLE64(uint64(s.block.hash.unpadded), buf[:]) + putLE64(uint64(s.block.hash.uncompressed), buf[8:]) + _, _ = s.block.hash.sha256.Write(buf[:]) + s.block.count++ + } + return ret +} + +/* Update the Index size and the CRC32 hash. */ +func indexUpdate(s *xzDec, b *xzBuf) { + inUsed := b.inPos - s.inStart + s.index.size += vliType(inUsed) + _, _ = s.crc32.Write(b.in[s.inStart : s.inStart+inUsed]) +} + +/* + * Decode the Number of Records, Unpadded Size, and Uncompressed Size + * fields from the Index field. That is, Index Padding and CRC32 are not + * decoded by this function. + * + * This can return xzOK (more input needed), xzStreamEnd (everything + * successfully decoded), or xzDataError (input is corrupt). + */ +func decIndex(s *xzDec, b *xzBuf) xzRet { + var ret xzRet + for { + ret = decVLI(s, b.in, &b.inPos) + if ret != xzStreamEnd { + indexUpdate(s, b) + return ret + } + switch s.index.sequence { + case seqIndexCount: + s.index.count = s.vli + /* + * Validate that the Number of Records field + * indicates the same number of Records as + * there were Blocks in the Stream. + */ + if s.index.count != s.block.count { + return xzDataError + } + s.index.sequence = seqIndexUnpadded + case seqIndexUnpadded: + s.index.hash.unpadded += s.vli + s.index.sequence = seqIndexUncompressed + case seqIndexUncompressed: + s.index.hash.uncompressed += s.vli + var buf [2 * 8]byte // 2*Sizeof(vliType) + putLE64(uint64(s.index.hash.unpadded), buf[:]) + putLE64(uint64(s.index.hash.uncompressed), buf[8:]) + _, _ = s.index.hash.sha256.Write(buf[:]) + s.index.count-- + s.index.sequence = seqIndexUnpadded + } + if !(s.index.count > 0) { + break + } + } + return xzStreamEnd +} + +/* + * Validate that the next 4 bytes match s.crc32.Sum(nil). s.pos must + * be zero when starting to validate the first byte. + */ +func crcValidate(s *xzDec, b *xzBuf) xzRet { + sum := s.crc32.Sum(nil) + // CRC32 - reverse slice + sum[0], sum[1], sum[2], sum[3] = sum[3], sum[2], sum[1], sum[0] + for { + if b.inPos == len(b.in) { + return xzOK + } + if sum[s.pos] != b.in[b.inPos] { + return xzDataError + } + b.inPos++ + s.pos++ + if !(s.pos < 4) { + break + } + } + s.crc32.Reset() + s.pos = 0 + return xzStreamEnd +} + +/* + * Validate that the next 4/8/32 bytes match s.check.Sum(nil). s.pos + * must be zero when starting to validate the first byte. + */ +func checkValidate(s *xzDec, b *xzBuf) xzRet { + sum := s.check.Sum(nil) + if s.CheckType == CheckCRC32 || s.CheckType == CheckCRC64 { + // CRC32/64 - reverse slice + for i, j := 0, len(sum)-1; i < j; i, j = i+1, j-1 { + sum[i], sum[j] = sum[j], sum[i] + } + } + for { + if b.inPos == len(b.in) { + return xzOK + } + if sum[s.pos] != b.in[b.inPos] { + return xzDataError + } + b.inPos++ + s.pos++ + if !(s.pos < len(sum)) { + break + } + } + s.check.Reset() + s.pos = 0 + return xzStreamEnd +} + +/* + * Skip over the Check field when the Check ID is not supported. + * Returns true once the whole Check field has been skipped over. + */ +func checkSkip(s *xzDec, b *xzBuf) bool { + for s.pos < int(checkSizes[s.CheckType]) { + if b.inPos == len(b.in) { + return false + } + b.inPos++ + s.pos++ + } + s.pos = 0 + return true +} + +/* polynomial table used in decStreamHeader below */ +var xzCRC64Table = crc64.MakeTable(crc64.ECMA) + +/* Decode the Stream Header field (the first 12 bytes of the .xz Stream). */ +func decStreamHeader(s *xzDec) xzRet { + if string(s.temp.buf[:len(headerMagic)]) != headerMagic { + return xzFormatError + } + if crc32.ChecksumIEEE(s.temp.buf[len(headerMagic):len(headerMagic)+2]) != + getLE32(s.temp.buf[len(headerMagic)+2:]) { + return xzDataError + } + if s.temp.buf[len(headerMagic)] != 0 { + return xzOptionsError + } + /* + * Of integrity checks, we support none (Check ID = 0), + * CRC32 (Check ID = 1), CRC64 (Check ID = 4) and SHA256 (Check ID = 10) + * However, we will accept other check types too, but then the check + * won't be verified and a warning (xzUnsupportedCheck) will be given. + */ + s.CheckType = CheckID(s.temp.buf[len(headerMagic)+1]) + if s.CheckType > checkMax { + return xzOptionsError + } + switch s.CheckType { + case CheckNone: + // CheckNone: no action needed + case CheckCRC32: + if s.checkCRC32 == nil { + s.checkCRC32 = crc32.NewIEEE() + } else { + s.checkCRC32.Reset() + } + s.check = s.checkCRC32 + case CheckCRC64: + if s.checkCRC64 == nil { + s.checkCRC64 = crc64.New(xzCRC64Table) + } else { + s.checkCRC64.Reset() + } + s.check = s.checkCRC64 + case CheckSHA256: + if s.checkSHA256 == nil { + s.checkSHA256 = sha256.New() + } else { + s.checkSHA256.Reset() + } + s.check = s.checkSHA256 + default: + return xzUnsupportedCheck + } + return xzOK +} + +/* Decode the Stream Footer field (the last 12 bytes of the .xz Stream) */ +func decStreamFooter(s *xzDec) xzRet { + if string(s.temp.buf[10:10+len(footerMagic)]) != footerMagic { + return xzDataError + } + if crc32.ChecksumIEEE(s.temp.buf[4:10]) != getLE32(s.temp.buf) { + return xzDataError + } + /* + * Validate Backward Size. Note that we never added the size of the + * Index CRC32 field to s->index.size, thus we use s->index.size / 4 + * instead of s->index.size / 4 - 1. + */ + if s.index.size>>2 != vliType(getLE32(s.temp.buf[4:])) { + return xzDataError + } + if s.temp.buf[8] != 0 || CheckID(s.temp.buf[9]) != s.CheckType { + return xzDataError + } + /* + * Use xzStreamEnd instead of xzOK to be more convenient + * for the caller. + */ + return xzStreamEnd +} + +/* Decode the Block Header and initialize the filter chain. */ +func decBlockHeader(s *xzDec) xzRet { + var ret xzRet + /* + * Validate the CRC32. We know that the temp buffer is at least + * eight bytes so this is safe. + */ + crc := getLE32(s.temp.buf[len(s.temp.buf)-4:]) + s.temp.buf = s.temp.buf[:len(s.temp.buf)-4] + if crc32.ChecksumIEEE(s.temp.buf) != crc { + return xzDataError + } + s.temp.pos = 2 + /* + * Catch unsupported Block Flags. + */ + if s.temp.buf[1]&0x3C != 0 { + return xzOptionsError + } + /* Compressed Size */ + if s.temp.buf[1]&0x40 != 0 { + if decVLI(s, s.temp.buf, &s.temp.pos) != xzStreamEnd { + return xzDataError + } + if s.vli >= 1<<63-8 { + // the whole block must stay smaller than 2^63 bytes + // the block header cannot be smaller than 8 bytes + return xzDataError + } + if s.vli == 0 { + // compressed size must be non-zero + return xzDataError + } + s.blockHeader.compressed = s.vli + } else { + s.blockHeader.compressed = vliUnknown + } + /* Uncompressed Size */ + if s.temp.buf[1]&0x80 != 0 { + if decVLI(s, s.temp.buf, &s.temp.pos) != xzStreamEnd { + return xzDataError + } + s.blockHeader.uncompressed = s.vli + } else { + s.blockHeader.uncompressed = vliUnknown + } + // get total number of filters (1-4) + filterTotal := int(s.temp.buf[1]&0x03) + 1 + // slice to hold decoded filters + filterList := make([]struct { + id xzFilterID + props uint32 + }, filterTotal) + // decode the non-last filters which cannot be LZMA2 + for i := 0; i < filterTotal-1; i++ { + /* Valid Filter Flags always take at least two bytes. */ + if len(s.temp.buf)-s.temp.pos < 2 { + return xzDataError + } + s.temp.pos += 2 + switch id := xzFilterID(s.temp.buf[s.temp.pos-2]); id { + case idDelta: + // delta filter + if s.temp.buf[s.temp.pos-1] != 0x01 { + return xzOptionsError + } + /* Filter Properties contains distance - 1 */ + if len(s.temp.buf)-s.temp.pos < 1 { + return xzDataError + } + props := uint32(s.temp.buf[s.temp.pos]) + s.temp.pos++ + filterList[i] = struct { + id xzFilterID + props uint32 + }{id: id, props: props} + case idBCJX86, idBCJPowerPC, idBCJIA64, + idBCJARM, idBCJARMThumb, idBCJSPARC: + // bcj filter + var props uint32 + switch s.temp.buf[s.temp.pos-1] { + case 0x00: + props = 0 + case 0x04: + if len(s.temp.buf)-s.temp.pos < 4 { + return xzDataError + } + props = getLE32(s.temp.buf[s.temp.pos:]) + s.temp.pos += 4 + default: + return xzOptionsError + } + filterList[i] = struct { + id xzFilterID + props uint32 + }{id: id, props: props} + default: + return xzOptionsError + } + } + /* + * decode the last filter which must be LZMA2 + */ + if len(s.temp.buf)-s.temp.pos < 2 { + return xzDataError + } + /* Filter ID = LZMA2 */ + if xzFilterID(s.temp.buf[s.temp.pos]) != idLZMA2 { + return xzOptionsError + } + s.temp.pos++ + /* Size of Properties = 1-byte Filter Properties */ + if s.temp.buf[s.temp.pos] != 0x01 { + return xzOptionsError + } + s.temp.pos++ + /* Filter Properties contains LZMA2 dictionary size. */ + if len(s.temp.buf)-s.temp.pos < 1 { + return xzDataError + } + props := uint32(s.temp.buf[s.temp.pos]) + s.temp.pos++ + filterList[filterTotal-1] = struct { + id xzFilterID + props uint32 + }{id: idLZMA2, props: props} + /* + * Process the filter list and create s.chain, going from last + * filter (LZMA2) to first filter + * + * First, LZMA2. + */ + ret = xzDecLZMA2Reset(s.lzma2, byte(filterList[filterTotal-1].props)) + if ret != xzOK { + return ret + } + s.chain = func(b *xzBuf) xzRet { + return xzDecLZMA2Run(s.lzma2, b) + } + /* + * Now the non-last filters + */ + for i := filterTotal - 2; i >= 0; i-- { + switch id := filterList[i].id; id { + case idDelta: + // delta filter + var delta *xzDecDelta + if s.deltasUsed < len(s.deltas) { + delta = s.deltas[s.deltasUsed] + } else { + delta = xzDecDeltaCreate() + s.deltas = append(s.deltas, delta) + } + s.deltasUsed++ + ret = xzDecDeltaReset(delta, int(filterList[i].props)+1) + if ret != xzOK { + return ret + } + chain := s.chain + s.chain = func(b *xzBuf) xzRet { + return xzDecDeltaRun(delta, b, chain) + } + case idBCJX86, idBCJPowerPC, idBCJIA64, + idBCJARM, idBCJARMThumb, idBCJSPARC: + // bcj filter + var bcj *xzDecBCJ + if s.bcjsUsed < len(s.bcjs) { + bcj = s.bcjs[s.bcjsUsed] + } else { + bcj = xzDecBCJCreate() + s.bcjs = append(s.bcjs, bcj) + } + s.bcjsUsed++ + ret = xzDecBCJReset(bcj, id, int(filterList[i].props)) + if ret != xzOK { + return ret + } + chain := s.chain + s.chain = func(b *xzBuf) xzRet { + return xzDecBCJRun(bcj, b, chain) + } + } + } + /* The rest must be Header Padding. */ + for s.temp.pos < len(s.temp.buf) { + if s.temp.buf[s.temp.pos] != 0x00 { + return xzOptionsError + } + s.temp.pos++ + } + s.temp.pos = 0 + s.block.compressed = 0 + s.block.uncompressed = 0 + return xzOK +} + +func decMain(s *xzDec, b *xzBuf) xzRet { + var ret xzRet + /* + * Store the start position for the case when we are in the middle + * of the Index field. + */ + s.inStart = b.inPos + for { + switch s.sequence { + case seqStreamHeader: + /* + * Stream Header is copied to s.temp, and then + * decoded from there. This way if the caller + * gives us only little input at a time, we can + * still keep the Stream Header decoding code + * simple. Similar approach is used in many places + * in this file. + */ + if !fillTemp(s, b) { + return xzOK + } + /* + * If decStreamHeader returns + * xzUnsupportedCheck, it is still possible + * to continue decoding. Thus, update s.sequence + * before calling decStreamHeader. + */ + s.sequence = seqBlockStart + ret = decStreamHeader(s) + if ret != xzOK { + return ret + } + fallthrough + case seqBlockStart: + /* We need one byte of input to continue. */ + if b.inPos == len(b.in) { + return xzOK + } + /* See if this is the beginning of the Index field. */ + if b.in[b.inPos] == 0 { + s.inStart = b.inPos + b.inPos++ + s.sequence = seqIndex + break + } + /* + * Calculate the size of the Block Header and + * prepare to decode it. + */ + s.blockHeader.size = (int(b.in[b.inPos]) + 1) * 4 + s.temp.buf = s.temp.bufArray[:s.blockHeader.size] + s.temp.pos = 0 + s.sequence = seqBlockHeader + fallthrough + case seqBlockHeader: + if !fillTemp(s, b) { + return xzOK + } + ret = decBlockHeader(s) + if ret != xzOK { + return ret + } + s.sequence = seqBlockUncompress + fallthrough + case seqBlockUncompress: + ret = decBlock(s, b) + if ret != xzStreamEnd { + return ret + } + s.sequence = seqBlockPadding + fallthrough + case seqBlockPadding: + /* + * Size of Compressed Data + Block Padding + * must be a multiple of four. We don't need + * s->block.compressed for anything else + * anymore, so we use it here to test the size + * of the Block Padding field. + */ + for s.block.compressed&3 != 0 { + if b.inPos == len(b.in) { + return xzOK + } + if b.in[b.inPos] != 0 { + return xzDataError + } + b.inPos++ + s.block.compressed++ + } + s.sequence = seqBlockCheck + fallthrough + case seqBlockCheck: + switch s.CheckType { + case CheckCRC32, CheckCRC64, CheckSHA256: + ret = checkValidate(s, b) + if ret != xzStreamEnd { + return ret + } + default: + if !checkSkip(s, b) { + return xzOK + } + } + s.sequence = seqBlockStart + case seqIndex: + ret = decIndex(s, b) + if ret != xzStreamEnd { + return ret + } + s.sequence = seqIndexPadding + fallthrough + case seqIndexPadding: + for (s.index.size+vliType(b.inPos-s.inStart))&3 != 0 { + if b.inPos == len(b.in) { + indexUpdate(s, b) + return xzOK + } + if b.in[b.inPos] != 0 { + return xzDataError + } + b.inPos++ + } + /* Finish the CRC32 value and Index size. */ + indexUpdate(s, b) + /* Compare the hashes to validate the Index field. */ + if !bytes.Equal( + s.block.hash.sha256.Sum(nil), s.index.hash.sha256.Sum(nil)) { + return xzDataError + } + s.sequence = seqIndexCRC32 + fallthrough + case seqIndexCRC32: + ret = crcValidate(s, b) + if ret != xzStreamEnd { + return ret + } + s.temp.buf = s.temp.bufArray[:streamHeaderSize] + s.sequence = seqStreamFooter + fallthrough + case seqStreamFooter: + if !fillTemp(s, b) { + return xzOK + } + return decStreamFooter(s) + } + } + /* Never reached */ +} + +/** + * xzDecRun - Run the XZ decoder + * @s: Decoder state allocated using xzDecInit + * @b: Input and output buffers + * + * See xzRet for details of return values. + * + * xzDecRun is a wrapper for decMain to handle some special cases. + * + * We must return xzBufError when it seems clear that we are not + * going to make any progress anymore. This is to prevent the caller + * from calling us infinitely when the input file is truncated or + * otherwise corrupt. Since zlib-style API allows that the caller + * fills the input buffer only when the decoder doesn't produce any + * new output, we have to be careful to avoid returning xzBufError + * too easily: xzBufError is returned only after the second + * consecutive call to xzDecRun that makes no progress. + */ +func xzDecRun(s *xzDec, b *xzBuf) xzRet { + inStart := b.inPos + outStart := b.outPos + ret := decMain(s, b) + if ret == xzOK && inStart == b.inPos && outStart == b.outPos { + if s.allowBufError { + ret = xzBufError + } + s.allowBufError = true + } else { + s.allowBufError = false + } + return ret +} + +/** + * xzDecInit - Allocate and initialize a XZ decoder state + * @dictMax: Maximum size of the LZMA2 dictionary (history buffer) for + * decoding. LZMA2 dictionary is always 2^n bytes + * or 2^n + 2^(n-1) bytes (the latter sizes are less common + * in practice), so other values for dictMax don't make sense. + * + * dictMax specifies the maximum allowed dictionary size that xzDecRun + * may allocate once it has parsed the dictionary size from the stream + * headers. This way excessive allocations can be avoided while still + * limiting the maximum memory usage to a sane value to prevent running the + * system out of memory when decompressing streams from untrusted sources. + * + * xzDecInit returns a pointer to an xzDec, which is ready to be used with + * xzDecRun. + */ +func xzDecInit(dictMax uint32, header *Header) *xzDec { + s := new(xzDec) + s.crc32 = crc32.NewIEEE() + s.Header = header + s.block.hash.sha256 = sha256.New() + s.index.hash.sha256 = sha256.New() + s.lzma2 = xzDecLZMA2Create(dictMax) + xzDecReset(s) + return s +} + +/** + * xzDecReset - Reset an already allocated decoder state + * @s: Decoder state allocated using xzDecInit + * + * This function can be used to reset the decoder state without + * reallocating memory with xzDecInit. + */ +func xzDecReset(s *xzDec) { + s.sequence = seqStreamHeader + s.allowBufError = false + s.pos = 0 + s.crc32.Reset() + s.check = nil + s.CheckType = checkUnset + s.block.compressed = 0 + s.block.uncompressed = 0 + s.block.count = 0 + s.block.hash.unpadded = 0 + s.block.hash.uncompressed = 0 + s.block.hash.sha256.Reset() + s.index.sequence = seqIndexCount + s.index.size = 0 + s.index.count = 0 + s.index.hash.unpadded = 0 + s.index.hash.uncompressed = 0 + s.index.hash.sha256.Reset() + s.temp.pos = 0 + s.temp.buf = s.temp.bufArray[:streamHeaderSize] + s.chain = nil + s.bcjsUsed = 0 + s.deltasUsed = 0 +} diff --git a/vendor/github.com/therootcompany/xz/dec_util.go b/vendor/github.com/therootcompany/xz/dec_util.go new file mode 100644 index 00000000..c4227522 --- /dev/null +++ b/vendor/github.com/therootcompany/xz/dec_util.go @@ -0,0 +1,52 @@ +/* + * XZ decompressor utility functions + * + * Author: Michael Cross + * + * This file has been put into the public domain. + * You can do whatever you want with this file. + */ + +package xz + +func getLE32(buf []byte) uint32 { + return uint32(buf[0]) | + uint32(buf[1])<<8 | + uint32(buf[2])<<16 | + uint32(buf[3])<<24 +} + +func getBE32(buf []byte) uint32 { + return uint32(buf[0])<<24 | + uint32(buf[1])<<16 | + uint32(buf[2])<<8 | + uint32(buf[3]) +} + +func putLE32(val uint32, buf []byte) { + buf[0] = byte(val) + buf[1] = byte(val >> 8) + buf[2] = byte(val >> 16) + buf[3] = byte(val >> 24) + return +} + +func putBE32(val uint32, buf []byte) { + buf[0] = byte(val >> 24) + buf[1] = byte(val >> 16) + buf[2] = byte(val >> 8) + buf[3] = byte(val) + return +} + +func putLE64(val uint64, buf []byte) { + buf[0] = byte(val) + buf[1] = byte(val >> 8) + buf[2] = byte(val >> 16) + buf[3] = byte(val >> 24) + buf[4] = byte(val >> 32) + buf[5] = byte(val >> 40) + buf[6] = byte(val >> 48) + buf[7] = byte(val >> 56) + return +} diff --git a/vendor/github.com/therootcompany/xz/dec_xz.go b/vendor/github.com/therootcompany/xz/dec_xz.go new file mode 100644 index 00000000..1b18a838 --- /dev/null +++ b/vendor/github.com/therootcompany/xz/dec_xz.go @@ -0,0 +1,124 @@ +/* + * XZ decompressor + * + * Authors: Lasse Collin + * Igor Pavlov + * + * Translation to Go: Michael Cross + * + * This file has been put into the public domain. + * You can do whatever you want with this file. + */ + +package xz + +/* from linux/include/linux/xz.h **************************************/ + +/** + * xzRet - Return codes + * @xzOK: Everything is OK so far. More input or more + * output space is required to continue. + * @xzStreamEnd: Operation finished successfully. + * @xzUnSupportedCheck: Integrity check type is not supported. Decoding + * is still possible by simply calling xzDecRun + * again. + * @xzMemlimitError: A bigger LZMA2 dictionary would be needed than + * allowed by the dictMax argument given to + * xzDecInit. + * @xzFormatError: File format was not recognized (wrong magic + * bytes). + * @xzOptionsError: This implementation doesn't support the requested + * compression options. In the decoder this means + * that the header CRC32 matches, but the header + * itself specifies something that we don't support. + * @xzDataError: Compressed data is corrupt. + * @xzBufError: Cannot make any progress. + * + * xzBufError is returned when two consecutive calls to XZ code cannot + * consume any input and cannot produce any new output. This happens + * when there is no new input available, or the output buffer is full + * while at least one output byte is still pending. Assuming your code + * is not buggy, you can get this error only when decoding a + * compressed stream that is truncated or otherwise corrupt. + */ +type xzRet int + +const ( + xzOK xzRet = iota + xzStreamEnd + xzUnsupportedCheck + xzMemlimitError + xzFormatError + xzOptionsError + xzDataError + xzBufError +) + +/** + * xzBuf - Passing input and output buffers to XZ code + * @in: Input buffer. + * @inPos: Current position in the input buffer. This must not exceed + * input buffer size. + * @out: Output buffer. + * @outPos: Current position in the output buffer. This must not exceed + * output buffer size. + * + * Only the contents of the output buffer from out[outPos] onward, and + * the variables inPos and outPos are modified by the XZ code. + */ +type xzBuf struct { + in []byte + inPos int + out []byte + outPos int +} + +/* All XZ filter IDs */ +type xzFilterID int64 + +const ( + idDelta xzFilterID = 0x03 + idBCJX86 xzFilterID = 0x04 + idBCJPowerPC xzFilterID = 0x05 + idBCJIA64 xzFilterID = 0x06 + idBCJARM xzFilterID = 0x07 + idBCJARMThumb xzFilterID = 0x08 + idBCJSPARC xzFilterID = 0x09 + idLZMA2 xzFilterID = 0x21 +) + +// CheckID is the type of the data integrity check in an XZ stream +// calculated from the uncompressed data. +type CheckID int + +func (id CheckID) String() string { + switch id { + case CheckNone: + return "None" + case CheckCRC32: + return "CRC32" + case CheckCRC64: + return "CRC64" + case CheckSHA256: + return "SHA256" + default: + return "Unknown" + } +} + +const ( + CheckNone CheckID = 0x00 + CheckCRC32 CheckID = 0x01 + CheckCRC64 CheckID = 0x04 + CheckSHA256 CheckID = 0x0A + checkMax CheckID = 0x0F + checkUnset CheckID = -1 +) + +// An XZ stream contains a stream header which holds information about +// the stream. That information is exposed as fields of the +// Reader. Currently it contains only the stream's data integrity +// check type. +type Header struct { + CheckType CheckID // type of the stream's data integrity check +} diff --git a/vendor/github.com/therootcompany/xz/doc.go b/vendor/github.com/therootcompany/xz/doc.go new file mode 100644 index 00000000..f8c62e62 --- /dev/null +++ b/vendor/github.com/therootcompany/xz/doc.go @@ -0,0 +1,35 @@ +// Package xz implements XZ decompression natively in Go. +// +// Usage +// +// For ease of use, this package is designed to have a similar API to +// compress/gzip. See the examples for further details. +// +// Implementation +// +// This package is a translation from C to Go of XZ Embedded +// (http://tukaani.org/xz/embedded.html) with enhancements made so as +// to implement all mandatory and optional parts of the XZ file format +// specification v1.0.4. It supports all filters and block check +// types, supports multiple streams, and performs index verification +// using SHA-256 as recommended by the specification. +// +// Speed +// +// On the author's Intel Ivybridge i5, decompression speed is about +// half that of the standard XZ Utils (tested with a recent linux +// kernel tarball). +// +// Thanks +// +// Thanks are due to Lasse Collin and Igor Pavlov, the authors of XZ +// Embedded, on whose code package xz is based. It would not exist +// without their decision to allow others to modify and reuse their +// code. +// +// Bug reports +// +// For bug reports relating to this package please contact the author +// through https://github.com/xi2/xz/issues, and not the authors of XZ +// Embedded. +package xz diff --git a/vendor/github.com/therootcompany/xz/reader.go b/vendor/github.com/therootcompany/xz/reader.go new file mode 100644 index 00000000..e321d755 --- /dev/null +++ b/vendor/github.com/therootcompany/xz/reader.go @@ -0,0 +1,256 @@ +/* + * Package xz Go Reader API + * + * Author: Michael Cross + * + * This file has been put into the public domain. + * You can do whatever you want with this file. + */ + +package xz + +import ( + "errors" + "io" +) + +// Package specific errors. +var ( + ErrUnsupportedCheck = errors.New("xz: integrity check type not supported") + ErrMemlimit = errors.New("xz: LZMA2 dictionary size exceeds max") + ErrFormat = errors.New("xz: file format not recognized") + ErrOptions = errors.New("xz: compression options not supported") + ErrData = errors.New("xz: data is corrupt") + ErrBuf = errors.New("xz: data is truncated or corrupt") +) + +// DefaultDictMax is the default maximum dictionary size in bytes used +// by the decoder. This value is sufficient to decompress files +// created with XZ Utils "xz -9". +const DefaultDictMax = 1 << 26 // 64 MiB + +// inBufSize is the input buffer size used by the decoder. +const inBufSize = 1 << 13 // 8 KiB + +// A Reader is an io.Reader that can be used to retrieve uncompressed +// data from an XZ file. +// +// In general, an XZ file can be a concatenation of other XZ +// files. Reads from the Reader return the concatenation of the +// uncompressed data of each. +type Reader struct { + Header + r io.Reader // the wrapped io.Reader + multistream bool // true if reader is in multistream mode + rEOF bool // true after io.EOF received on r + dEOF bool // true after decoder has completed + padding int // bytes of stream padding read (or -1) + in [inBufSize]byte // backing array for buf.in + buf *xzBuf // decoder input/output buffers + dec *xzDec // decoder state + err error // the result of the last decoder call +} + +// NewReader creates a new Reader reading from r. The decompressor +// will use an LZMA2 dictionary size up to dictMax bytes in +// size. Passing a value of zero sets dictMax to DefaultDictMax. If +// an individual XZ stream requires a dictionary size greater than +// dictMax in order to decompress, Read will return ErrMemlimit. +// +// If NewReader is passed a value of nil for r then a Reader is +// created such that all read attempts will return io.EOF. This is +// useful if you just want to allocate memory for a Reader which will +// later be initialized with Reset. +// +// Due to internal buffering, the Reader may read more data than +// necessary from r. +func NewReader(r io.Reader, dictMax uint32) (*Reader, error) { + if dictMax == 0 { + dictMax = DefaultDictMax + } + z := &Reader{ + r: r, + multistream: true, + padding: -1, + buf: &xzBuf{}, + } + if r == nil { + z.rEOF, z.dEOF = true, true + } + z.dec = xzDecInit(dictMax, &z.Header) + var err error + if r != nil { + _, err = z.Read(nil) // read stream header + } + return z, err +} + +// decode is a wrapper around xzDecRun that additionally handles +// stream padding. It treats the padding as a kind of stream that +// decodes to nothing. +// +// When decoding padding, z.padding >= 0 +// When decoding a real stream, z.padding == -1 +func (z *Reader) decode() (ret xzRet) { + if z.padding >= 0 { + // read all padding in input buffer + for z.buf.inPos < len(z.buf.in) && + z.buf.in[z.buf.inPos] == 0 { + z.buf.inPos++ + z.padding++ + } + switch { + case z.buf.inPos == len(z.buf.in) && z.rEOF: + // case: out of padding. no more input data available + if z.padding%4 != 0 { + ret = xzDataError + } else { + ret = xzStreamEnd + } + case z.buf.inPos == len(z.buf.in): + // case: read more padding next loop iteration + ret = xzOK + default: + // case: out of padding. more input data available + if z.padding%4 != 0 { + ret = xzDataError + } else { + xzDecReset(z.dec) + ret = xzStreamEnd + } + } + } else { + ret = xzDecRun(z.dec, z.buf) + } + return +} + +func (z *Reader) Read(p []byte) (n int, err error) { + // restore err + err = z.err + // set decoder output buffer to p + z.buf.out = p + z.buf.outPos = 0 + for { + // update n + n = z.buf.outPos + // if last call to decoder ended with an error, return that error + if err != nil { + break + } + // if decoder has finished, return with err == io.EOF + if z.dEOF { + err = io.EOF + break + } + // if p full, return with err == nil, unless we have not yet + // read the stream header with Read(nil) + if n == len(p) && z.CheckType != checkUnset { + break + } + // if needed, read more data from z.r + if z.buf.inPos == len(z.buf.in) && !z.rEOF { + rn, e := z.r.Read(z.in[:]) + if e != nil && e != io.EOF { + // read error + err = e + break + } + if e == io.EOF { + z.rEOF = true + } + // set new input buffer in z.buf + z.buf.in = z.in[:rn] + z.buf.inPos = 0 + } + // decode more data + ret := z.decode() + switch ret { + case xzOK: + // no action needed + case xzStreamEnd: + if z.padding >= 0 { + z.padding = -1 + if !z.multistream || z.rEOF { + z.dEOF = true + } + } else { + z.padding = 0 + } + case xzUnsupportedCheck: + err = ErrUnsupportedCheck + case xzMemlimitError: + err = ErrMemlimit + case xzFormatError: + err = ErrFormat + case xzOptionsError: + err = ErrOptions + case xzDataError: + err = ErrData + case xzBufError: + err = ErrBuf + } + // save err + z.err = err + } + return +} + +// Multistream controls whether the reader is operating in multistream +// mode. +// +// If enabled (the default), the Reader expects the input to be a +// sequence of XZ streams, possibly interspersed with stream padding, +// which it reads one after another. The effect is that the +// concatenation of a sequence of XZ streams or XZ files is +// treated as equivalent to the compressed result of the concatenation +// of the sequence. This is standard behaviour for XZ readers. +// +// Calling Multistream(false) disables this behaviour; disabling the +// behaviour can be useful when reading file formats that distinguish +// individual XZ streams. In this mode, when the Reader reaches the +// end of the stream, Read returns io.EOF. To start the next stream, +// call z.Reset(nil) followed by z.Multistream(false). If there is no +// next stream, z.Reset(nil) will return io.EOF. +func (z *Reader) Multistream(ok bool) { + z.multistream = ok +} + +// Reset, for non-nil values of io.Reader r, discards the Reader z's +// state and makes it equivalent to the result of its original state +// from NewReader, but reading from r instead. This permits reusing a +// Reader rather than allocating a new one. +// +// If you wish to leave r unchanged use z.Reset(nil). This keeps r +// unchanged and ensures internal buffering is preserved. If the +// Reader was at the end of a stream it is then ready to read any +// follow on streams. If there are no follow on streams z.Reset(nil) +// returns io.EOF. If the Reader was not at the end of a stream then +// z.Reset(nil) does nothing. +func (z *Reader) Reset(r io.Reader) error { + switch { + case r == nil: + z.multistream = true + if !z.dEOF { + return nil + } + if z.rEOF { + return io.EOF + } + z.dEOF = false + _, err := z.Read(nil) // read stream header + return err + default: + z.r = r + z.multistream = true + z.rEOF = false + z.dEOF = false + z.padding = -1 + z.buf.in = nil + z.buf.inPos = 0 + xzDecReset(z.dec) + z.err = nil + _, err := z.Read(nil) // read stream header + return err + } +} diff --git a/vendor/github.com/ulikunitz/xz/.gitignore b/vendor/github.com/ulikunitz/xz/.gitignore deleted file mode 100644 index eb3d5f51..00000000 --- a/vendor/github.com/ulikunitz/xz/.gitignore +++ /dev/null @@ -1,28 +0,0 @@ -# .gitignore - -TODO.html -README.html - -lzma/writer.txt -lzma/reader.txt - -cmd/gxz/gxz -cmd/xb/xb - -# test executables -*.test - -# profile files -*.out - -# vim swap file -.*.swp - -# executables on windows -*.exe - -# default compression test file -enwik8* - -# file generated by example -example.xz \ No newline at end of file diff --git a/vendor/github.com/ulikunitz/xz/LICENSE b/vendor/github.com/ulikunitz/xz/LICENSE deleted file mode 100644 index 8a7f0877..00000000 --- a/vendor/github.com/ulikunitz/xz/LICENSE +++ /dev/null @@ -1,26 +0,0 @@ -Copyright (c) 2014-2022 Ulrich Kunitz -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - -* Redistributions of source code must retain the above copyright notice, this - list of conditions and the following disclaimer. - -* Redistributions in binary form must reproduce the above copyright notice, - this list of conditions and the following disclaimer in the documentation - and/or other materials provided with the distribution. - -* My name, Ulrich Kunitz, may not be used to endorse or promote products - derived from this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/vendor/github.com/ulikunitz/xz/README.md b/vendor/github.com/ulikunitz/xz/README.md deleted file mode 100644 index 56d49275..00000000 --- a/vendor/github.com/ulikunitz/xz/README.md +++ /dev/null @@ -1,88 +0,0 @@ -# Package xz - -This Go language package supports the reading and writing of xz -compressed streams. It includes also a gxz command for compressing and -decompressing data. The package is completely written in Go and doesn't -have any dependency on any C code. - -The package is currently under development. There might be bugs and APIs -are not considered stable. At this time the package cannot compete with -the xz tool regarding compression speed and size. The algorithms there -have been developed over a long time and are highly optimized. However -there are a number of improvements planned and I'm very optimistic about -parallel compression and decompression. Stay tuned! - -## Using the API - -The following example program shows how to use the API. - -```go -package main - -import ( - "bytes" - "io" - "log" - "os" - - "github.com/ulikunitz/xz" -) - -func main() { - const text = "The quick brown fox jumps over the lazy dog.\n" - var buf bytes.Buffer - // compress text - w, err := xz.NewWriter(&buf) - if err != nil { - log.Fatalf("xz.NewWriter error %s", err) - } - if _, err := io.WriteString(w, text); err != nil { - log.Fatalf("WriteString error %s", err) - } - if err := w.Close(); err != nil { - log.Fatalf("w.Close error %s", err) - } - // decompress buffer and write output to stdout - r, err := xz.NewReader(&buf) - if err != nil { - log.Fatalf("NewReader error %s", err) - } - if _, err = io.Copy(os.Stdout, r); err != nil { - log.Fatalf("io.Copy error %s", err) - } -} -``` - -## Documentation - -You can find the full documentation at [pkg.go.dev](https://pkg.go.dev/github.com/ulikunitz/xz). - -## Using the gxz compression tool - -The package includes a gxz command line utility for compression and -decompression. - -Use following command for installation: - - $ go get github.com/ulikunitz/xz/cmd/gxz - -To test it call the following command. - - $ gxz bigfile - -After some time a much smaller file bigfile.xz will replace bigfile. -To decompress it use the following command. - - $ gxz -d bigfile.xz - -## Security & Vulnerabilities - -The security policy is documented in [SECURITY.md](SECURITY.md). - -The software is not affected by the supply chain attack on the original xz -implementation, [CVE-2024-3094](https://nvd.nist.gov/vuln/detail/CVE-2024-3094). -This implementation doesn't share any files with the original xz implementation -and no patches or pull requests are accepted without a review. - -All security advisories for this project are published under -[github.com/ulikunitz/xz/security/advisories](https://github.com/ulikunitz/xz/security/advisories?state=published). diff --git a/vendor/github.com/ulikunitz/xz/SECURITY.md b/vendor/github.com/ulikunitz/xz/SECURITY.md deleted file mode 100644 index 1bdc8887..00000000 --- a/vendor/github.com/ulikunitz/xz/SECURITY.md +++ /dev/null @@ -1,19 +0,0 @@ -# Security Policy - -## Supported Versions - -Currently the last minor version v0.5.x is supported. - -## Reporting a Vulnerability - -You can privately report a vulnerability following this -[procedure](https://docs.github.com/en/code-security/security-advisories/guidance-on-reporting-and-writing-information-about-vulnerabilities/privately-reporting-a-security-vulnerability#privately-reporting-a-security-vulnerability). -Alternatively you can create a Github issue at -. - -In both cases expect a response in at least 7 days. - -## Security Advisories - -All security advisories for this project are published under -[github.com/ulikunitz/xz/security/advisories](https://github.com/ulikunitz/xz/security/advisories?state=published). diff --git a/vendor/github.com/ulikunitz/xz/TODO.md b/vendor/github.com/ulikunitz/xz/TODO.md deleted file mode 100644 index 5496e79a..00000000 --- a/vendor/github.com/ulikunitz/xz/TODO.md +++ /dev/null @@ -1,395 +0,0 @@ -# TODO list - -## Release v0.6 - -1. Review encoder and check for lzma improvements under xz. -2. Fix binary tree matcher. -3. Compare compression ratio with xz tool using comparable parameters and optimize parameters -4. rename operation action and make it a simple type of size 8 -5. make maxMatches, wordSize parameters -6. stop searching after a certain length is found (parameter sweetLen) - -## Release v0.7 - -1. Optimize code -2. Do statistical analysis to get linear presets. -3. Test sync.Pool compatability for xz and lzma Writer and Reader -4. Fuzz optimized code. - -## Release v0.8 - -1. Support parallel go routines for writing and reading xz files. -2. Support a ReaderAt interface for xz files with small block sizes. -3. Improve compatibility between gxz and xz -4. Provide manual page for gxz - -## Release v0.9 - -1. Improve documentation -2. Fuzz again - -## Release v1.0 - -1. Full functioning gxz -2. Add godoc URL to README.md (godoc.org) -3. Resolve all issues. -4. Define release candidates. -5. Public announcement. - -## Package lzma - -### v0.6 - -* Rewrite Encoder into a simple greedy one-op-at-a-time encoder including - * simple scan at the dictionary head for the same byte - * use the killer byte (requiring matches to get longer, the first test should be the byte that would make the match longer) - -## Optimizations - -* There may be a lot of false sharing in lzma. State; check whether this can be improved by reorganizing the internal structure of it. - -* Check whether batching encoding and decoding improves speed. - -### DAG optimizations - -* Use full buffer to create minimal bit-length above range encoder. -* Might be too slow (see v0.4) - -### Different match finders - -* hashes with 2, 3 characters additional to 4 characters -* binary trees with 2-7 characters (uint64 as key, use uint32 as - - pointers into a an array) - -* rb-trees with 2-7 characters (uint64 as key, use uint32 as pointers - - into an array with bit-steeling for the colors) - -## Release Procedure - -* execute goch -l for all packages; probably with lower param like 0.5. -* check orthography with gospell -* Write release notes in doc/relnotes. -* Update README.md -* xb copyright . in xz directory to ensure all new files have Copyright header -* `VERSION= go generate github.com/ulikunitz/xz/...` to update version files -* Execute [test for Linux/amd64, Linux/x86 and Windows/amd64](.github/workflows/go-test-platforms.yml). -* Update TODO.md - write short log entry -* `git checkout master && git merge dev` -* `git tag -a ` -* `git push` - -## Log - -## 2026-09-19 - -Includes a fix for lzma.breader handling 0, nil Reader returns correctly. - -## 2026-07-20 - -New release v0.5.16 to support other operating systems like Illumos using a -simple fallback function for IsTerminal to avoid adding a dependency. - -## 2025-08-28 - -Release v0.5.14 addresses the security vulnerability CVE-2025-58058. If you put -bytes in from of a LZMA stream, the header might not be read correctly and -memory for the dictionary buffer allocated. I have implemented mitigations for -the problem. - -### 2025-08-20 - -Release v0.5.13 addressed issue #61 regarding handling of multiple WriteClosers -together. So I added a new package xio with a WriteCloserStack to address the -issue. - -### 2024-04-03 - -Release v0.5.12 updates README.md and SECURITY.md to address the supply chain -attack on the original xz implementation. - -### 2022-12-12 - -Matt Dantay (@bodgit) reported an issue with the LZMA reader. The implementation -returned an error if the dictionary size was less than 4096 byte, but the -recommendation stated the actual used window size should be set to 4096 byte in -that case. It actually was the pull request -[#52](https://github.com/ulikunitz/xz/pull/52). The new patch v0.5.11 will fix -it. - -### 2021-02-02 - -Mituo Heijo has fuzzed xz and found a bug in the function readIndexBody. The -function allocated a slice of records immediately after reading the value -without further checks. Since the number has been too large the make function -did panic. The fix is to check the number against the expected number of records -before allocating the records. - -### 2020-12-17 - -Release v0.5.9 fixes warnings, a typo and adds SECURITY.md. - -One fix is interesting. - -```go -const ( - a byte = 0x1 - b = 0x2 -) -``` - -The constants a and b don't have the same type. Correct is - -```go -const ( - a byte = 0x1 - b byte = 0x2 -) -``` - -### 2020-08-19 - -Release v0.5.8 fixes issue -[issue #35](https://github.com/ulikunitz/xz/issues/35). - -### 2020-02-24 - -Release v0.5.7 supports the check-ID None and fixes -[issue #27](https://github.com/ulikunitz/xz/issues/27). - -### 2019-02-20 - -Release v0.5.6 supports the go.mod file. - -### 2018-10-28 - -Release v0.5.5 fixes issues #19 observing ErrLimit outputs. - -### 2017-06-05 - -Release v0.5.4 fixes issues #15 of another problem with the padding size -check for the xz block header. I removed the check completely. - -### 2017-02-15 - -Release v0.5.3 fixes issue #12 regarding the decompression of an empty -XZ stream. Many thanks to Tomasz Kłak, who reported the issue. - -### 2016-12-02 - -Release v0.5.2 became necessary to allow the decoding of xz files with -4-byte padding in the block header. Many thanks to Greg, who reported -the issue. - -### 2016-07-23 - -Release v0.5.1 became necessary to fix problems with 32-bit platforms. -Many thanks to Bruno Brigas, who reported the issue. - -### 2016-07-04 - -Release v0.5 provides improvements to the compressor and provides support for -the decompression of xz files with multiple xz streams. - -### 2016-01-31 - -Another compression rate increase by checking the byte at length of the -best match first, before checking the whole prefix. This makes the -compressor even faster. We have now a large time budget to beat the -compression ratio of the xz tool. For enwik8 we have now over 40 seconds -to reduce the compressed file size for another 7 MiB. - -### 2016-01-30 - -I simplified the encoder. Speed and compression rate increased -dramatically. A high compression rate affects also the decompression -speed. The approach with the buffer and optimizing for operation -compression rate has not been successful. Going for the maximum length -appears to be the best approach. - -### 2016-01-28 - -The release v0.4 is ready. It provides a working xz implementation, -which is rather slow, but works and is interoperable with the xz tool. -It is an important milestone. - -### 2016-01-10 - -I have the first working implementation of an xz reader and writer. I'm -happy about reaching this milestone. - -### 2015-12-02 - -I'm now ready to implement xz because, I have a working LZMA2 -implementation. I decided today that v0.4 will use the slow encoder -using the operations buffer to be able to go back, if I intend to do so. - -### 2015-10-21 - -I have restarted the work on the library. While trying to implement -LZMA2, I discovered that I need to resimplify the encoder and decoder -functions. The option approach is too complicated. Using a limited byte -writer and not caring for written bytes at all and not to try to handle -uncompressed data simplifies the LZMA encoder and decoder much. -Processing uncompressed data and handling limits is a feature of the -LZMA2 format not of LZMA. - -I learned an interesting method from the LZO format. If the last copy is -too far away they are moving the head one 2 bytes and not 1 byte to -reduce processing times. - -### 2015-08-26 - -I have now reimplemented the lzma package. The code is reasonably fast, -but can still be optimized. The next step is to implement LZMA2 and then -xz. - -### 2015-07-05 - -Created release v0.3. The version is the foundation for a full xz -implementation that is the target of v0.4. - -### 2015-06-11 - -The gflag package has been developed because I couldn't use flag and -pflag for a fully compatible support of gzip's and lzma's options. It -seems to work now quite nicely. - -### 2015-06-05 - -The overflow issue was interesting to research, however Henry S. Warren -Jr. Hacker's Delight book was very helpful as usual and had the issue -explained perfectly. Fefe's information on his website was based on the -C FAQ and quite bad, because it didn't address the issue of -MININT == -MININT. - -### 2015-06-04 - -It has been a productive day. I improved the interface of lzma. Reader -and lzma. Writer and fixed the error handling. - -### 2015-06-01 - -By computing the bit length of the LZMA operations I was able to -improve the greedy algorithm implementation. By using an 8 MByte buffer -the compression rate was not as good as for xz but already better then -gzip default. - -Compression is currently slow, but this is something we will be able to -improve over time. - -### 2015-05-26 - -Checked the license of ogier/pflag. The binary lzmago binary should -include the license terms for the pflag library. - -I added the endorsement clause as used by Google for the Go sources the -LICENSE file. - -### 2015-05-22 - -The package lzb contains now the basic implementation for creating or -reading LZMA byte streams. It allows the support for the implementation -of the DAG-shortest-path algorithm for the compression function. - -### 2015-04-23 - -Completed yesterday the lzbase classes. I'm a little bit concerned that -using the components may require too much code, but on the other hand -there is a lot of flexibility. - -### 2015-04-22 - -Implemented Reader and Writer during the Bayern game against Porto. The -second half gave me enough time. - -### 2015-04-21 - -While showering today morning I discovered that the design for OpEncoder -and OpDecoder doesn't work, because encoding/decoding might depend on -the current status of the dictionary. This is not exactly the right way -to start the day. - -Therefore we need to keep the Reader and Writer design. This time around -we simplify it by ignoring size limits. These can be added by wrappers -around the Reader and Writer interfaces. The Parameters type isn't -needed anymore. - -However I will implement a ReaderState and WriterState type to use -static typing to ensure the right State object is combined with the -right lzbase. Reader and lzbase. Writer. - -As a start I have implemented ReaderState and WriterState to ensure -that the state for reading is only used by readers and WriterState only -used by Writers. - -### 2015-04-20 - -Today I implemented the OpDecoder and tested OpEncoder and OpDecoder. - -### 2015-04-08 - -Came up with a new simplified design for lzbase. I implemented already -the type State that replaces OpCodec. - -### 2015-04-06 - -The new lzma package is now fully usable and lzmago is using it now. The -old lzma package has been completely removed. - -### 2015-04-05 - -Implemented lzma. Reader and tested it. - -### 2015-04-04 - -Implemented baseReader by adapting code form lzma. Reader. - -### 2015-04-03 - -The opCodec has been copied yesterday to lzma2. opCodec has a high -number of dependencies on other files in lzma2. Therefore I had to copy -almost all files from lzma. - -### 2015-03-31 - -Removed only a TODO item. - -However in Francesco Campoy's presentation "Go for Javaneros -(Javaïstes?)" is the the idea that using an embedded field E, all the -methods of E will be defined on T. If E is an interface T satisfies E. - - - -I have never used this, but it seems to be a cool idea. - -### 2015-03-30 - -Finished the type writerDict and wrote a simple test. - -### 2015-03-25 - -I started to implement the writerDict. - -### 2015-03-24 - -After thinking long about the LZMA2 code and several false starts, I -have now a plan to create a self-sufficient lzma2 package that supports -the classic LZMA format as well as LZMA2. The core idea is to support a -baseReader and baseWriter type that support the basic LZMA stream -without any headers. Both types must support the reuse of dictionaries -and the opCodec. - -### 2015-01-10 - -1. Implemented simple lzmago tool -2. Tested tool against large 4.4G file - * compression worked correctly; tested decompression with lzma - * decompression hits a full buffer condition -3. Fixed a bug in the compressor and wrote a test for it -4. Executed full cycle for 4.4 GB file; performance can be improved ;-) - -### 2015-01-11 - -* Release v0.2 because of the working LZMA encoder and decoder diff --git a/vendor/github.com/ulikunitz/xz/bits.go b/vendor/github.com/ulikunitz/xz/bits.go deleted file mode 100644 index b30f1ec9..00000000 --- a/vendor/github.com/ulikunitz/xz/bits.go +++ /dev/null @@ -1,79 +0,0 @@ -// Copyright 2014-2022 Ulrich Kunitz. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package xz - -import ( - "errors" - "io" -) - -// putUint32LE puts the little-endian representation of x into the first -// four bytes of p. -func putUint32LE(p []byte, x uint32) { - p[0] = byte(x) - p[1] = byte(x >> 8) - p[2] = byte(x >> 16) - p[3] = byte(x >> 24) -} - -// putUint64LE puts the little-endian representation of x into the first -// eight bytes of p. -func putUint64LE(p []byte, x uint64) { - p[0] = byte(x) - p[1] = byte(x >> 8) - p[2] = byte(x >> 16) - p[3] = byte(x >> 24) - p[4] = byte(x >> 32) - p[5] = byte(x >> 40) - p[6] = byte(x >> 48) - p[7] = byte(x >> 56) -} - -// uint32LE converts a little endian representation to an uint32 value. -func uint32LE(p []byte) uint32 { - return uint32(p[0]) | uint32(p[1])<<8 | uint32(p[2])<<16 | - uint32(p[3])<<24 -} - -// putUvarint puts a uvarint representation of x into the byte slice. -func putUvarint(p []byte, x uint64) int { - i := 0 - for x >= 0x80 { - p[i] = byte(x) | 0x80 - x >>= 7 - i++ - } - p[i] = byte(x) - return i + 1 -} - -// errOverflow indicates an overflow of the 64-bit unsigned integer. -var errOverflowU64 = errors.New("xz: uvarint overflows 64-bit unsigned integer") - -// readUvarint reads a uvarint from the given byte reader. -func readUvarint(r io.ByteReader) (x uint64, n int, err error) { - const maxUvarintLen = 10 - - var s uint - i := 0 - for { - b, err := r.ReadByte() - if err != nil { - return x, i, err - } - i++ - if i > maxUvarintLen { - return x, i, errOverflowU64 - } - if b < 0x80 { - if i == maxUvarintLen && b > 1 { - return x, i, errOverflowU64 - } - return x | uint64(b)< 0 { - k = 4 - k - } - return k -} - -/*** Header ***/ - -// headerMagic stores the magic bytes for the header -var headerMagic = []byte{0xfd, '7', 'z', 'X', 'Z', 0x00} - -// HeaderLen provides the length of the xz file header. -const HeaderLen = 12 - -// Constants for the checksum methods supported by xz. -const ( - None byte = 0x0 - CRC32 byte = 0x1 - CRC64 byte = 0x4 - SHA256 byte = 0xa -) - -// errInvalidFlags indicates that flags are invalid. -var errInvalidFlags = errors.New("xz: invalid flags") - -// verifyFlags returns the error errInvalidFlags if the value is -// invalid. -func verifyFlags(flags byte) error { - switch flags { - case None, CRC32, CRC64, SHA256: - return nil - default: - return errInvalidFlags - } -} - -// flagstrings maps flag values to strings. -var flagstrings = map[byte]string{ - None: "None", - CRC32: "CRC-32", - CRC64: "CRC-64", - SHA256: "SHA-256", -} - -// flagString returns the string representation for the given flags. -func flagString(flags byte) string { - s, ok := flagstrings[flags] - if !ok { - return "invalid" - } - return s -} - -// newHashFunc returns a function that creates hash instances for the -// hash method encoded in flags. -func newHashFunc(flags byte) (newHash func() hash.Hash, err error) { - switch flags { - case None: - newHash = newNoneHash - case CRC32: - newHash = newCRC32 - case CRC64: - newHash = newCRC64 - case SHA256: - newHash = sha256.New - default: - err = errInvalidFlags - } - return -} - -// header provides the actual content of the xz file header: the flags. -type header struct { - flags byte -} - -// Errors returned by readHeader. -var errHeaderMagic = errors.New("xz: invalid header magic bytes") - -// ValidHeader checks whether data is a correct xz file header. The -// length of data must be HeaderLen. -func ValidHeader(data []byte) bool { - var h header - err := h.UnmarshalBinary(data) - return err == nil -} - -// String returns a string representation of the flags. -func (h header) String() string { - return flagString(h.flags) -} - -// UnmarshalBinary reads header from the provided data slice. -func (h *header) UnmarshalBinary(data []byte) error { - // header length - if len(data) != HeaderLen { - return errors.New("xz: wrong file header length") - } - - // magic header - if !bytes.Equal(headerMagic, data[:6]) { - return errHeaderMagic - } - - // checksum - crc := crc32.NewIEEE() - crc.Write(data[6:8]) - if uint32LE(data[8:]) != crc.Sum32() { - return errors.New("xz: invalid checksum for file header") - } - - // stream flags - if data[6] != 0 { - return errInvalidFlags - } - flags := data[7] - if err := verifyFlags(flags); err != nil { - return err - } - - h.flags = flags - return nil -} - -// MarshalBinary generates the xz file header. -func (h *header) MarshalBinary() (data []byte, err error) { - if err = verifyFlags(h.flags); err != nil { - return nil, err - } - - data = make([]byte, 12) - copy(data, headerMagic) - data[7] = h.flags - - crc := crc32.NewIEEE() - crc.Write(data[6:8]) - putUint32LE(data[8:], crc.Sum32()) - - return data, nil -} - -/*** Footer ***/ - -// footerLen defines the length of the footer. -const footerLen = 12 - -// footerMagic contains the footer magic bytes. -var footerMagic = []byte{'Y', 'Z'} - -// footer represents the content of the xz file footer. -type footer struct { - indexSize int64 - flags byte -} - -// String prints a string representation of the footer structure. -func (f footer) String() string { - return fmt.Sprintf("%s index size %d", flagString(f.flags), f.indexSize) -} - -// Minimum and maximum for the size of the index (backward size). -const ( - minIndexSize = 4 - maxIndexSize = (1 << 32) * 4 -) - -// MarshalBinary converts footer values into an xz file footer. Note -// that the footer value is checked for correctness. -func (f *footer) MarshalBinary() (data []byte, err error) { - if err = verifyFlags(f.flags); err != nil { - return nil, err - } - if !(minIndexSize <= f.indexSize && f.indexSize <= maxIndexSize) { - return nil, errors.New("xz: index size out of range") - } - if f.indexSize%4 != 0 { - return nil, errors.New( - "xz: index size not aligned to four bytes") - } - - data = make([]byte, footerLen) - - // backward size (index size) - s := (f.indexSize / 4) - 1 - putUint32LE(data[4:], uint32(s)) - // flags - data[9] = f.flags - // footer magic - copy(data[10:], footerMagic) - - // CRC-32 - crc := crc32.NewIEEE() - crc.Write(data[4:10]) - putUint32LE(data, crc.Sum32()) - - return data, nil -} - -// UnmarshalBinary sets the footer value by unmarshalling an xz file -// footer. -func (f *footer) UnmarshalBinary(data []byte) error { - if len(data) != footerLen { - return errors.New("xz: wrong footer length") - } - - // magic bytes - if !bytes.Equal(data[10:], footerMagic) { - return errors.New("xz: footer magic invalid") - } - - // CRC-32 - crc := crc32.NewIEEE() - crc.Write(data[4:10]) - if uint32LE(data) != crc.Sum32() { - return errors.New("xz: footer checksum error") - } - - var g footer - // backward size (index size) - g.indexSize = (int64(uint32LE(data[4:])) + 1) * 4 - - // flags - if data[8] != 0 { - return errInvalidFlags - } - g.flags = data[9] - if err := verifyFlags(g.flags); err != nil { - return err - } - - *f = g - return nil -} - -/*** Block Header ***/ - -// blockHeader represents the content of an xz block header. -type blockHeader struct { - compressedSize int64 - uncompressedSize int64 - filters []filter -} - -// String converts the block header into a string. -func (h blockHeader) String() string { - var buf bytes.Buffer - first := true - if h.compressedSize >= 0 { - fmt.Fprintf(&buf, "compressed size %d", h.compressedSize) - first = false - } - if h.uncompressedSize >= 0 { - if !first { - buf.WriteString(" ") - } - fmt.Fprintf(&buf, "uncompressed size %d", h.uncompressedSize) - first = false - } - for _, f := range h.filters { - if !first { - buf.WriteString(" ") - } - fmt.Fprintf(&buf, "filter %s", f) - first = false - } - return buf.String() -} - -// Masks for the block flags. -const ( - filterCountMask = 0x03 - compressedSizePresent = 0x40 - uncompressedSizePresent = 0x80 - reservedBlockFlags = 0x3C -) - -// errIndexIndicator signals that an index indicator (0x00) has been found -// instead of an expected block header indicator. -var errIndexIndicator = errors.New("xz: found index indicator") - -// readBlockHeader reads the block header. -func readBlockHeader(r io.Reader) (h *blockHeader, n int, err error) { - var buf bytes.Buffer - buf.Grow(20) - - // block header size - z, err := io.CopyN(&buf, r, 1) - n = int(z) - if err != nil { - return nil, n, err - } - s := buf.Bytes()[0] - if s == 0 { - return nil, n, errIndexIndicator - } - - // read complete header - headerLen := (int(s) + 1) * 4 - buf.Grow(headerLen - 1) - z, err = io.CopyN(&buf, r, int64(headerLen-1)) - n += int(z) - if err != nil { - return nil, n, err - } - - // unmarshal block header - h = new(blockHeader) - if err = h.UnmarshalBinary(buf.Bytes()); err != nil { - return nil, n, err - } - - return h, n, nil -} - -// readSizeInBlockHeader reads the uncompressed or compressed size -// fields in the block header. The present value informs the function -// whether the respective field is actually present in the header. -func readSizeInBlockHeader(r io.ByteReader, present bool) (n int64, err error) { - if !present { - return -1, nil - } - x, _, err := readUvarint(r) - if err != nil { - return 0, err - } - if x >= 1<<63 { - return 0, errors.New("xz: size overflow in block header") - } - return int64(x), nil -} - -// UnmarshalBinary unmarshals the block header. -func (h *blockHeader) UnmarshalBinary(data []byte) error { - // Check header length - s := data[0] - if data[0] == 0 { - return errIndexIndicator - } - headerLen := (int(s) + 1) * 4 - if len(data) != headerLen { - return fmt.Errorf("xz: data length %d; want %d", len(data), - headerLen) - } - n := headerLen - 4 - - // Check CRC-32 - crc := crc32.NewIEEE() - crc.Write(data[:n]) - if crc.Sum32() != uint32LE(data[n:]) { - return errors.New("xz: checksum error for block header") - } - - // Block header flags - flags := data[1] - if flags&reservedBlockFlags != 0 { - return errors.New("xz: reserved block header flags set") - } - - r := bytes.NewReader(data[2:n]) - - // Compressed size - var err error - h.compressedSize, err = readSizeInBlockHeader( - r, flags&compressedSizePresent != 0) - if err != nil { - return err - } - - // Uncompressed size - h.uncompressedSize, err = readSizeInBlockHeader( - r, flags&uncompressedSizePresent != 0) - if err != nil { - return err - } - - h.filters, err = readFilters(r, int(flags&filterCountMask)+1) - if err != nil { - return err - } - - // Check padding - // Since headerLen is a multiple of 4 we don't need to check - // alignment. - k := r.Len() - // The standard spec says that the padding should have not more - // than 3 bytes. However we found paddings of 4 or 5 in the - // wild. See https://github.com/ulikunitz/xz/pull/11 and - // https://github.com/ulikunitz/xz/issues/15 - // - // The only reasonable approach seems to be to ignore the - // padding size. We still check that all padding bytes are zero. - if !allZeros(data[n-k : n]) { - return errPadding - } - return nil -} - -// MarshalBinary marshals the binary header. -func (h *blockHeader) MarshalBinary() (data []byte, err error) { - if !(minFilters <= len(h.filters) && len(h.filters) <= maxFilters) { - return nil, errors.New("xz: filter count wrong") - } - for i, f := range h.filters { - if i < len(h.filters)-1 { - if f.id() == lzmaFilterID { - return nil, errors.New( - "xz: LZMA2 filter is not the last") - } - } else { - // last filter - if f.id() != lzmaFilterID { - return nil, errors.New("xz: " + - "last filter must be the LZMA2 filter") - } - } - } - - var buf bytes.Buffer - // header size must set at the end - buf.WriteByte(0) - - // flags - flags := byte(len(h.filters) - 1) - if h.compressedSize >= 0 { - flags |= compressedSizePresent - } - if h.uncompressedSize >= 0 { - flags |= uncompressedSizePresent - } - buf.WriteByte(flags) - - p := make([]byte, 10) - if h.compressedSize >= 0 { - k := putUvarint(p, uint64(h.compressedSize)) - buf.Write(p[:k]) - } - if h.uncompressedSize >= 0 { - k := putUvarint(p, uint64(h.uncompressedSize)) - buf.Write(p[:k]) - } - - for _, f := range h.filters { - fp, err := f.MarshalBinary() - if err != nil { - return nil, err - } - buf.Write(fp) - } - - // padding - for i := padLen(int64(buf.Len())); i > 0; i-- { - buf.WriteByte(0) - } - - // crc place holder - buf.Write(p[:4]) - - data = buf.Bytes() - if len(data)%4 != 0 { - panic("data length not aligned") - } - s := len(data)/4 - 1 - if !(1 < s && s <= 255) { - panic("wrong block header size") - } - data[0] = byte(s) - - crc := crc32.NewIEEE() - crc.Write(data[:len(data)-4]) - putUint32LE(data[len(data)-4:], crc.Sum32()) - - return data, nil -} - -// Constants used for marshalling and unmarshalling filters in the xz -// block header. -const ( - minFilters = 1 - maxFilters = 4 - minReservedID = 1 << 62 -) - -// filter represents a filter in the block header. -type filter interface { - id() uint64 - UnmarshalBinary(data []byte) error - MarshalBinary() (data []byte, err error) - reader(r io.Reader, c *ReaderConfig) (fr io.Reader, err error) - writeCloser(w io.WriteCloser, c *WriterConfig) (fw io.WriteCloser, err error) - // filter must be last filter - last() bool -} - -// readFilter reads a block filter from the block header. At this point -// in time only the LZMA2 filter is supported. -func readFilter(r io.Reader) (f filter, err error) { - br := lzma.ByteReader(r) - - // index - id, _, err := readUvarint(br) - if err != nil { - return nil, err - } - - var data []byte - switch id { - case lzmaFilterID: - data = make([]byte, lzmaFilterLen) - data[0] = lzmaFilterID - if _, err = io.ReadFull(r, data[1:]); err != nil { - return nil, err - } - f = new(lzmaFilter) - default: - if id >= minReservedID { - return nil, errors.New( - "xz: reserved filter id in block stream header") - } - return nil, errors.New("xz: invalid filter id") - } - if err = f.UnmarshalBinary(data); err != nil { - return nil, err - } - return f, err -} - -// readFilters reads count filters. At this point in time only the count -// 1 is supported. -func readFilters(r io.Reader, count int) (filters []filter, err error) { - if count != 1 { - return nil, errors.New("xz: unsupported filter count") - } - f, err := readFilter(r) - if err != nil { - return nil, err - } - return []filter{f}, err -} - -/*** Index ***/ - -// record describes a block in the xz file index. -type record struct { - unpaddedSize int64 - uncompressedSize int64 -} - -// readRecord reads an index record. -func readRecord(r io.ByteReader) (rec record, n int, err error) { - u, k, err := readUvarint(r) - n += k - if err != nil { - return rec, n, err - } - rec.unpaddedSize = int64(u) - if rec.unpaddedSize < 0 { - return rec, n, errors.New("xz: unpadded size negative") - } - - u, k, err = readUvarint(r) - n += k - if err != nil { - return rec, n, err - } - rec.uncompressedSize = int64(u) - if rec.uncompressedSize < 0 { - return rec, n, errors.New("xz: uncompressed size negative") - } - - return rec, n, nil -} - -// MarshalBinary converts an index record in its binary encoding. -func (rec *record) MarshalBinary() (data []byte, err error) { - // maximum length of a uvarint is 10 - p := make([]byte, 20) - n := putUvarint(p, uint64(rec.unpaddedSize)) - n += putUvarint(p[n:], uint64(rec.uncompressedSize)) - return p[:n], nil -} - -// writeIndex writes the index, a sequence of records. -func writeIndex(w io.Writer, index []record) (n int64, err error) { - crc := crc32.NewIEEE() - mw := io.MultiWriter(w, crc) - - // index indicator - k, err := mw.Write([]byte{0}) - n += int64(k) - if err != nil { - return n, err - } - - // number of records - p := make([]byte, 10) - k = putUvarint(p, uint64(len(index))) - k, err = mw.Write(p[:k]) - n += int64(k) - if err != nil { - return n, err - } - - // list of records - for _, rec := range index { - p, err := rec.MarshalBinary() - if err != nil { - return n, err - } - k, err = mw.Write(p) - n += int64(k) - if err != nil { - return n, err - } - } - - // index padding - k, err = mw.Write(make([]byte, padLen(int64(n)))) - n += int64(k) - if err != nil { - return n, err - } - - // crc32 checksum - putUint32LE(p, crc.Sum32()) - k, err = w.Write(p[:4]) - n += int64(k) - - return n, err -} - -// readIndexBody reads the index from the reader. It assumes that the -// index indicator has already been read. -func readIndexBody(r io.Reader, expectedRecordLen int) (records []record, n int64, err error) { - crc := crc32.NewIEEE() - // index indicator - crc.Write([]byte{0}) - - br := lzma.ByteReader(io.TeeReader(r, crc)) - - // number of records - u, k, err := readUvarint(br) - n += int64(k) - if err != nil { - return nil, n, err - } - recLen := int(u) - if recLen < 0 || uint64(recLen) != u { - return nil, n, errors.New("xz: record number overflow") - } - if recLen != expectedRecordLen { - return nil, n, fmt.Errorf( - "xz: index length is %d; want %d", - recLen, expectedRecordLen) - } - - // list of records - records = make([]record, recLen) - for i := range records { - records[i], k, err = readRecord(br) - n += int64(k) - if err != nil { - return nil, n, err - } - } - - p := make([]byte, padLen(int64(n+1)), 4) - k, err = io.ReadFull(br.(io.Reader), p) - n += int64(k) - if err != nil { - return nil, n, err - } - if !allZeros(p) { - return nil, n, errors.New("xz: non-zero byte in index padding") - } - - // crc32 - s := crc.Sum32() - p = p[:4] - k, err = io.ReadFull(br.(io.Reader), p) - n += int64(k) - if err != nil { - return records, n, err - } - if uint32LE(p) != s { - return nil, n, errors.New("xz: wrong checksum for index") - } - - return records, n, nil -} diff --git a/vendor/github.com/ulikunitz/xz/fox-check-none.xz b/vendor/github.com/ulikunitz/xz/fox-check-none.xz deleted file mode 100644 index 46043f7dc89b610dc3badb9db3426620c4c97462..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 96 zcmexsUKJ6=z`*cd=%ynRgCe6CkX@qxbTK1?PDnLRM*R tL9s%9S!$6&2~avGv8qxbB|lw{3#g5Ofzej?!NQIFY(?{`7{LOOQ2>-O93KDx diff --git a/vendor/github.com/ulikunitz/xz/fox.xz b/vendor/github.com/ulikunitz/xz/fox.xz deleted file mode 100644 index 4b820bd5a16e83fe5db4fb315639a4337f862483..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 104 zcmexsUKJ6=z`*kC+7>q^21Q0O1_p)_{ill=8FWH2QWXkIGn2Cwl8W-n^AytZD-^Oy za|?dFO$zmVVdxt0+m!4eq- E0K@hlng9R* diff --git a/vendor/github.com/ulikunitz/xz/internal/hash/cyclic_poly.go b/vendor/github.com/ulikunitz/xz/internal/hash/cyclic_poly.go deleted file mode 100644 index dae159db..00000000 --- a/vendor/github.com/ulikunitz/xz/internal/hash/cyclic_poly.go +++ /dev/null @@ -1,181 +0,0 @@ -// Copyright 2014-2022 Ulrich Kunitz. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package hash - -// CyclicPoly provides a cyclic polynomial rolling hash. -type CyclicPoly struct { - h uint64 - p []uint64 - i int -} - -// ror rotates the unsigned 64-bit integer to right. The argument s must be -// less than 64. -func ror(x uint64, s uint) uint64 { - return (x >> s) | (x << (64 - s)) -} - -// NewCyclicPoly creates a new instance of the CyclicPoly structure. The -// argument n gives the number of bytes for which a hash will be executed. -// This number must be positive; the method panics if this isn't the case. -func NewCyclicPoly(n int) *CyclicPoly { - if n < 1 { - panic("argument n must be positive") - } - return &CyclicPoly{p: make([]uint64, 0, n)} -} - -// Len returns the length of the byte sequence for which a hash is generated. -func (r *CyclicPoly) Len() int { - return cap(r.p) -} - -// RollByte hashes the next byte and returns a hash value. The complete becomes -// available after at least Len() bytes have been hashed. -func (r *CyclicPoly) RollByte(x byte) uint64 { - y := hash[x] - if len(r.p) < cap(r.p) { - r.h = ror(r.h, 1) ^ y - r.p = append(r.p, y) - } else { - r.h ^= ror(r.p[r.i], uint(cap(r.p)-1)) - r.h = ror(r.h, 1) ^ y - r.p[r.i] = y - r.i = (r.i + 1) % cap(r.p) - } - return r.h -} - -// Stores the hash for the individual bytes. -var hash = [256]uint64{ - 0x2e4fc3f904065142, 0xc790984cfbc99527, - 0x879f95eb8c62f187, 0x3b61be86b5021ef2, - 0x65a896a04196f0a5, 0xc5b307b80470b59e, - 0xd3bff376a70df14b, 0xc332f04f0b3f1701, - 0x753b5f0e9abf3e0d, 0xb41538fdfe66ef53, - 0x1906a10c2c1c0208, 0xfb0c712a03421c0d, - 0x38be311a65c9552b, 0xfee7ee4ca6445c7e, - 0x71aadeded184f21e, 0xd73426fccda23b2d, - 0x29773fb5fb9600b5, 0xce410261cd32981a, - 0xfe2848b3c62dbc2d, 0x459eaaff6e43e11c, - 0xc13e35fc9c73a887, 0xf30ed5c201e76dbc, - 0xa5f10b3910482cea, 0x2945d59be02dfaad, - 0x06ee334ff70571b5, 0xbabf9d8070f44380, - 0xee3e2e9912ffd27c, 0x2a7118d1ea6b8ea7, - 0x26183cb9f7b1664c, 0xea71dac7da068f21, - 0xea92eca5bd1d0bb7, 0x415595862defcd75, - 0x248a386023c60648, 0x9cf021ab284b3c8a, - 0xfc9372df02870f6c, 0x2b92d693eeb3b3fc, - 0x73e799d139dc6975, 0x7b15ae312486363c, - 0xb70e5454a2239c80, 0x208e3fb31d3b2263, - 0x01f563cabb930f44, 0x2ac4533d2a3240d8, - 0x84231ed1064f6f7c, 0xa9f020977c2a6d19, - 0x213c227271c20122, 0x09fe8a9a0a03d07a, - 0x4236dc75bcaf910c, 0x460a8b2bead8f17e, - 0xd9b27be1aa07055f, 0xd202d5dc4b11c33e, - 0x70adb010543bea12, 0xcdae938f7ea6f579, - 0x3f3d870208672f4d, 0x8e6ccbce9d349536, - 0xe4c0871a389095ae, 0xf5f2a49152bca080, - 0x9a43f9b97269934e, 0xc17b3753cb6f475c, - 0xd56d941e8e206bd4, 0xac0a4f3e525eda00, - 0xa06d5a011912a550, 0x5537ed19537ad1df, - 0xa32fe713d611449d, 0x2a1d05b47c3b579f, - 0x991d02dbd30a2a52, 0x39e91e7e28f93eb0, - 0x40d06adb3e92c9ac, 0x9b9d3afde1c77c97, - 0x9a3f3f41c02c616f, 0x22ecd4ba00f60c44, - 0x0b63d5d801708420, 0x8f227ca8f37ffaec, - 0x0256278670887c24, 0x107e14877dbf540b, - 0x32c19f2786ac1c05, 0x1df5b12bb4bc9c61, - 0xc0cac129d0d4c4e2, 0x9fdb52ee9800b001, - 0x31f601d5d31c48c4, 0x72ff3c0928bcaec7, - 0xd99264421147eb03, 0x535a2d6d38aefcfe, - 0x6ba8b4454a916237, 0xfa39366eaae4719c, - 0x10f00fd7bbb24b6f, 0x5bd23185c76c84d4, - 0xb22c3d7e1b00d33f, 0x3efc20aa6bc830a8, - 0xd61c2503fe639144, 0x30ce625441eb92d3, - 0xe5d34cf359e93100, 0xa8e5aa13f2b9f7a5, - 0x5c2b8d851ca254a6, 0x68fb6c5e8b0d5fdf, - 0xc7ea4872c96b83ae, 0x6dd5d376f4392382, - 0x1be88681aaa9792f, 0xfef465ee1b6c10d9, - 0x1f98b65ed43fcb2e, 0x4d1ca11eb6e9a9c9, - 0x7808e902b3857d0b, 0x171c9c4ea4607972, - 0x58d66274850146df, 0x42b311c10d3981d1, - 0x647fa8c621c41a4c, 0xf472771c66ddfedc, - 0x338d27e3f847b46b, 0x6402ce3da97545ce, - 0x5162db616fc38638, 0x9c83be97bc22a50e, - 0x2d3d7478a78d5e72, 0xe621a9b938fd5397, - 0x9454614eb0f81c45, 0x395fb6e742ed39b6, - 0x77dd9179d06037bf, 0xc478d0fee4d2656d, - 0x35d9d6cb772007af, 0x83a56e92c883f0f6, - 0x27937453250c00a1, 0x27bd6ebc3a46a97d, - 0x9f543bf784342d51, 0xd158f38c48b0ed52, - 0x8dd8537c045f66b4, 0x846a57230226f6d5, - 0x6b13939e0c4e7cdf, 0xfca25425d8176758, - 0x92e5fc6cd52788e6, 0x9992e13d7a739170, - 0x518246f7a199e8ea, 0xf104c2a71b9979c7, - 0x86b3ffaabea4768f, 0x6388061cf3e351ad, - 0x09d9b5295de5bbb5, 0x38bf1638c2599e92, - 0x1d759846499e148d, 0x4c0ff015e5f96ef4, - 0xa41a94cfa270f565, 0x42d76f9cb2326c0b, - 0x0cf385dd3c9c23ba, 0x0508a6c7508d6e7a, - 0x337523aabbe6cf8d, 0x646bb14001d42b12, - 0xc178729d138adc74, 0xf900ef4491f24086, - 0xee1a90d334bb5ac4, 0x9755c92247301a50, - 0xb999bf7c4ff1b610, 0x6aeeb2f3b21e8fc9, - 0x0fa8084cf91ac6ff, 0x10d226cf136e6189, - 0xd302057a07d4fb21, 0x5f03800e20a0fcc3, - 0x80118d4ae46bd210, 0x58ab61a522843733, - 0x51edd575c5432a4b, 0x94ee6ff67f9197f7, - 0x765669e0e5e8157b, 0xa5347830737132f0, - 0x3ba485a69f01510c, 0x0b247d7b957a01c3, - 0x1b3d63449fd807dc, 0x0fdc4721c30ad743, - 0x8b535ed3829b2b14, 0xee41d0cad65d232c, - 0xe6a99ed97a6a982f, 0x65ac6194c202003d, - 0x692accf3a70573eb, 0xcc3c02c3e200d5af, - 0x0d419e8b325914a3, 0x320f160f42c25e40, - 0x00710d647a51fe7a, 0x3c947692330aed60, - 0x9288aa280d355a7a, 0xa1806a9b791d1696, - 0x5d60e38496763da1, 0x6c69e22e613fd0f4, - 0x977fc2a5aadffb17, 0xfb7bd063fc5a94ba, - 0x460c17992cbaece1, 0xf7822c5444d3297f, - 0x344a9790c69b74aa, 0xb80a42e6cae09dce, - 0x1b1361eaf2b1e757, 0xd84c1e758e236f01, - 0x88e0b7be347627cc, 0x45246009b7a99490, - 0x8011c6dd3fe50472, 0xc341d682bffb99d7, - 0x2511be93808e2d15, 0xd5bc13d7fd739840, - 0x2a3cd030679ae1ec, 0x8ad9898a4b9ee157, - 0x3245fef0a8eaf521, 0x3d6d8dbbb427d2b0, - 0x1ed146d8968b3981, 0x0c6a28bf7d45f3fc, - 0x4a1fd3dbcee3c561, 0x4210ff6a476bf67e, - 0xa559cce0d9199aac, 0xde39d47ef3723380, - 0xe5b69d848ce42e35, 0xefa24296f8e79f52, - 0x70190b59db9a5afc, 0x26f166cdb211e7bf, - 0x4deaf2df3c6b8ef5, 0xf171dbdd670f1017, - 0xb9059b05e9420d90, 0x2f0da855c9388754, - 0x611d5e9ab77949cc, 0x2912038ac01163f4, - 0x0231df50402b2fba, 0x45660fc4f3245f58, - 0xb91cc97c7c8dac50, 0xb72d2aafe4953427, - 0xfa6463f87e813d6b, 0x4515f7ee95d5c6a2, - 0x1310e1c1a48d21c3, 0xad48a7810cdd8544, - 0x4d5bdfefd5c9e631, 0xa43ed43f1fdcb7de, - 0xe70cfc8fe1ee9626, 0xef4711b0d8dda442, - 0xb80dd9bd4dab6c93, 0xa23be08d31ba4d93, - 0x9b37db9d0335a39c, 0x494b6f870f5cfebc, - 0x6d1b3c1149dda943, 0x372c943a518c1093, - 0xad27af45e77c09c4, 0x3b6f92b646044604, - 0xac2917909f5fcf4f, 0x2069a60e977e5557, - 0x353a469e71014de5, 0x24be356281f55c15, - 0x2b6d710ba8e9adea, 0x404ad1751c749c29, - 0xed7311bf23d7f185, 0xba4f6976b4acc43e, - 0x32d7198d2bc39000, 0xee667019014d6e01, - 0x494ef3e128d14c83, 0x1f95a152baecd6be, - 0x201648dff1f483a5, 0x68c28550c8384af6, - 0x5fc834a6824a7f48, 0x7cd06cb7365eaf28, - 0xd82bbd95e9b30909, 0x234f0d1694c53f6d, - 0xd2fb7f4a96d83f4a, 0xff0d5da83acac05e, - 0xf8f6b97f5585080a, 0x74236084be57b95b, - 0xa25e40c03bbc36ad, 0x6b6e5c14ce88465b, - 0x4378ffe93e1528c5, 0x94ca92a17118e2d2, -} diff --git a/vendor/github.com/ulikunitz/xz/internal/hash/doc.go b/vendor/github.com/ulikunitz/xz/internal/hash/doc.go deleted file mode 100644 index b4cf8b75..00000000 --- a/vendor/github.com/ulikunitz/xz/internal/hash/doc.go +++ /dev/null @@ -1,14 +0,0 @@ -// Copyright 2014-2022 Ulrich Kunitz. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -/* -Package hash provides rolling hashes. - -Rolling hashes have to be used for maintaining the positions of n-byte -sequences in the dictionary buffer. - -The package provides currently the Rabin-Karp rolling hash and a Cyclic -Polynomial hash. Both support the Hashes method to be used with an interface. -*/ -package hash diff --git a/vendor/github.com/ulikunitz/xz/internal/hash/rabin_karp.go b/vendor/github.com/ulikunitz/xz/internal/hash/rabin_karp.go deleted file mode 100644 index 5322342e..00000000 --- a/vendor/github.com/ulikunitz/xz/internal/hash/rabin_karp.go +++ /dev/null @@ -1,66 +0,0 @@ -// Copyright 2014-2022 Ulrich Kunitz. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package hash - -// A is the default constant for Robin-Karp rolling hash. This is a random -// prime. -const A = 0x97b548add41d5da1 - -// RabinKarp supports the computation of a rolling hash. -type RabinKarp struct { - A uint64 - // a^n - aOldest uint64 - h uint64 - p []byte - i int -} - -// NewRabinKarp creates a new RabinKarp value. The argument n defines the -// length of the byte sequence to be hashed. The default constant will will be -// used. -func NewRabinKarp(n int) *RabinKarp { - return NewRabinKarpConst(n, A) -} - -// NewRabinKarpConst creates a new RabinKarp value. The argument n defines the -// length of the byte sequence to be hashed. The argument a provides the -// constant used to compute the hash. -func NewRabinKarpConst(n int, a uint64) *RabinKarp { - if n <= 0 { - panic("number of bytes n must be positive") - } - aOldest := uint64(1) - // There are faster methods. For the small n required by the LZMA - // compressor O(n) is sufficient. - for i := 0; i < n; i++ { - aOldest *= a - } - return &RabinKarp{ - A: a, aOldest: aOldest, - p: make([]byte, 0, n), - } -} - -// Len returns the length of the byte sequence. -func (r *RabinKarp) Len() int { - return cap(r.p) -} - -// RollByte computes the hash after x has been added. -func (r *RabinKarp) RollByte(x byte) uint64 { - if len(r.p) < cap(r.p) { - r.h += uint64(x) - r.h *= r.A - r.p = append(r.p, x) - } else { - r.h -= uint64(r.p[r.i]) * r.aOldest - r.h += uint64(x) - r.h *= r.A - r.p[r.i] = x - r.i = (r.i + 1) % cap(r.p) - } - return r.h -} diff --git a/vendor/github.com/ulikunitz/xz/internal/hash/roller.go b/vendor/github.com/ulikunitz/xz/internal/hash/roller.go deleted file mode 100644 index a9898335..00000000 --- a/vendor/github.com/ulikunitz/xz/internal/hash/roller.go +++ /dev/null @@ -1,29 +0,0 @@ -// Copyright 2014-2022 Ulrich Kunitz. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package hash - -// Roller provides an interface for rolling hashes. The hash value will become -// valid after hash has been called Len times. -type Roller interface { - Len() int - RollByte(x byte) uint64 -} - -// Hashes computes all hash values for the array p. Note that the state of the -// roller is changed. -func Hashes(r Roller, p []byte) []uint64 { - n := r.Len() - if len(p) < n { - return nil - } - h := make([]uint64, len(p)-n+1) - for i := 0; i < n-1; i++ { - r.RollByte(p[i]) - } - for i := range h { - h[i] = r.RollByte(p[i+n-1]) - } - return h -} diff --git a/vendor/github.com/ulikunitz/xz/internal/xlog/xlog.go b/vendor/github.com/ulikunitz/xz/internal/xlog/xlog.go deleted file mode 100644 index f4627ea1..00000000 --- a/vendor/github.com/ulikunitz/xz/internal/xlog/xlog.go +++ /dev/null @@ -1,456 +0,0 @@ -// Copyright 2014-2022 Ulrich Kunitz. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// Package xlog provides a simple logging package that allows to disable -// certain message categories. It defines a type, Logger, with multiple -// methods for formatting output. The package has also a predefined -// 'standard' Logger accessible through helper function Print[f|ln], -// Fatal[f|ln], Panic[f|ln], Warn[f|ln], Print[f|ln] and Debug[f|ln] -// that are easier to use then creating a Logger manually. That logger -// writes to standard error and prints the date and time of each logged -// message, which can be configured using the function SetFlags. -// -// The Fatal functions call os.Exit(1) after the message is output -// unless not suppressed by the flags. The Panic functions call panic -// after the writing the log message unless suppressed. -package xlog - -import ( - "fmt" - "io" - "os" - "runtime" - "sync" - "time" -) - -// The flags define what information is prefixed to each log entry -// generated by the Logger. The Lno* versions allow the suppression of -// specific output. The bits are or'ed together to control what will be -// printed. There is no control over the order of the items printed and -// the format. The full format is: -// -// 2009-01-23 01:23:23.123123 /a/b/c/d.go:23: message -const ( - Ldate = 1 << iota // the date: 2009-01-23 - Ltime // the time: 01:23:23 - Lmicroseconds // microsecond resolution: 01:23:23.123123 - Llongfile // full file name and line number: /a/b/c/d.go:23 - Lshortfile // final file name element and line number: d.go:23 - Lnopanic // suppresses output from Panic[f|ln] but not the panic call - Lnofatal // suppresses output from Fatal[f|ln] but not the exit - Lnowarn // suppresses output from Warn[f|ln] - Lnoprint // suppresses output from Print[f|ln] - Lnodebug // suppresses output from Debug[f|ln] - // initial values for the standard logger - Lstdflags = Ldate | Ltime | Lnodebug -) - -// A Logger represents an active logging object that generates lines of -// output to an io.Writer. Each logging operation if not suppressed -// makes a single call to the Writer's Write method. A Logger can be -// used simultaneously from multiple goroutines; it guarantees to -// serialize access to the Writer. -type Logger struct { - mu sync.Mutex // ensures atomic writes; and protects the following - // fields - prefix string // prefix to write at beginning of each line - flag int // properties - out io.Writer // destination for output - buf []byte // for accumulating text to write -} - -// New creates a new Logger. The out argument sets the destination to -// which the log output will be written. The prefix appears at the -// beginning of each log line. The flag argument defines the logging -// properties. -func New(out io.Writer, prefix string, flag int) *Logger { - return &Logger{out: out, prefix: prefix, flag: flag} -} - -// std is the standard logger used by the package scope functions. -var std = New(os.Stderr, "", Lstdflags) - -// itoa converts the integer to ASCII. A negative widths will avoid -// zero-padding. The function supports only non-negative integers. -func itoa(buf *[]byte, i int, wid int) { - var u = uint(i) - if u == 0 && wid <= 1 { - *buf = append(*buf, '0') - return - } - var b [32]byte - bp := len(b) - for ; u > 0 || wid > 0; u /= 10 { - bp-- - wid-- - b[bp] = byte(u%10) + '0' - } - *buf = append(*buf, b[bp:]...) -} - -// formatHeader puts the header into the buf field of the buffer. -func (l *Logger) formatHeader(t time.Time, file string, line int) { - l.buf = append(l.buf, l.prefix...) - if l.flag&(Ldate|Ltime|Lmicroseconds) != 0 { - if l.flag&Ldate != 0 { - year, month, day := t.Date() - itoa(&l.buf, year, 4) - l.buf = append(l.buf, '-') - itoa(&l.buf, int(month), 2) - l.buf = append(l.buf, '-') - itoa(&l.buf, day, 2) - l.buf = append(l.buf, ' ') - } - if l.flag&(Ltime|Lmicroseconds) != 0 { - hour, min, sec := t.Clock() - itoa(&l.buf, hour, 2) - l.buf = append(l.buf, ':') - itoa(&l.buf, min, 2) - l.buf = append(l.buf, ':') - itoa(&l.buf, sec, 2) - if l.flag&Lmicroseconds != 0 { - l.buf = append(l.buf, '.') - itoa(&l.buf, t.Nanosecond()/1e3, 6) - } - l.buf = append(l.buf, ' ') - } - } - if l.flag&(Lshortfile|Llongfile) != 0 { - if l.flag&Lshortfile != 0 { - short := file - for i := len(file) - 1; i > 0; i-- { - if file[i] == '/' { - short = file[i+1:] - break - } - } - file = short - } - l.buf = append(l.buf, file...) - l.buf = append(l.buf, ':') - itoa(&l.buf, line, -1) - l.buf = append(l.buf, ": "...) - } -} - -func (l *Logger) output(calldepth int, now time.Time, s string) error { - var file string - var line int - if l.flag&(Lshortfile|Llongfile) != 0 { - l.mu.Unlock() - var ok bool - _, file, line, ok = runtime.Caller(calldepth) - if !ok { - file = "???" - line = 0 - } - l.mu.Lock() - } - l.buf = l.buf[:0] - l.formatHeader(now, file, line) - l.buf = append(l.buf, s...) - if len(s) == 0 || s[len(s)-1] != '\n' { - l.buf = append(l.buf, '\n') - } - _, err := l.out.Write(l.buf) - return err -} - -// Output writes the string s with the header controlled by the flags to -// the l.out writer. A newline will be appended if s doesn't end in a -// newline. Calldepth is used to recover the PC, although all current -// calls of Output use the call depth 2. Access to the function is serialized. -func (l *Logger) Output(calldepth, noflag int, v ...interface{}) error { - now := time.Now() - l.mu.Lock() - defer l.mu.Unlock() - if l.flag&noflag != 0 { - return nil - } - s := fmt.Sprint(v...) - return l.output(calldepth+1, now, s) -} - -// Outputf works like output but formats the output like Printf. -func (l *Logger) Outputf(calldepth int, noflag int, format string, v ...interface{}) error { - now := time.Now() - l.mu.Lock() - defer l.mu.Unlock() - if l.flag&noflag != 0 { - return nil - } - s := fmt.Sprintf(format, v...) - return l.output(calldepth+1, now, s) -} - -// Outputln works like output but formats the output like Println. -func (l *Logger) Outputln(calldepth int, noflag int, v ...interface{}) error { - now := time.Now() - l.mu.Lock() - defer l.mu.Unlock() - if l.flag&noflag != 0 { - return nil - } - s := fmt.Sprintln(v...) - return l.output(calldepth+1, now, s) -} - -// Panic prints the message like Print and calls panic. The printing -// might be suppressed by the flag Lnopanic. -func (l *Logger) Panic(v ...interface{}) { - l.Output(2, Lnopanic, v...) - s := fmt.Sprint(v...) - panic(s) -} - -// Panic prints the message like Print and calls panic. The printing -// might be suppressed by the flag Lnopanic. -func Panic(v ...interface{}) { - std.Output(2, Lnopanic, v...) - s := fmt.Sprint(v...) - panic(s) -} - -// Panicf prints the message like Printf and calls panic. The printing -// might be suppressed by the flag Lnopanic. -func (l *Logger) Panicf(format string, v ...interface{}) { - l.Outputf(2, Lnopanic, format, v...) - s := fmt.Sprintf(format, v...) - panic(s) -} - -// Panicf prints the message like Printf and calls panic. The printing -// might be suppressed by the flag Lnopanic. -func Panicf(format string, v ...interface{}) { - std.Outputf(2, Lnopanic, format, v...) - s := fmt.Sprintf(format, v...) - panic(s) -} - -// Panicln prints the message like Println and calls panic. The printing -// might be suppressed by the flag Lnopanic. -func (l *Logger) Panicln(v ...interface{}) { - l.Outputln(2, Lnopanic, v...) - s := fmt.Sprintln(v...) - panic(s) -} - -// Panicln prints the message like Println and calls panic. The printing -// might be suppressed by the flag Lnopanic. -func Panicln(v ...interface{}) { - std.Outputln(2, Lnopanic, v...) - s := fmt.Sprintln(v...) - panic(s) -} - -// Fatal prints the message like Print and calls os.Exit(1). The -// printing might be suppressed by the flag Lnofatal. -func (l *Logger) Fatal(v ...interface{}) { - l.Output(2, Lnofatal, v...) - os.Exit(1) -} - -// Fatal prints the message like Print and calls os.Exit(1). The -// printing might be suppressed by the flag Lnofatal. -func Fatal(v ...interface{}) { - std.Output(2, Lnofatal, v...) - os.Exit(1) -} - -// Fatalf prints the message like Printf and calls os.Exit(1). The -// printing might be suppressed by the flag Lnofatal. -func (l *Logger) Fatalf(format string, v ...interface{}) { - l.Outputf(2, Lnofatal, format, v...) - os.Exit(1) -} - -// Fatalf prints the message like Printf and calls os.Exit(1). The -// printing might be suppressed by the flag Lnofatal. -func Fatalf(format string, v ...interface{}) { - std.Outputf(2, Lnofatal, format, v...) - os.Exit(1) -} - -// Fatalln prints the message like Println and calls os.Exit(1). The -// printing might be suppressed by the flag Lnofatal. -func (l *Logger) Fatalln(format string, v ...interface{}) { - l.Outputln(2, Lnofatal, v...) - os.Exit(1) -} - -// Fatalln prints the message like Println and calls os.Exit(1). The -// printing might be suppressed by the flag Lnofatal. -func Fatalln(format string, v ...interface{}) { - std.Outputln(2, Lnofatal, v...) - os.Exit(1) -} - -// Warn prints the message like Print. The printing might be suppressed -// by the flag Lnowarn. -func (l *Logger) Warn(v ...interface{}) { - l.Output(2, Lnowarn, v...) -} - -// Warn prints the message like Print. The printing might be suppressed -// by the flag Lnowarn. -func Warn(v ...interface{}) { - std.Output(2, Lnowarn, v...) -} - -// Warnf prints the message like Printf. The printing might be suppressed -// by the flag Lnowarn. -func (l *Logger) Warnf(format string, v ...interface{}) { - l.Outputf(2, Lnowarn, format, v...) -} - -// Warnf prints the message like Printf. The printing might be suppressed -// by the flag Lnowarn. -func Warnf(format string, v ...interface{}) { - std.Outputf(2, Lnowarn, format, v...) -} - -// Warnln prints the message like Println. The printing might be suppressed -// by the flag Lnowarn. -func (l *Logger) Warnln(v ...interface{}) { - l.Outputln(2, Lnowarn, v...) -} - -// Warnln prints the message like Println. The printing might be suppressed -// by the flag Lnowarn. -func Warnln(v ...interface{}) { - std.Outputln(2, Lnowarn, v...) -} - -// Print prints the message like fmt.Print. The printing might be suppressed -// by the flag Lnoprint. -func (l *Logger) Print(v ...interface{}) { - l.Output(2, Lnoprint, v...) -} - -// Print prints the message like fmt.Print. The printing might be suppressed -// by the flag Lnoprint. -func Print(v ...interface{}) { - std.Output(2, Lnoprint, v...) -} - -// Printf prints the message like fmt.Printf. The printing might be suppressed -// by the flag Lnoprint. -func (l *Logger) Printf(format string, v ...interface{}) { - l.Outputf(2, Lnoprint, format, v...) -} - -// Printf prints the message like fmt.Printf. The printing might be suppressed -// by the flag Lnoprint. -func Printf(format string, v ...interface{}) { - std.Outputf(2, Lnoprint, format, v...) -} - -// Println prints the message like fmt.Println. The printing might be -// suppressed by the flag Lnoprint. -func (l *Logger) Println(v ...interface{}) { - l.Outputln(2, Lnoprint, v...) -} - -// Println prints the message like fmt.Println. The printing might be -// suppressed by the flag Lnoprint. -func Println(v ...interface{}) { - std.Outputln(2, Lnoprint, v...) -} - -// Debug prints the message like Print. The printing might be suppressed -// by the flag Lnodebug. -func (l *Logger) Debug(v ...interface{}) { - l.Output(2, Lnodebug, v...) -} - -// Debug prints the message like Print. The printing might be suppressed -// by the flag Lnodebug. -func Debug(v ...interface{}) { - std.Output(2, Lnodebug, v...) -} - -// Debugf prints the message like Printf. The printing might be suppressed -// by the flag Lnodebug. -func (l *Logger) Debugf(format string, v ...interface{}) { - l.Outputf(2, Lnodebug, format, v...) -} - -// Debugf prints the message like Printf. The printing might be suppressed -// by the flag Lnodebug. -func Debugf(format string, v ...interface{}) { - std.Outputf(2, Lnodebug, format, v...) -} - -// Debugln prints the message like Println. The printing might be suppressed -// by the flag Lnodebug. -func (l *Logger) Debugln(v ...interface{}) { - l.Outputln(2, Lnodebug, v...) -} - -// Debugln prints the message like Println. The printing might be suppressed -// by the flag Lnodebug. -func Debugln(v ...interface{}) { - std.Outputln(2, Lnodebug, v...) -} - -// Flags returns the current flags used by the logger. -func (l *Logger) Flags() int { - l.mu.Lock() - defer l.mu.Unlock() - return l.flag -} - -// Flags returns the current flags used by the standard logger. -func Flags() int { - return std.Flags() -} - -// SetFlags sets the flags of the logger. -func (l *Logger) SetFlags(flag int) { - l.mu.Lock() - defer l.mu.Unlock() - l.flag = flag -} - -// SetFlags sets the flags for the standard logger. -func SetFlags(flag int) { - std.SetFlags(flag) -} - -// Prefix returns the prefix used by the logger. -func (l *Logger) Prefix() string { - l.mu.Lock() - defer l.mu.Unlock() - return l.prefix -} - -// Prefix returns the prefix used by the standard logger of the package. -func Prefix() string { - return std.Prefix() -} - -// SetPrefix sets the prefix for the logger. -func (l *Logger) SetPrefix(prefix string) { - l.mu.Lock() - defer l.mu.Unlock() - l.prefix = prefix -} - -// SetPrefix sets the prefix of the standard logger of the package. -func SetPrefix(prefix string) { - std.SetPrefix(prefix) -} - -// SetOutput sets the output of the logger. -func (l *Logger) SetOutput(w io.Writer) { - l.mu.Lock() - defer l.mu.Unlock() - l.out = w -} - -// SetOutput sets the output for the standard logger of the package. -func SetOutput(w io.Writer) { - std.SetOutput(w) -} diff --git a/vendor/github.com/ulikunitz/xz/lzma/bintree.go b/vendor/github.com/ulikunitz/xz/lzma/bintree.go deleted file mode 100644 index 2b39da6f..00000000 --- a/vendor/github.com/ulikunitz/xz/lzma/bintree.go +++ /dev/null @@ -1,522 +0,0 @@ -// Copyright 2014-2022 Ulrich Kunitz. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package lzma - -import ( - "errors" - "unicode" -) - -// node represents a node in the binary tree. -type node struct { - // x is the search value - x uint32 - // p parent node - p uint32 - // l left child - l uint32 - // r right child - r uint32 -} - -// wordLen is the number of bytes represented by the v field of a node. -const wordLen = 4 - -// binTree supports the identification of the next operation based on a -// binary tree. -// -// Nodes will be identified by their index into the ring buffer. -type binTree struct { - dict *encoderDict - // ring buffer of nodes - node []node - // absolute offset of the entry for the next node. Position 4 - // byte larger. - hoff int64 - // front position in the node ring buffer - front uint32 - // index of the root node - root uint32 - // current x value - x uint32 - // preallocated array - data []byte -} - -// null represents the nonexistent index. We can't use zero because it -// would always exist or we would need to decrease the index for each -// reference. -const null uint32 = 1<<32 - 1 - -// newBinTree initializes the binTree structure. The capacity defines -// the size of the buffer and defines the maximum distance for which -// matches will be found. -func newBinTree(capacity int) (t *binTree, err error) { - if capacity < 1 { - return nil, errors.New( - "newBinTree: capacity must be larger than zero") - } - if int64(capacity) >= int64(null) { - return nil, errors.New( - "newBinTree: capacity must less 2^{32}-1") - } - t = &binTree{ - node: make([]node, capacity), - hoff: -int64(wordLen), - root: null, - data: make([]byte, maxMatchLen), - } - return t, nil -} - -func (t *binTree) SetDict(d *encoderDict) { t.dict = d } - -// WriteByte writes a single byte into the binary tree. -func (t *binTree) WriteByte(c byte) error { - t.x = (t.x << 8) | uint32(c) - t.hoff++ - if t.hoff < 0 { - return nil - } - v := t.front - if int64(v) < t.hoff { - // We are overwriting old nodes stored in the tree. - t.remove(v) - } - t.node[v].x = t.x - t.add(v) - t.front++ - if int64(t.front) >= int64(len(t.node)) { - t.front = 0 - } - return nil -} - -// Writes writes a sequence of bytes into the binTree structure. -func (t *binTree) Write(p []byte) (n int, err error) { - for _, c := range p { - t.WriteByte(c) - } - return len(p), nil -} - -// add puts the node v into the tree. The node must not be part of the -// tree before. -func (t *binTree) add(v uint32) { - vn := &t.node[v] - // Set left and right to null indices. - vn.l, vn.r = null, null - // If the binary tree is empty make v the root. - if t.root == null { - t.root = v - vn.p = null - return - } - x := vn.x - p := t.root - // Search for the right leave link and add the new node. - for { - pn := &t.node[p] - if x <= pn.x { - if pn.l == null { - pn.l = v - vn.p = p - return - } - p = pn.l - } else { - if pn.r == null { - pn.r = v - vn.p = p - return - } - p = pn.r - } - } -} - -// parent returns the parent node index of v and the pointer to v value -// in the parent. -func (t *binTree) parent(v uint32) (p uint32, ptr *uint32) { - if t.root == v { - return null, &t.root - } - p = t.node[v].p - if t.node[p].l == v { - ptr = &t.node[p].l - } else { - ptr = &t.node[p].r - } - return -} - -// Remove node v. -func (t *binTree) remove(v uint32) { - vn := &t.node[v] - p, ptr := t.parent(v) - l, r := vn.l, vn.r - if l == null { - // Move the right child up. - *ptr = r - if r != null { - t.node[r].p = p - } - return - } - if r == null { - // Move the left child up. - *ptr = l - t.node[l].p = p - return - } - - // Search the in-order predecessor u. - un := &t.node[l] - ur := un.r - if ur == null { - // In order predecessor is l. Move it up. - un.r = r - t.node[r].p = l - un.p = p - *ptr = l - return - } - var u uint32 - for { - // Look for the max value in the tree where l is root. - u = ur - ur = t.node[u].r - if ur == null { - break - } - } - // replace u with ul - un = &t.node[u] - ul := un.l - up := un.p - t.node[up].r = ul - if ul != null { - t.node[ul].p = up - } - - // replace v by u - un.l, un.r = l, r - t.node[l].p = u - t.node[r].p = u - *ptr = u - un.p = p -} - -// search looks for the node that have the value x or for the nodes that -// brace it. The node highest in the tree with the value x will be -// returned. All other nodes with the same value live in left subtree of -// the returned node. -func (t *binTree) search(v uint32, x uint32) (a, b uint32) { - a, b = null, null - if v == null { - return - } - for { - vn := &t.node[v] - if x <= vn.x { - if x == vn.x { - return v, v - } - b = v - if vn.l == null { - return - } - v = vn.l - } else { - a = v - if vn.r == null { - return - } - v = vn.r - } - } -} - -// max returns the node with maximum value in the subtree with v as -// root. -func (t *binTree) max(v uint32) uint32 { - if v == null { - return null - } - for { - r := t.node[v].r - if r == null { - return v - } - v = r - } -} - -// min returns the node with the minimum value in the subtree with v as -// root. -func (t *binTree) min(v uint32) uint32 { - if v == null { - return null - } - for { - l := t.node[v].l - if l == null { - return v - } - v = l - } -} - -// pred returns the in-order predecessor of node v. -func (t *binTree) pred(v uint32) uint32 { - if v == null { - return null - } - u := t.max(t.node[v].l) - if u != null { - return u - } - for { - p := t.node[v].p - if p == null { - return null - } - if t.node[p].r == v { - return p - } - v = p - } -} - -// succ returns the in-order successor of node v. -func (t *binTree) succ(v uint32) uint32 { - if v == null { - return null - } - u := t.min(t.node[v].r) - if u != null { - return u - } - for { - p := t.node[v].p - if p == null { - return null - } - if t.node[p].l == v { - return p - } - v = p - } -} - -// xval converts the first four bytes of a into an 32-bit unsigned -// integer in big-endian order. -func xval(a []byte) uint32 { - var x uint32 - switch len(a) { - default: - x |= uint32(a[3]) - fallthrough - case 3: - x |= uint32(a[2]) << 8 - fallthrough - case 2: - x |= uint32(a[1]) << 16 - fallthrough - case 1: - x |= uint32(a[0]) << 24 - case 0: - } - return x -} - -// dumpX converts value x into a four-letter string. -func dumpX(x uint32) string { - a := make([]byte, 4) - for i := 0; i < 4; i++ { - c := byte(x >> uint((3-i)*8)) - if unicode.IsGraphic(rune(c)) { - a[i] = c - } else { - a[i] = '.' - } - } - return string(a) -} - -/* -// dumpNode writes a representation of the node v into the io.Writer. -func (t *binTree) dumpNode(w io.Writer, v uint32, indent int) { - if v == null { - return - } - - vn := &t.node[v] - - t.dumpNode(w, vn.r, indent+2) - - for i := 0; i < indent; i++ { - fmt.Fprint(w, " ") - } - if vn.p == null { - fmt.Fprintf(w, "node %d %q parent null\n", v, dumpX(vn.x)) - } else { - fmt.Fprintf(w, "node %d %q parent %d\n", v, dumpX(vn.x), vn.p) - } - - t.dumpNode(w, vn.l, indent+2) -} - -// dump prints a representation of the binary tree into the writer. -func (t *binTree) dump(w io.Writer) error { - bw := bufio.NewWriter(w) - t.dumpNode(bw, t.root, 0) - return bw.Flush() -} -*/ - -func (t *binTree) distance(v uint32) int { - dist := int(t.front) - int(v) - if dist <= 0 { - dist += len(t.node) - } - return dist -} - -type matchParams struct { - rep [4]uint32 - // length when match will be accepted - nAccept int - // nodes to check - check int - // finish if length get shorter - stopShorter bool -} - -func (t *binTree) match(m match, distIter func() (int, bool), p matchParams, -) (r match, checked int, accepted bool) { - buf := &t.dict.buf - for { - if checked >= p.check { - return m, checked, true - } - dist, ok := distIter() - if !ok { - return m, checked, false - } - checked++ - if m.n > 0 { - i := buf.rear - dist + m.n - 1 - if i < 0 { - i += len(buf.data) - } else if i >= len(buf.data) { - i -= len(buf.data) - } - if buf.data[i] != t.data[m.n-1] { - if p.stopShorter { - return m, checked, false - } - continue - } - } - n := buf.matchLen(dist, t.data) - switch n { - case 0: - if p.stopShorter { - return m, checked, false - } - continue - case 1: - if uint32(dist-minDistance) != p.rep[0] { - continue - } - } - if n < m.n || (n == m.n && int64(dist) >= m.distance) { - continue - } - m = match{int64(dist), n} - if n >= p.nAccept { - return m, checked, true - } - } -} - -func (t *binTree) NextOp(rep [4]uint32) operation { - // retrieve maxMatchLen data - n, _ := t.dict.buf.Peek(t.data[:maxMatchLen]) - if n == 0 { - panic("no data in buffer") - } - t.data = t.data[:n] - - var ( - m match - x, u, v uint32 - iterPred, iterSucc func() (int, bool) - ) - p := matchParams{ - rep: rep, - nAccept: maxMatchLen, - check: 32, - } - i := 4 - iterSmall := func() (dist int, ok bool) { - i-- - if i <= 0 { - return 0, false - } - return i, true - } - m, checked, accepted := t.match(m, iterSmall, p) - if accepted { - goto end - } - p.check -= checked - x = xval(t.data) - u, v = t.search(t.root, x) - if u == v && len(t.data) == 4 { - iter := func() (dist int, ok bool) { - if u == null { - return 0, false - } - dist = t.distance(u) - u, v = t.search(t.node[u].l, x) - if u != v { - u = null - } - return dist, true - } - m, _, _ = t.match(m, iter, p) - goto end - } - p.stopShorter = true - iterSucc = func() (dist int, ok bool) { - if v == null { - return 0, false - } - dist = t.distance(v) - v = t.succ(v) - return dist, true - } - m, checked, accepted = t.match(m, iterSucc, p) - if accepted { - goto end - } - p.check -= checked - iterPred = func() (dist int, ok bool) { - if u == null { - return 0, false - } - dist = t.distance(u) - u = t.pred(u) - return dist, true - } - m, _, _ = t.match(m, iterPred, p) -end: - if m.n == 0 { - return lit{t.data[0]} - } - return m -} diff --git a/vendor/github.com/ulikunitz/xz/lzma/bitops.go b/vendor/github.com/ulikunitz/xz/lzma/bitops.go deleted file mode 100644 index 20109170..00000000 --- a/vendor/github.com/ulikunitz/xz/lzma/bitops.go +++ /dev/null @@ -1,47 +0,0 @@ -// Copyright 2014-2022 Ulrich Kunitz. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package lzma - -/* Naming conventions follows the CodeReviewComments in the Go Wiki. */ - -// ntz32Const is used by the functions NTZ and NLZ. -const ntz32Const = 0x04d7651f - -// ntz32Table is a helper table for de Bruijn algorithm by Danny Dubé. -// See Henry S. Warren, Jr. "Hacker's Delight" section 5-1 figure 5-26. -var ntz32Table = [32]int8{ - 0, 1, 2, 24, 3, 19, 6, 25, - 22, 4, 20, 10, 16, 7, 12, 26, - 31, 23, 18, 5, 21, 9, 15, 11, - 30, 17, 8, 14, 29, 13, 28, 27, -} - -/* -// ntz32 computes the number of trailing zeros for an unsigned 32-bit integer. -func ntz32(x uint32) int { - if x == 0 { - return 32 - } - x = (x & -x) * ntz32Const - return int(ntz32Table[x>>27]) -} -*/ - -// nlz32 computes the number of leading zeros for an unsigned 32-bit integer. -func nlz32(x uint32) int { - // Smear left most bit to the right - x |= x >> 1 - x |= x >> 2 - x |= x >> 4 - x |= x >> 8 - x |= x >> 16 - // Use ntz mechanism to calculate nlz. - x++ - if x == 0 { - return 0 - } - x *= ntz32Const - return 32 - int(ntz32Table[x>>27]) -} diff --git a/vendor/github.com/ulikunitz/xz/lzma/breader.go b/vendor/github.com/ulikunitz/xz/lzma/breader.go deleted file mode 100644 index 81bd6718..00000000 --- a/vendor/github.com/ulikunitz/xz/lzma/breader.go +++ /dev/null @@ -1,40 +0,0 @@ -// Copyright 2014-2022 Ulrich Kunitz. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package lzma - -import ( - "io" -) - -// breader provides the ReadByte function for a Reader. It doesn't read -// more data from the reader than absolutely necessary. -type breader struct { - io.Reader - // helper slice to save allocations - p []byte -} - -// ByteReader converts an io.Reader into an io.ByteReader. -func ByteReader(r io.Reader) io.ByteReader { - br, ok := r.(io.ByteReader) - if !ok { - return &breader{r, make([]byte, 1)} - } - return br -} - -// ReadByte read byte function. -func (r *breader) ReadByte() (c byte, err error) { - for i := 0; i < 100; i++ { - n, err := r.Reader.Read(r.p) - if n > 0 { - return r.p[0], nil - } - if err != nil { - return 0, err - } - } - return 0, io.ErrNoProgress -} diff --git a/vendor/github.com/ulikunitz/xz/lzma/buffer.go b/vendor/github.com/ulikunitz/xz/lzma/buffer.go deleted file mode 100644 index af41d5b2..00000000 --- a/vendor/github.com/ulikunitz/xz/lzma/buffer.go +++ /dev/null @@ -1,171 +0,0 @@ -// Copyright 2014-2022 Ulrich Kunitz. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package lzma - -import ( - "errors" -) - -// buffer provides a circular buffer of bytes. If the front index equals -// the rear index the buffer is empty. As a consequence front cannot be -// equal rear for a full buffer. So a full buffer has a length that is -// one byte less the the length of the data slice. -type buffer struct { - data []byte - front int - rear int -} - -// newBuffer creates a buffer with the given size. -func newBuffer(size int) *buffer { - return &buffer{data: make([]byte, size+1)} -} - -// Cap returns the capacity of the buffer. -func (b *buffer) Cap() int { - return len(b.data) - 1 -} - -// Resets the buffer. The front and rear index are set to zero. -func (b *buffer) Reset() { - b.front = 0 - b.rear = 0 -} - -// Buffered returns the number of bytes buffered. -func (b *buffer) Buffered() int { - delta := b.front - b.rear - if delta < 0 { - delta += len(b.data) - } - return delta -} - -// Available returns the number of bytes available for writing. -func (b *buffer) Available() int { - delta := b.rear - 1 - b.front - if delta < 0 { - delta += len(b.data) - } - return delta -} - -// addIndex adds a non-negative integer to the index i and returns the -// resulting index. The function takes care of wrapping the index as -// well as potential overflow situations. -func (b *buffer) addIndex(i int, n int) int { - // subtraction of len(b.data) prevents overflow - i += n - len(b.data) - if i < 0 { - i += len(b.data) - } - return i -} - -// Read reads bytes from the buffer into p and returns the number of -// bytes read. The function never returns an error but might return less -// data than requested. -func (b *buffer) Read(p []byte) (n int, err error) { - n, err = b.Peek(p) - b.rear = b.addIndex(b.rear, n) - return n, err -} - -// Peek reads bytes from the buffer into p without changing the buffer. -// Peek will never return an error but might return less data than -// requested. -func (b *buffer) Peek(p []byte) (n int, err error) { - m := b.Buffered() - n = len(p) - if m < n { - n = m - p = p[:n] - } - k := copy(p, b.data[b.rear:]) - if k < n { - copy(p[k:], b.data) - } - return n, nil -} - -// Discard skips the n next bytes to read from the buffer, returning the -// bytes discarded. -// -// If Discards skips fewer than n bytes, it returns an error. -func (b *buffer) Discard(n int) (discarded int, err error) { - if n < 0 { - return 0, errors.New("buffer.Discard: negative argument") - } - m := b.Buffered() - if m < n { - n = m - err = errors.New( - "buffer.Discard: discarded less bytes then requested") - } - b.rear = b.addIndex(b.rear, n) - return n, err -} - -// ErrNoSpace indicates that there is insufficient space for the Write -// operation. -var ErrNoSpace = errors.New("insufficient space") - -// Write puts data into the buffer. If less bytes are written than -// requested ErrNoSpace is returned. -func (b *buffer) Write(p []byte) (n int, err error) { - m := b.Available() - n = len(p) - if m < n { - n = m - p = p[:m] - err = ErrNoSpace - } - k := copy(b.data[b.front:], p) - if k < n { - copy(b.data, p[k:]) - } - b.front = b.addIndex(b.front, n) - return n, err -} - -// WriteByte writes a single byte into the buffer. The error ErrNoSpace -// is returned if no single byte is available in the buffer for writing. -func (b *buffer) WriteByte(c byte) error { - if b.Available() < 1 { - return ErrNoSpace - } - b.data[b.front] = c - b.front = b.addIndex(b.front, 1) - return nil -} - -// prefixLen returns the length of the common prefix of a and b. -func prefixLen(a, b []byte) int { - if len(a) > len(b) { - a, b = b, a - } - for i, c := range a { - if b[i] != c { - return i - } - } - return len(a) -} - -// matchLen returns the length of the common prefix for the given -// distance from the rear and the byte slice p. -func (b *buffer) matchLen(distance int, p []byte) int { - var n int - i := b.rear - distance - if i < 0 { - if n = prefixLen(p, b.data[len(b.data)+i:]); n < -i { - return n - } - p = p[n:] - i = 0 - } - n += prefixLen(p, b.data[i:]) - return n -} diff --git a/vendor/github.com/ulikunitz/xz/lzma/bytewriter.go b/vendor/github.com/ulikunitz/xz/lzma/bytewriter.go deleted file mode 100644 index f27e31a4..00000000 --- a/vendor/github.com/ulikunitz/xz/lzma/bytewriter.go +++ /dev/null @@ -1,37 +0,0 @@ -// Copyright 2014-2022 Ulrich Kunitz. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package lzma - -import ( - "errors" - "io" -) - -// ErrLimit indicates that the limit of the LimitedByteWriter has been -// reached. -var ErrLimit = errors.New("limit reached") - -// LimitedByteWriter provides a byte writer that can be written until a -// limit is reached. The field N provides the number of remaining -// bytes. -type LimitedByteWriter struct { - BW io.ByteWriter - N int64 -} - -// WriteByte writes a single byte to the limited byte writer. It returns -// ErrLimit if the limit has been reached. If the byte is successfully -// written the field N of the LimitedByteWriter will be decremented by -// one. -func (l *LimitedByteWriter) WriteByte(c byte) error { - if l.N <= 0 { - return ErrLimit - } - if err := l.BW.WriteByte(c); err != nil { - return err - } - l.N-- - return nil -} diff --git a/vendor/github.com/ulikunitz/xz/lzma/decoder.go b/vendor/github.com/ulikunitz/xz/lzma/decoder.go deleted file mode 100644 index 3765484e..00000000 --- a/vendor/github.com/ulikunitz/xz/lzma/decoder.go +++ /dev/null @@ -1,277 +0,0 @@ -// Copyright 2014-2022 Ulrich Kunitz. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package lzma - -import ( - "errors" - "fmt" - "io" -) - -// decoder decodes a raw LZMA stream without any header. -type decoder struct { - // dictionary; the rear pointer of the buffer will be used for - // reading the data. - Dict *decoderDict - // decoder state - State *state - // range decoder - rd *rangeDecoder - // start stores the head value of the dictionary for the LZMA - // stream - start int64 - // size of uncompressed data - size int64 - // end-of-stream encountered - eos bool - // EOS marker found - eosMarker bool -} - -// newDecoder creates a new decoder instance. The parameter size provides -// the expected byte size of the decompressed data. If the size is -// unknown use a negative value. In that case the decoder will look for -// a terminating end-of-stream marker. -func newDecoder(br io.ByteReader, state *state, dict *decoderDict, size int64) (d *decoder, err error) { - rd, err := newRangeDecoder(br) - if err != nil { - return nil, err - } - d = &decoder{ - State: state, - Dict: dict, - rd: rd, - size: size, - start: dict.pos(), - } - return d, nil -} - -// Reopen restarts the decoder with a new byte reader and a new size. Reopen -// resets the Decompressed counter to zero. -func (d *decoder) Reopen(br io.ByteReader, size int64) error { - var err error - if d.rd, err = newRangeDecoder(br); err != nil { - return err - } - d.start = d.Dict.pos() - d.size = size - d.eos = false - return nil -} - -// decodeLiteral decodes a single literal from the LZMA stream. -func (d *decoder) decodeLiteral() (op operation, err error) { - litState := d.State.litState(d.Dict.byteAt(1), d.Dict.head) - match := d.Dict.byteAt(int(d.State.rep[0]) + 1) - s, err := d.State.litCodec.Decode(d.rd, d.State.state, match, litState) - if err != nil { - return nil, err - } - return lit{s}, nil -} - -// errEOS indicates that an EOS marker has been found. -var errEOS = errors.New("EOS marker found") - -// readOp decodes the next operation from the compressed stream. It -// returns the operation. If an explicit end of stream marker is -// identified the eos error is returned. -func (d *decoder) readOp() (op operation, err error) { - // Value of the end of stream (EOS) marker - const eosDist = 1<<32 - 1 - - state, state2, posState := d.State.states(d.Dict.head) - - b, err := d.State.isMatch[state2].Decode(d.rd) - if err != nil { - return nil, err - } - if b == 0 { - // literal - op, err := d.decodeLiteral() - if err != nil { - return nil, err - } - d.State.updateStateLiteral() - return op, nil - } - b, err = d.State.isRep[state].Decode(d.rd) - if err != nil { - return nil, err - } - if b == 0 { - // simple match - d.State.rep[3], d.State.rep[2], d.State.rep[1] = - d.State.rep[2], d.State.rep[1], d.State.rep[0] - - d.State.updateStateMatch() - // The length decoder returns the length offset. - n, err := d.State.lenCodec.Decode(d.rd, posState) - if err != nil { - return nil, err - } - // The dist decoder returns the distance offset. The actual - // distance is 1 higher. - d.State.rep[0], err = d.State.distCodec.Decode(d.rd, n) - if err != nil { - return nil, err - } - if d.State.rep[0] == eosDist { - d.eosMarker = true - return nil, errEOS - } - op = match{n: int(n) + minMatchLen, - distance: int64(d.State.rep[0]) + minDistance} - return op, nil - } - b, err = d.State.isRepG0[state].Decode(d.rd) - if err != nil { - return nil, err - } - dist := d.State.rep[0] - if b == 0 { - // rep match 0 - b, err = d.State.isRepG0Long[state2].Decode(d.rd) - if err != nil { - return nil, err - } - if b == 0 { - d.State.updateStateShortRep() - op = match{n: 1, distance: int64(dist) + minDistance} - return op, nil - } - } else { - b, err = d.State.isRepG1[state].Decode(d.rd) - if err != nil { - return nil, err - } - if b == 0 { - dist = d.State.rep[1] - } else { - b, err = d.State.isRepG2[state].Decode(d.rd) - if err != nil { - return nil, err - } - if b == 0 { - dist = d.State.rep[2] - } else { - dist = d.State.rep[3] - d.State.rep[3] = d.State.rep[2] - } - d.State.rep[2] = d.State.rep[1] - } - d.State.rep[1] = d.State.rep[0] - d.State.rep[0] = dist - } - n, err := d.State.repLenCodec.Decode(d.rd, posState) - if err != nil { - return nil, err - } - d.State.updateStateRep() - op = match{n: int(n) + minMatchLen, distance: int64(dist) + minDistance} - return op, nil -} - -// apply takes the operation and transforms the decoder dictionary accordingly. -func (d *decoder) apply(op operation) error { - var err error - switch x := op.(type) { - case match: - err = d.Dict.writeMatch(x.distance, x.n) - case lit: - err = d.Dict.WriteByte(x.b) - default: - panic("op is neither a match nor a literal") - } - return err -} - -// decompress fills the dictionary unless no space for new data is -// available. If the end of the LZMA stream has been reached io.EOF will -// be returned. -func (d *decoder) decompress() error { - if d.eos { - return io.EOF - } - for d.Dict.Available() >= maxMatchLen { - op, err := d.readOp() - switch err { - case nil: - // break - case errEOS: - d.eos = true - if !d.rd.possiblyAtEnd() { - return errDataAfterEOS - } - if d.size >= 0 && d.size != d.Decompressed() { - return errSize - } - return io.EOF - case io.EOF: - d.eos = true - return io.ErrUnexpectedEOF - default: - return err - } - if err = d.apply(op); err != nil { - return err - } - if d.size >= 0 && d.Decompressed() >= d.size { - d.eos = true - if d.Decompressed() > d.size { - return errSize - } - if !d.rd.possiblyAtEnd() { - switch _, err = d.readOp(); err { - case nil: - return errSize - case io.EOF: - return io.ErrUnexpectedEOF - case errEOS: - break - default: - return err - } - } - return io.EOF - } - } - return nil -} - -// Errors that may be returned while decoding data. -var ( - errDataAfterEOS = errors.New("lzma: data after end of stream marker") - errSize = errors.New("lzma: wrong uncompressed data size") -) - -// Read reads data from the buffer. If no more data is available io.EOF is -// returned. -func (d *decoder) Read(p []byte) (n int, err error) { - var k int - for { - // Read of decoder dict never returns an error. - k, err = d.Dict.Read(p[n:]) - if err != nil { - panic(fmt.Errorf("dictionary read error %s", err)) - } - if k == 0 && d.eos { - return n, io.EOF - } - n += k - if n >= len(p) { - return n, nil - } - if err = d.decompress(); err != nil && err != io.EOF { - return n, err - } - } -} - -// Decompressed returns the number of bytes decompressed by the decoder. -func (d *decoder) Decompressed() int64 { - return d.Dict.pos() - d.start -} diff --git a/vendor/github.com/ulikunitz/xz/lzma/decoderdict.go b/vendor/github.com/ulikunitz/xz/lzma/decoderdict.go deleted file mode 100644 index d5b814f0..00000000 --- a/vendor/github.com/ulikunitz/xz/lzma/decoderdict.go +++ /dev/null @@ -1,128 +0,0 @@ -// Copyright 2014-2022 Ulrich Kunitz. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package lzma - -import ( - "errors" - "fmt" -) - -// decoderDict provides the dictionary for the decoder. The whole -// dictionary is used as reader buffer. -type decoderDict struct { - buf buffer - head int64 -} - -// newDecoderDict creates a new decoder dictionary. The whole dictionary -// will be used as reader buffer. -func newDecoderDict(dictCap int) (d *decoderDict, err error) { - // lower limit supports easy test cases - if !(1 <= dictCap && int64(dictCap) <= MaxDictCap) { - return nil, errors.New("lzma: dictCap out of range") - } - d = &decoderDict{buf: *newBuffer(dictCap)} - return d, nil -} - -// Reset clears the dictionary. The read buffer is not changed, so the -// buffered data can still be read. -func (d *decoderDict) Reset() { - d.head = 0 -} - -// WriteByte writes a single byte into the dictionary. It is used to -// write literals into the dictionary. -func (d *decoderDict) WriteByte(c byte) error { - if err := d.buf.WriteByte(c); err != nil { - return err - } - d.head++ - return nil -} - -// pos returns the position of the dictionary head. -func (d *decoderDict) pos() int64 { return d.head } - -// dictLen returns the actual length of the dictionary. -func (d *decoderDict) dictLen() int { - capacity := d.buf.Cap() - if d.head >= int64(capacity) { - return capacity - } - return int(d.head) -} - -// byteAt returns a byte stored in the dictionary. If the distance is -// non-positive or exceeds the current length of the dictionary the zero -// byte is returned. -func (d *decoderDict) byteAt(dist int) byte { - if !(0 < dist && dist <= d.dictLen()) { - return 0 - } - i := d.buf.front - dist - if i < 0 { - i += len(d.buf.data) - } - return d.buf.data[i] -} - -// writeMatch writes the match at the top of the dictionary. The given -// distance must point in the current dictionary and the length must not -// exceed the maximum length 273 supported in LZMA. -// -// The error value ErrNoSpace indicates that no space is available in -// the dictionary for writing. You need to read from the dictionary -// first. -func (d *decoderDict) writeMatch(dist int64, length int) error { - if !(0 < dist && dist <= int64(d.dictLen())) { - return errors.New("writeMatch: distance out of range") - } - if !(0 < length && length <= maxMatchLen) { - return errors.New("writeMatch: length out of range") - } - if length > d.buf.Available() { - return ErrNoSpace - } - d.head += int64(length) - - i := d.buf.front - int(dist) - if i < 0 { - i += len(d.buf.data) - } - for length > 0 { - var p []byte - if i >= d.buf.front { - p = d.buf.data[i:] - i = 0 - } else { - p = d.buf.data[i:d.buf.front] - i = d.buf.front - } - if len(p) > length { - p = p[:length] - } - if _, err := d.buf.Write(p); err != nil { - panic(fmt.Errorf("d.buf.Write returned error %s", err)) - } - length -= len(p) - } - return nil -} - -// Write writes the given bytes into the dictionary and advances the -// head. -func (d *decoderDict) Write(p []byte) (n int, err error) { - n, err = d.buf.Write(p) - d.head += int64(n) - return n, err -} - -// Available returns the number of available bytes for writing into the -// decoder dictionary. -func (d *decoderDict) Available() int { return d.buf.Available() } - -// Read reads data from the buffer contained in the decoder dictionary. -func (d *decoderDict) Read(p []byte) (n int, err error) { return d.buf.Read(p) } diff --git a/vendor/github.com/ulikunitz/xz/lzma/directcodec.go b/vendor/github.com/ulikunitz/xz/lzma/directcodec.go deleted file mode 100644 index 76b71310..00000000 --- a/vendor/github.com/ulikunitz/xz/lzma/directcodec.go +++ /dev/null @@ -1,38 +0,0 @@ -// Copyright 2014-2022 Ulrich Kunitz. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package lzma - -// directCodec allows the encoding and decoding of values with a fixed number -// of bits. The number of bits must be in the range [1,32]. -type directCodec byte - -// Bits returns the number of bits supported by this codec. -func (dc directCodec) Bits() int { - return int(dc) -} - -// Encode uses the range encoder to encode a value with the fixed number of -// bits. The most-significant bit is encoded first. -func (dc directCodec) Encode(e *rangeEncoder, v uint32) error { - for i := int(dc) - 1; i >= 0; i-- { - if err := e.DirectEncodeBit(v >> uint(i)); err != nil { - return err - } - } - return nil -} - -// Decode uses the range decoder to decode a value with the given number of -// given bits. The most-significant bit is decoded first. -func (dc directCodec) Decode(d *rangeDecoder) (v uint32, err error) { - for i := int(dc) - 1; i >= 0; i-- { - x, err := d.DirectDecodeBit() - if err != nil { - return 0, err - } - v = (v << 1) | x - } - return v, nil -} diff --git a/vendor/github.com/ulikunitz/xz/lzma/distcodec.go b/vendor/github.com/ulikunitz/xz/lzma/distcodec.go deleted file mode 100644 index b447d8ec..00000000 --- a/vendor/github.com/ulikunitz/xz/lzma/distcodec.go +++ /dev/null @@ -1,140 +0,0 @@ -// Copyright 2014-2022 Ulrich Kunitz. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package lzma - -// Constants used by the distance codec. -const ( - // minimum supported distance - minDistance = 1 - // maximum supported distance, value is used for the eos marker. - maxDistance = 1 << 32 - // number of the supported len states - lenStates = 4 - // start for the position models - startPosModel = 4 - // first index with align bits support - endPosModel = 14 - // bits for the position slots - posSlotBits = 6 - // number of align bits - alignBits = 4 -) - -// distCodec provides encoding and decoding of distance values. -type distCodec struct { - posSlotCodecs [lenStates]treeCodec - posModel [endPosModel - startPosModel]treeReverseCodec - alignCodec treeReverseCodec -} - -// deepcopy initializes dc as deep copy of the source. -func (dc *distCodec) deepcopy(src *distCodec) { - if dc == src { - return - } - for i := range dc.posSlotCodecs { - dc.posSlotCodecs[i].deepcopy(&src.posSlotCodecs[i]) - } - for i := range dc.posModel { - dc.posModel[i].deepcopy(&src.posModel[i]) - } - dc.alignCodec.deepcopy(&src.alignCodec) -} - -// newDistCodec creates a new distance codec. -func (dc *distCodec) init() { - for i := range dc.posSlotCodecs { - dc.posSlotCodecs[i] = makeTreeCodec(posSlotBits) - } - for i := range dc.posModel { - posSlot := startPosModel + i - bits := (posSlot >> 1) - 1 - dc.posModel[i] = makeTreeReverseCodec(bits) - } - dc.alignCodec = makeTreeReverseCodec(alignBits) -} - -// lenState converts the value l to a supported lenState value. -func lenState(l uint32) uint32 { - if l >= lenStates { - l = lenStates - 1 - } - return l -} - -// Encode encodes the distance using the parameter l. Dist can have values from -// the full range of uint32 values. To get the distance offset the actual match -// distance has to be decreased by 1. A distance offset of 0xffffffff (eos) -// indicates the end of the stream. -func (dc *distCodec) Encode(e *rangeEncoder, dist uint32, l uint32) (err error) { - // Compute the posSlot using nlz32 - var posSlot uint32 - var bits uint32 - if dist < startPosModel { - posSlot = dist - } else { - bits = uint32(30 - nlz32(dist)) - posSlot = startPosModel - 2 + (bits << 1) - posSlot += (dist >> uint(bits)) & 1 - } - - if err = dc.posSlotCodecs[lenState(l)].Encode(e, posSlot); err != nil { - return - } - - switch { - case posSlot < startPosModel: - return nil - case posSlot < endPosModel: - tc := &dc.posModel[posSlot-startPosModel] - return tc.Encode(dist, e) - } - dic := directCodec(bits - alignBits) - if err = dic.Encode(e, dist>>alignBits); err != nil { - return - } - return dc.alignCodec.Encode(dist, e) -} - -// Decode decodes the distance offset using the parameter l. The dist value -// 0xffffffff (eos) indicates the end of the stream. Add one to the distance -// offset to get the actual match distance. -func (dc *distCodec) Decode(d *rangeDecoder, l uint32) (dist uint32, err error) { - posSlot, err := dc.posSlotCodecs[lenState(l)].Decode(d) - if err != nil { - return - } - - // posSlot equals distance - if posSlot < startPosModel { - return posSlot, nil - } - - // posSlot uses the individual models - bits := (posSlot >> 1) - 1 - dist = (2 | (posSlot & 1)) << bits - var u uint32 - if posSlot < endPosModel { - tc := &dc.posModel[posSlot-startPosModel] - if u, err = tc.Decode(d); err != nil { - return 0, err - } - dist += u - return dist, nil - } - - // posSlots use direct encoding and a single model for the four align - // bits. - dic := directCodec(bits - alignBits) - if u, err = dic.Decode(d); err != nil { - return 0, err - } - dist += u << alignBits - if u, err = dc.alignCodec.Decode(d); err != nil { - return 0, err - } - dist += u - return dist, nil -} diff --git a/vendor/github.com/ulikunitz/xz/lzma/encoder.go b/vendor/github.com/ulikunitz/xz/lzma/encoder.go deleted file mode 100644 index e4093831..00000000 --- a/vendor/github.com/ulikunitz/xz/lzma/encoder.go +++ /dev/null @@ -1,268 +0,0 @@ -// Copyright 2014-2022 Ulrich Kunitz. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package lzma - -import ( - "fmt" - "io" -) - -// opLenMargin provides the upper limit of the number of bytes required -// to encode a single operation. -const opLenMargin = 16 - -// compressFlags control the compression process. -type compressFlags uint32 - -// Values for compressFlags. -const ( - // all data should be compressed, even if compression is not - // optimal. - all compressFlags = 1 << iota -) - -// encoderFlags provide the flags for an encoder. -type encoderFlags uint32 - -// Flags for the encoder. -const ( - // eosMarker requests an EOS marker to be written. - eosMarker encoderFlags = 1 << iota -) - -// Encoder compresses data buffered in the encoder dictionary and writes -// it into a byte writer. -type encoder struct { - dict *encoderDict - state *state - re *rangeEncoder - start int64 - // generate eos marker - marker bool - limit bool - margin int -} - -// newEncoder creates a new encoder. If the byte writer must be -// limited use LimitedByteWriter provided by this package. The flags -// argument supports the eosMarker flag, controlling whether a -// terminating end-of-stream marker must be written. -func newEncoder(bw io.ByteWriter, state *state, dict *encoderDict, - flags encoderFlags) (e *encoder, err error) { - - re, err := newRangeEncoder(bw) - if err != nil { - return nil, err - } - e = &encoder{ - dict: dict, - state: state, - re: re, - marker: flags&eosMarker != 0, - start: dict.Pos(), - margin: opLenMargin, - } - if e.marker { - e.margin += 5 - } - return e, nil -} - -// Write writes the bytes from p into the dictionary. If not enough -// space is available the data in the dictionary buffer will be -// compressed to make additional space available. If the limit of the -// underlying writer has been reached ErrLimit will be returned. -func (e *encoder) Write(p []byte) (n int, err error) { - for { - k, err := e.dict.Write(p[n:]) - n += k - if err == ErrNoSpace { - if err = e.compress(0); err != nil { - return n, err - } - continue - } - return n, err - } -} - -// Reopen reopens the encoder with a new byte writer. -func (e *encoder) Reopen(bw io.ByteWriter) error { - var err error - if e.re, err = newRangeEncoder(bw); err != nil { - return err - } - e.start = e.dict.Pos() - e.limit = false - return nil -} - -// writeLiteral writes a literal into the LZMA stream -func (e *encoder) writeLiteral(l lit) error { - var err error - state, state2, _ := e.state.states(e.dict.Pos()) - if err = e.state.isMatch[state2].Encode(e.re, 0); err != nil { - return err - } - litState := e.state.litState(e.dict.ByteAt(1), e.dict.Pos()) - match := e.dict.ByteAt(int(e.state.rep[0]) + 1) - err = e.state.litCodec.Encode(e.re, l.b, state, match, litState) - if err != nil { - return err - } - e.state.updateStateLiteral() - return nil -} - -// iverson implements the Iverson operator as proposed by Donald Knuth in his -// book Concrete Mathematics. -func iverson(ok bool) uint32 { - if ok { - return 1 - } - return 0 -} - -// writeMatch writes a repetition operation into the operation stream -func (e *encoder) writeMatch(m match) error { - var err error - if !(minDistance <= m.distance && m.distance <= maxDistance) { - panic(fmt.Errorf("match distance %d out of range", m.distance)) - } - dist := uint32(m.distance - minDistance) - if !(minMatchLen <= m.n && m.n <= maxMatchLen) && - !(dist == e.state.rep[0] && m.n == 1) { - panic(fmt.Errorf( - "match length %d out of range; dist %d rep[0] %d", - m.n, dist, e.state.rep[0])) - } - state, state2, posState := e.state.states(e.dict.Pos()) - if err = e.state.isMatch[state2].Encode(e.re, 1); err != nil { - return err - } - g := 0 - for ; g < 4; g++ { - if e.state.rep[g] == dist { - break - } - } - b := iverson(g < 4) - if err = e.state.isRep[state].Encode(e.re, b); err != nil { - return err - } - n := uint32(m.n - minMatchLen) - if b == 0 { - // simple match - e.state.rep[3], e.state.rep[2], e.state.rep[1], e.state.rep[0] = - e.state.rep[2], e.state.rep[1], e.state.rep[0], dist - e.state.updateStateMatch() - if err = e.state.lenCodec.Encode(e.re, n, posState); err != nil { - return err - } - return e.state.distCodec.Encode(e.re, dist, n) - } - b = iverson(g != 0) - if err = e.state.isRepG0[state].Encode(e.re, b); err != nil { - return err - } - if b == 0 { - // g == 0 - b = iverson(m.n != 1) - if err = e.state.isRepG0Long[state2].Encode(e.re, b); err != nil { - return err - } - if b == 0 { - e.state.updateStateShortRep() - return nil - } - } else { - // g in {1,2,3} - b = iverson(g != 1) - if err = e.state.isRepG1[state].Encode(e.re, b); err != nil { - return err - } - if b == 1 { - // g in {2,3} - b = iverson(g != 2) - err = e.state.isRepG2[state].Encode(e.re, b) - if err != nil { - return err - } - if b == 1 { - e.state.rep[3] = e.state.rep[2] - } - e.state.rep[2] = e.state.rep[1] - } - e.state.rep[1] = e.state.rep[0] - e.state.rep[0] = dist - } - e.state.updateStateRep() - return e.state.repLenCodec.Encode(e.re, n, posState) -} - -// writeOp writes a single operation to the range encoder. The function -// checks whether there is enough space available to close the LZMA -// stream. -func (e *encoder) writeOp(op operation) error { - if e.re.Available() < int64(e.margin) { - return ErrLimit - } - switch x := op.(type) { - case lit: - return e.writeLiteral(x) - case match: - return e.writeMatch(x) - default: - panic("unexpected operation") - } -} - -// compress compressed data from the dictionary buffer. If the flag all -// is set, all data in the dictionary buffer will be compressed. The -// function returns ErrLimit if the underlying writer has reached its -// limit. -func (e *encoder) compress(flags compressFlags) error { - n := 0 - if flags&all == 0 { - n = maxMatchLen - 1 - } - d := e.dict - m := d.m - for d.Buffered() > n { - op := m.NextOp(e.state.rep) - if err := e.writeOp(op); err != nil { - return err - } - d.Discard(op.Len()) - } - return nil -} - -// eosMatch is a pseudo operation that indicates the end of the stream. -var eosMatch = match{distance: maxDistance, n: minMatchLen} - -// Close terminates the LZMA stream. If requested the end-of-stream -// marker will be written. If the byte writer limit has been or will be -// reached during compression of the remaining data in the buffer the -// LZMA stream will be closed and data will remain in the buffer. -func (e *encoder) Close() error { - err := e.compress(all) - if err != nil && err != ErrLimit { - return err - } - if e.marker { - if err := e.writeMatch(eosMatch); err != nil { - return err - } - } - err = e.re.Close() - return err -} - -// Compressed returns the number bytes of the input data that been -// compressed. -func (e *encoder) Compressed() int64 { - return e.dict.Pos() - e.start -} diff --git a/vendor/github.com/ulikunitz/xz/lzma/encoderdict.go b/vendor/github.com/ulikunitz/xz/lzma/encoderdict.go deleted file mode 100644 index 4b3916ea..00000000 --- a/vendor/github.com/ulikunitz/xz/lzma/encoderdict.go +++ /dev/null @@ -1,149 +0,0 @@ -// Copyright 2014-2022 Ulrich Kunitz. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package lzma - -import ( - "errors" - "fmt" - "io" -) - -// matcher is an interface that supports the identification of the next -// operation. -type matcher interface { - io.Writer - SetDict(d *encoderDict) - NextOp(rep [4]uint32) operation -} - -// encoderDict provides the dictionary of the encoder. It includes an -// additional buffer atop of the actual dictionary. -type encoderDict struct { - buf buffer - m matcher - head int64 - capacity int - // preallocated array - data [maxMatchLen]byte -} - -// newEncoderDict creates the encoder dictionary. The argument bufSize -// defines the size of the additional buffer. -func newEncoderDict(dictCap, bufSize int, m matcher) (d *encoderDict, err error) { - if !(1 <= dictCap && int64(dictCap) <= MaxDictCap) { - return nil, errors.New( - "lzma: dictionary capacity out of range") - } - if bufSize < 1 { - return nil, errors.New( - "lzma: buffer size must be larger than zero") - } - d = &encoderDict{ - buf: *newBuffer(dictCap + bufSize), - capacity: dictCap, - m: m, - } - m.SetDict(d) - return d, nil -} - -// Discard discards n bytes. Note that n must not be larger than -// MaxMatchLen. -func (d *encoderDict) Discard(n int) { - p := d.data[:n] - k, _ := d.buf.Read(p) - if k < n { - panic(fmt.Errorf("lzma: can't discard %d bytes", n)) - } - d.head += int64(n) - d.m.Write(p) -} - -// Len returns the data available in the encoder dictionary. -func (d *encoderDict) Len() int { - n := d.buf.Available() - if int64(n) > d.head { - return int(d.head) - } - return n -} - -// DictLen returns the actual length of data in the dictionary. -func (d *encoderDict) DictLen() int { - if d.head < int64(d.capacity) { - return int(d.head) - } - return d.capacity -} - -// Available returns the number of bytes that can be written by a -// following Write call. -func (d *encoderDict) Available() int { - return d.buf.Available() - d.DictLen() -} - -// Write writes data into the dictionary buffer. Note that the position -// of the dictionary head will not be moved. If there is not enough -// space in the buffer ErrNoSpace will be returned. -func (d *encoderDict) Write(p []byte) (n int, err error) { - m := d.Available() - if len(p) > m { - p = p[:m] - err = ErrNoSpace - } - var e error - if n, e = d.buf.Write(p); e != nil { - err = e - } - return n, err -} - -// Pos returns the position of the head. -func (d *encoderDict) Pos() int64 { return d.head } - -// ByteAt returns the byte at the given distance. -func (d *encoderDict) ByteAt(distance int) byte { - if !(0 < distance && distance <= d.Len()) { - return 0 - } - i := d.buf.rear - distance - if i < 0 { - i += len(d.buf.data) - } - return d.buf.data[i] -} - -// CopyN copies the last n bytes from the dictionary into the provided -// writer. This is used for copying uncompressed data into an -// uncompressed segment. -func (d *encoderDict) CopyN(w io.Writer, n int) (written int, err error) { - if n <= 0 { - return 0, nil - } - m := d.Len() - if n > m { - n = m - err = ErrNoSpace - } - i := d.buf.rear - n - var e error - if i < 0 { - i += len(d.buf.data) - if written, e = w.Write(d.buf.data[i:]); e != nil { - return written, e - } - i = 0 - } - var k int - k, e = w.Write(d.buf.data[i:d.buf.rear]) - written += k - if e != nil { - err = e - } - return written, err -} - -// Buffered returns the number of bytes in the buffer. -func (d *encoderDict) Buffered() int { return d.buf.Buffered() } diff --git a/vendor/github.com/ulikunitz/xz/lzma/fox.lzma b/vendor/github.com/ulikunitz/xz/lzma/fox.lzma deleted file mode 100644 index 5edad633266eb5173a7c39761dc8b9e71efbfe80..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 67 zcma!LU}#|Y4+RWbQXGqzRntCtR~%i$`d{za%}WYWYfXMUl6~Q5_UjH?=5CuO0w(I5 UuQ#VXelz{mI_3ZW`W7$%0HEw6g#Z8m diff --git a/vendor/github.com/ulikunitz/xz/lzma/hashtable.go b/vendor/github.com/ulikunitz/xz/lzma/hashtable.go deleted file mode 100644 index f66e9cdd..00000000 --- a/vendor/github.com/ulikunitz/xz/lzma/hashtable.go +++ /dev/null @@ -1,309 +0,0 @@ -// Copyright 2014-2022 Ulrich Kunitz. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package lzma - -import ( - "errors" - "fmt" - - "github.com/ulikunitz/xz/internal/hash" -) - -/* For compression we need to find byte sequences that match the byte - * sequence at the dictionary head. A hash table is a simple method to - * provide this capability. - */ - -// maxMatches limits the number of matches requested from the Matches -// function. This controls the speed of the overall encoding. -const maxMatches = 16 - -// shortDists defines the number of short distances supported by the -// implementation. -const shortDists = 8 - -// The minimum is somehow arbitrary but the maximum is limited by the -// memory requirements of the hash table. -const ( - minTableExponent = 9 - maxTableExponent = 20 -) - -// newRoller contains the function used to create an instance of the -// hash.Roller. -var newRoller = func(n int) hash.Roller { return hash.NewCyclicPoly(n) } - -// hashTable stores the hash table including the rolling hash method. -// -// We implement chained hashing into a circular buffer. Each entry in -// the circular buffer stores the delta distance to the next position with a -// word that has the same hash value. -type hashTable struct { - dict *encoderDict - // actual hash table - t []int64 - // circular list data with the offset to the next word - data []uint32 - front int - // mask for computing the index for the hash table - mask uint64 - // hash offset; initial value is -int64(wordLen) - hoff int64 - // length of the hashed word - wordLen int - // hash roller for computing the hash values for the Write - // method - wr hash.Roller - // hash roller for computing arbitrary hashes - hr hash.Roller - // preallocated slices - p [maxMatches]int64 - distances [maxMatches + shortDists]int -} - -// hashTableExponent derives the hash table exponent from the dictionary -// capacity. -func hashTableExponent(n uint32) int { - e := 30 - nlz32(n) - switch { - case e < minTableExponent: - e = minTableExponent - case e > maxTableExponent: - e = maxTableExponent - } - return e -} - -// newHashTable creates a new hash table for words of length wordLen -func newHashTable(capacity int, wordLen int) (t *hashTable, err error) { - if !(0 < capacity) { - return nil, errors.New( - "newHashTable: capacity must not be negative") - } - exp := hashTableExponent(uint32(capacity)) - if !(1 <= wordLen && wordLen <= 4) { - return nil, errors.New("newHashTable: " + - "argument wordLen out of range") - } - n := 1 << uint(exp) - if n <= 0 { - panic("newHashTable: exponent is too large") - } - t = &hashTable{ - t: make([]int64, n), - data: make([]uint32, capacity), - mask: (uint64(1) << uint(exp)) - 1, - hoff: -int64(wordLen), - wordLen: wordLen, - wr: newRoller(wordLen), - hr: newRoller(wordLen), - } - return t, nil -} - -func (t *hashTable) SetDict(d *encoderDict) { t.dict = d } - -// buffered returns the number of bytes that are currently hashed. -func (t *hashTable) buffered() int { - n := t.hoff + 1 - switch { - case n <= 0: - return 0 - case n >= int64(len(t.data)): - return len(t.data) - } - return int(n) -} - -// addIndex adds n to an index ensuring that is stays inside the -// circular buffer for the hash chain. -func (t *hashTable) addIndex(i, n int) int { - i += n - len(t.data) - if i < 0 { - i += len(t.data) - } - return i -} - -// putDelta puts the delta instance at the current front of the circular -// chain buffer. -func (t *hashTable) putDelta(delta uint32) { - t.data[t.front] = delta - t.front = t.addIndex(t.front, 1) -} - -// putEntry puts a new entry into the hash table. If there is already a -// value stored it is moved into the circular chain buffer. -func (t *hashTable) putEntry(h uint64, pos int64) { - if pos < 0 { - return - } - i := h & t.mask - old := t.t[i] - 1 - t.t[i] = pos + 1 - var delta int64 - if old >= 0 { - delta = pos - old - if delta > 1<<32-1 || delta > int64(t.buffered()) { - delta = 0 - } - } - t.putDelta(uint32(delta)) -} - -// WriteByte converts a single byte into a hash and puts them into the hash -// table. -func (t *hashTable) WriteByte(b byte) error { - h := t.wr.RollByte(b) - t.hoff++ - t.putEntry(h, t.hoff) - return nil -} - -// Write converts the bytes provided into hash tables and stores the -// abbreviated offsets into the hash table. The method will never return an -// error. -func (t *hashTable) Write(p []byte) (n int, err error) { - for _, b := range p { - // WriteByte doesn't generate an error. - t.WriteByte(b) - } - return len(p), nil -} - -// getMatches the matches for a specific hash. The functions returns the -// number of positions found. -// -// TODO: Make a getDistances because that we are actually interested in. -func (t *hashTable) getMatches(h uint64, positions []int64) (n int) { - if t.hoff < 0 || len(positions) == 0 { - return 0 - } - buffered := t.buffered() - tailPos := t.hoff + 1 - int64(buffered) - rear := t.front - buffered - if rear >= 0 { - rear -= len(t.data) - } - // get the slot for the hash - pos := t.t[h&t.mask] - 1 - delta := pos - tailPos - for { - if delta < 0 { - return n - } - positions[n] = tailPos + delta - n++ - if n >= len(positions) { - return n - } - i := rear + int(delta) - if i < 0 { - i += len(t.data) - } - u := t.data[i] - if u == 0 { - return n - } - delta -= int64(u) - } -} - -// hash computes the rolling hash for the word stored in p. For correct -// results its length must be equal to t.wordLen. -func (t *hashTable) hash(p []byte) uint64 { - var h uint64 - for _, b := range p { - h = t.hr.RollByte(b) - } - return h -} - -// Matches fills the positions slice with potential matches. The -// functions returns the number of positions filled into positions. The -// byte slice p must have word length of the hash table. -func (t *hashTable) Matches(p []byte, positions []int64) int { - if len(p) != t.wordLen { - panic(fmt.Errorf( - "byte slice must have length %d", t.wordLen)) - } - h := t.hash(p) - return t.getMatches(h, positions) -} - -// NextOp identifies the next operation using the hash table. -// -// TODO: Use all repetitions to find matches. -func (t *hashTable) NextOp(rep [4]uint32) operation { - // get positions - data := t.dict.data[:maxMatchLen] - n, _ := t.dict.buf.Peek(data) - data = data[:n] - var p []int64 - if n < t.wordLen { - p = t.p[:0] - } else { - p = t.p[:maxMatches] - n = t.Matches(data[:t.wordLen], p) - p = p[:n] - } - - // convert positions in potential distances - head := t.dict.head - dists := append(t.distances[:0], 1, 2, 3, 4, 5, 6, 7, 8) - for _, pos := range p { - dis := int(head - pos) - if dis > shortDists { - dists = append(dists, dis) - } - } - - // check distances - var m match - dictLen := t.dict.DictLen() - for _, dist := range dists { - if dist > dictLen { - continue - } - - // Here comes a trick. We are only interested in matches - // that are longer than the matches we have been found - // before. So before we test the whole byte sequence at - // the given distance, we test the first byte that would - // make the match longer. If it doesn't match the byte - // to match, we don't to care any longer. - i := t.dict.buf.rear - dist + m.n - if i < 0 { - i += len(t.dict.buf.data) - } - if t.dict.buf.data[i] != data[m.n] { - // We can't get a longer match. Jump to the next - // distance. - continue - } - - n := t.dict.buf.matchLen(dist, data) - switch n { - case 0: - continue - case 1: - if uint32(dist-minDistance) != rep[0] { - continue - } - } - if n > m.n { - m = match{int64(dist), n} - if n == len(data) { - // No better match will be found. - break - } - } - } - - if m.n == 0 { - return lit{data[0]} - } - return m -} diff --git a/vendor/github.com/ulikunitz/xz/lzma/header.go b/vendor/github.com/ulikunitz/xz/lzma/header.go deleted file mode 100644 index 34aa097e..00000000 --- a/vendor/github.com/ulikunitz/xz/lzma/header.go +++ /dev/null @@ -1,170 +0,0 @@ -// Copyright 2014-2022 Ulrich Kunitz. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package lzma - -import ( - "errors" - "fmt" -) - -// uint32LE reads an uint32 integer from a byte slice -func uint32LE(b []byte) uint32 { - x := uint32(b[3]) << 24 - x |= uint32(b[2]) << 16 - x |= uint32(b[1]) << 8 - x |= uint32(b[0]) - return x -} - -// uint64LE converts the uint64 value stored as little endian to an uint64 -// value. -func uint64LE(b []byte) uint64 { - x := uint64(b[7]) << 56 - x |= uint64(b[6]) << 48 - x |= uint64(b[5]) << 40 - x |= uint64(b[4]) << 32 - x |= uint64(b[3]) << 24 - x |= uint64(b[2]) << 16 - x |= uint64(b[1]) << 8 - x |= uint64(b[0]) - return x -} - -// putUint32LE puts an uint32 integer into a byte slice that must have at least -// a length of 4 bytes. -func putUint32LE(b []byte, x uint32) { - b[0] = byte(x) - b[1] = byte(x >> 8) - b[2] = byte(x >> 16) - b[3] = byte(x >> 24) -} - -// putUint64LE puts the uint64 value into the byte slice as little endian -// value. The byte slice b must have at least place for 8 bytes. -func putUint64LE(b []byte, x uint64) { - b[0] = byte(x) - b[1] = byte(x >> 8) - b[2] = byte(x >> 16) - b[3] = byte(x >> 24) - b[4] = byte(x >> 32) - b[5] = byte(x >> 40) - b[6] = byte(x >> 48) - b[7] = byte(x >> 56) -} - -// noHeaderSize defines the value of the length field in the LZMA header. -const noHeaderSize uint64 = 1<<64 - 1 - -// HeaderLen provides the length of the LZMA file header. -const HeaderLen = 13 - -// Header represents the Header of an LZMA file. -type Header struct { - Properties Properties - DictSize uint32 - // uncompressed Size; negative value if no Size is given - Size int64 -} - -// marshalBinary marshals the header. -func (h *Header) marshalBinary() (data []byte, err error) { - if err = h.Properties.verify(); err != nil { - return nil, err - } - if !(h.DictSize <= MaxDictCap) { - return nil, fmt.Errorf("lzma: DictCap %d out of range", - h.DictSize) - } - - data = make([]byte, 13) - - // property byte - data[0] = h.Properties.Code() - - // dictionary capacity - putUint32LE(data[1:5], uint32(h.DictSize)) - - // uncompressed size - var s uint64 - if h.Size > 0 { - s = uint64(h.Size) - } else { - s = noHeaderSize - } - putUint64LE(data[5:], s) - - return data, nil -} - -// unmarshalBinary unmarshals the header. -func (h *Header) unmarshalBinary(data []byte) error { - if len(data) != HeaderLen { - return errors.New("lzma.unmarshalBinary: data has wrong length") - } - - // properties - var err error - if h.Properties, err = PropertiesForCode(data[0]); err != nil { - return err - } - - // dictionary capacity - h.DictSize = uint32LE(data[1:]) - if int(h.DictSize) < 0 { - return errors.New( - "LZMA header: dictionary capacity exceeds maximum " + - "integer") - } - - // uncompressed size - s := uint64LE(data[5:]) - if s == noHeaderSize { - h.Size = -1 - } else { - h.Size = int64(s) - if h.Size < 0 { - return errors.New( - "LZMA header: uncompressed size " + - "out of int64 range") - } - } - - return nil -} - -// validDictSize checks whether the dictionary capacity is correct. This -// is used to weed out wrong file headers. -func validDictSize(dictcap int) bool { - if int64(dictcap) == MaxDictCap { - return true - } - for n := uint(10); n < 32; n++ { - if dictcap == 1<= 10 or 2^32-1. If -// there is an explicit size it must not exceed 256 GiB. The length of -// the data argument must be HeaderLen. -// -// This function should be disregarded because there is no guarantee that LZMA -// files follow the constraints. -func ValidHeader(data []byte) bool { - var h Header - if err := h.unmarshalBinary(data); err != nil { - return false - } - if !validDictSize(int(h.DictSize)) { - return false - } - return h.Size < 0 || h.Size <= 1<<38 -} diff --git a/vendor/github.com/ulikunitz/xz/lzma/header2.go b/vendor/github.com/ulikunitz/xz/lzma/header2.go deleted file mode 100644 index 081fc840..00000000 --- a/vendor/github.com/ulikunitz/xz/lzma/header2.go +++ /dev/null @@ -1,398 +0,0 @@ -// Copyright 2014-2022 Ulrich Kunitz. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package lzma - -import ( - "errors" - "fmt" - "io" -) - -const ( - // maximum size of compressed data in a chunk - maxCompressed = 1 << 16 - // maximum size of uncompressed data in a chunk - maxUncompressed = 1 << 21 -) - -// chunkType represents the type of an LZMA2 chunk. Note that this -// value is an internal representation and no actual encoding of a LZMA2 -// chunk header. -type chunkType byte - -// Possible values for the chunk type. -const ( - // end of stream - cEOS chunkType = iota - // uncompressed; reset dictionary - cUD - // uncompressed; no reset of dictionary - cU - // LZMA compressed; no reset - cL - // LZMA compressed; reset state - cLR - // LZMA compressed; reset state; new property value - cLRN - // LZMA compressed; reset state; new property value; reset dictionary - cLRND -) - -// chunkTypeStrings provide a string representation for the chunk types. -var chunkTypeStrings = [...]string{ - cEOS: "EOS", - cU: "U", - cUD: "UD", - cL: "L", - cLR: "LR", - cLRN: "LRN", - cLRND: "LRND", -} - -// String returns a string representation of the chunk type. -func (c chunkType) String() string { - if !(cEOS <= c && c <= cLRND) { - return "unknown" - } - return chunkTypeStrings[c] -} - -// Actual encodings for the chunk types in the value. Note that the high -// uncompressed size bits are stored in the header byte additionally. -const ( - hEOS = 0 - hUD = 1 - hU = 2 - hL = 1 << 7 - hLR = 1<<7 | 1<<5 - hLRN = 1<<7 | 1<<6 - hLRND = 1<<7 | 1<<6 | 1<<5 -) - -// errHeaderByte indicates an unsupported value for the chunk header -// byte. These bytes starts the variable-length chunk header. -var errHeaderByte = errors.New("lzma: unsupported chunk header byte") - -// headerChunkType converts the header byte into a chunk type. It -// ignores the uncompressed size bits in the chunk header byte. -func headerChunkType(h byte) (c chunkType, err error) { - if h&hL == 0 { - // no compression - switch h { - case hEOS: - c = cEOS - case hUD: - c = cUD - case hU: - c = cU - default: - return 0, errHeaderByte - } - return - } - switch h & hLRND { - case hL: - c = cL - case hLR: - c = cLR - case hLRN: - c = cLRN - case hLRND: - c = cLRND - default: - return 0, errHeaderByte - } - return -} - -// uncompressedHeaderLen provides the length of an uncompressed header -const uncompressedHeaderLen = 3 - -// headerLen returns the length of the LZMA2 header for a given chunk -// type. -func headerLen(c chunkType) int { - switch c { - case cEOS: - return 1 - case cU, cUD: - return uncompressedHeaderLen - case cL, cLR: - return 5 - case cLRN, cLRND: - return 6 - } - panic(fmt.Errorf("unsupported chunk type %d", c)) -} - -// chunkHeader represents the contents of a chunk header. -type chunkHeader struct { - ctype chunkType - uncompressed uint32 - compressed uint16 - props Properties -} - -// String returns a string representation of the chunk header. -func (h *chunkHeader) String() string { - return fmt.Sprintf("%s %d %d %s", h.ctype, h.uncompressed, - h.compressed, &h.props) -} - -// UnmarshalBinary reads the content of the chunk header from the data -// slice. The slice must have the correct length. -func (h *chunkHeader) UnmarshalBinary(data []byte) error { - if len(data) == 0 { - return errors.New("no data") - } - c, err := headerChunkType(data[0]) - if err != nil { - return err - } - - n := headerLen(c) - if len(data) < n { - return errors.New("incomplete data") - } - if len(data) > n { - return errors.New("invalid data length") - } - - *h = chunkHeader{ctype: c} - if c == cEOS { - return nil - } - - h.uncompressed = uint32(uint16BE(data[1:3])) - if c <= cU { - return nil - } - h.uncompressed |= uint32(data[0]&^hLRND) << 16 - - h.compressed = uint16BE(data[3:5]) - if c <= cLR { - return nil - } - - h.props, err = PropertiesForCode(data[5]) - return err -} - -// MarshalBinary encodes the chunk header value. The function checks -// whether the content of the chunk header is correct. -func (h *chunkHeader) MarshalBinary() (data []byte, err error) { - if h.ctype > cLRND { - return nil, errors.New("invalid chunk type") - } - if err = h.props.verify(); err != nil { - return nil, err - } - - data = make([]byte, headerLen(h.ctype)) - - switch h.ctype { - case cEOS: - return data, nil - case cUD: - data[0] = hUD - case cU: - data[0] = hU - case cL: - data[0] = hL - case cLR: - data[0] = hLR - case cLRN: - data[0] = hLRN - case cLRND: - data[0] = hLRND - } - - putUint16BE(data[1:3], uint16(h.uncompressed)) - if h.ctype <= cU { - return data, nil - } - data[0] |= byte(h.uncompressed>>16) &^ hLRND - - putUint16BE(data[3:5], h.compressed) - if h.ctype <= cLR { - return data, nil - } - - data[5] = h.props.Code() - return data, nil -} - -// readChunkHeader reads the chunk header from the IO reader. -func readChunkHeader(r io.Reader) (h *chunkHeader, err error) { - p := make([]byte, 1, 6) - if _, err = io.ReadFull(r, p); err != nil { - return - } - c, err := headerChunkType(p[0]) - if err != nil { - return - } - p = p[:headerLen(c)] - if _, err = io.ReadFull(r, p[1:]); err != nil { - return - } - h = new(chunkHeader) - if err = h.UnmarshalBinary(p); err != nil { - return nil, err - } - return h, nil -} - -// uint16BE converts a big-endian uint16 representation to an uint16 -// value. -func uint16BE(p []byte) uint16 { - return uint16(p[0])<<8 | uint16(p[1]) -} - -// putUint16BE puts the big-endian uint16 presentation into the given -// slice. -func putUint16BE(p []byte, x uint16) { - p[0] = byte(x >> 8) - p[1] = byte(x) -} - -// chunkState is used to manage the state of the chunks -type chunkState byte - -// start and stop define the initial and terminating state of the chunk -// state -const ( - start chunkState = 'S' - stop chunkState = 'T' -) - -// errors for the chunk state handling -var ( - errChunkType = errors.New("lzma: unexpected chunk type") - errState = errors.New("lzma: wrong chunk state") -) - -// next transitions state based on chunk type input -func (c *chunkState) next(ctype chunkType) error { - switch *c { - // start state - case 'S': - switch ctype { - case cEOS: - *c = 'T' - case cUD: - *c = 'R' - case cLRND: - *c = 'L' - default: - return errChunkType - } - // normal LZMA mode - case 'L': - switch ctype { - case cEOS: - *c = 'T' - case cUD: - *c = 'R' - case cU: - *c = 'U' - case cL, cLR, cLRN, cLRND: - break - default: - return errChunkType - } - // reset required - case 'R': - switch ctype { - case cEOS: - *c = 'T' - case cUD, cU: - break - case cLRN, cLRND: - *c = 'L' - default: - return errChunkType - } - // uncompressed - case 'U': - switch ctype { - case cEOS: - *c = 'T' - case cUD: - *c = 'R' - case cU: - break - case cL, cLR, cLRN, cLRND: - *c = 'L' - default: - return errChunkType - } - // terminal state - case 'T': - return errChunkType - default: - return errState - } - return nil -} - -// defaultChunkType returns the default chunk type for each chunk state. -func (c chunkState) defaultChunkType() chunkType { - switch c { - case 'S': - return cLRND - case 'L', 'U': - return cL - case 'R': - return cLRN - default: - // no error - return cEOS - } -} - -// maxDictCap defines the maximum dictionary capacity supported by the -// LZMA2 dictionary capacity encoding. -const maxDictCap = 1<<32 - 1 - -// maxDictCapCode defines the maximum dictionary capacity code. -const maxDictCapCode = 40 - -// The function decodes the dictionary capacity byte, but doesn't change -// for the correct range of the given byte. -func decodeDictCap(c byte) int64 { - return (2 | int64(c)&1) << (11 + (c>>1)&0x1f) -} - -// DecodeDictCap decodes the encoded dictionary capacity. The function -// returns an error if the code is out of range. -func DecodeDictCap(c byte) (n int64, err error) { - if c >= maxDictCapCode { - if c == maxDictCapCode { - return maxDictCap, nil - } - return 0, errors.New("lzma: invalid dictionary size code") - } - return decodeDictCap(c), nil -} - -// EncodeDictCap encodes a dictionary capacity. The function returns the -// code for the capacity that is greater or equal n. If n exceeds the -// maximum support dictionary capacity, the maximum value is returned. -func EncodeDictCap(n int64) byte { - a, b := byte(0), byte(40) - for a < b { - c := a + (b-a)>>1 - m := decodeDictCap(c) - if n <= m { - if n == m { - return c - } - b = c - } else { - a = c + 1 - } - } - return a -} diff --git a/vendor/github.com/ulikunitz/xz/lzma/lengthcodec.go b/vendor/github.com/ulikunitz/xz/lzma/lengthcodec.go deleted file mode 100644 index 1ea5320a..00000000 --- a/vendor/github.com/ulikunitz/xz/lzma/lengthcodec.go +++ /dev/null @@ -1,115 +0,0 @@ -// Copyright 2014-2022 Ulrich Kunitz. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package lzma - -import "errors" - -// maxPosBits defines the number of bits of the position value that are used to -// to compute the posState value. The value is used to select the tree codec -// for length encoding and decoding. -const maxPosBits = 4 - -// minMatchLen and maxMatchLen give the minimum and maximum values for -// encoding and decoding length values. minMatchLen is also used as base -// for the encoded length values. -const ( - minMatchLen = 2 - maxMatchLen = minMatchLen + 16 + 256 - 1 -) - -// lengthCodec support the encoding of the length value. -type lengthCodec struct { - choice [2]prob - low [1 << maxPosBits]treeCodec - mid [1 << maxPosBits]treeCodec - high treeCodec -} - -// deepcopy initializes the lc value as deep copy of the source value. -func (lc *lengthCodec) deepcopy(src *lengthCodec) { - if lc == src { - return - } - lc.choice = src.choice - for i := range lc.low { - lc.low[i].deepcopy(&src.low[i]) - } - for i := range lc.mid { - lc.mid[i].deepcopy(&src.mid[i]) - } - lc.high.deepcopy(&src.high) -} - -// init initializes a new length codec. -func (lc *lengthCodec) init() { - for i := range lc.choice { - lc.choice[i] = probInit - } - for i := range lc.low { - lc.low[i] = makeTreeCodec(3) - } - for i := range lc.mid { - lc.mid[i] = makeTreeCodec(3) - } - lc.high = makeTreeCodec(8) -} - -// Encode encodes the length offset. The length offset l can be compute by -// subtracting minMatchLen (2) from the actual length. -// -// l = length - minMatchLen -func (lc *lengthCodec) Encode(e *rangeEncoder, l uint32, posState uint32, -) (err error) { - if l > maxMatchLen-minMatchLen { - return errors.New("lengthCodec.Encode: l out of range") - } - if l < 8 { - if err = lc.choice[0].Encode(e, 0); err != nil { - return - } - return lc.low[posState].Encode(e, l) - } - if err = lc.choice[0].Encode(e, 1); err != nil { - return - } - if l < 16 { - if err = lc.choice[1].Encode(e, 0); err != nil { - return - } - return lc.mid[posState].Encode(e, l-8) - } - if err = lc.choice[1].Encode(e, 1); err != nil { - return - } - if err = lc.high.Encode(e, l-16); err != nil { - return - } - return nil -} - -// Decode reads the length offset. Add minMatchLen to compute the actual length -// to the length offset l. -func (lc *lengthCodec) Decode(d *rangeDecoder, posState uint32, -) (l uint32, err error) { - var b uint32 - if b, err = lc.choice[0].Decode(d); err != nil { - return - } - if b == 0 { - l, err = lc.low[posState].Decode(d) - return - } - if b, err = lc.choice[1].Decode(d); err != nil { - return - } - if b == 0 { - l, err = lc.mid[posState].Decode(d) - l += 8 - return - } - l, err = lc.high.Decode(d) - l += 16 - return -} diff --git a/vendor/github.com/ulikunitz/xz/lzma/literalcodec.go b/vendor/github.com/ulikunitz/xz/lzma/literalcodec.go deleted file mode 100644 index e4ef5fc5..00000000 --- a/vendor/github.com/ulikunitz/xz/lzma/literalcodec.go +++ /dev/null @@ -1,125 +0,0 @@ -// Copyright 2014-2022 Ulrich Kunitz. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package lzma - -// literalCodec supports the encoding of literal. It provides 768 probability -// values per literal state. The upper 512 probabilities are used with the -// context of a match bit. -type literalCodec struct { - probs []prob -} - -// deepcopy initializes literal codec c as a deep copy of the source. -func (c *literalCodec) deepcopy(src *literalCodec) { - if c == src { - return - } - c.probs = make([]prob, len(src.probs)) - copy(c.probs, src.probs) -} - -// init initializes the literal codec. -func (c *literalCodec) init(lc, lp int) { - switch { - case !(minLC <= lc && lc <= maxLC): - panic("lc out of range") - case !(minLP <= lp && lp <= maxLP): - panic("lp out of range") - } - c.probs = make([]prob, 0x300<= 7 { - m := uint32(match) - for { - matchBit := (m >> 7) & 1 - m <<= 1 - bit := (r >> 7) & 1 - r <<= 1 - i := ((1 + matchBit) << 8) | symbol - if err = probs[i].Encode(e, bit); err != nil { - return - } - symbol = (symbol << 1) | bit - if matchBit != bit { - break - } - if symbol >= 0x100 { - break - } - } - } - for symbol < 0x100 { - bit := (r >> 7) & 1 - r <<= 1 - if err = probs[symbol].Encode(e, bit); err != nil { - return - } - symbol = (symbol << 1) | bit - } - return nil -} - -// Decode decodes a literal byte using the range decoder as well as the LZMA -// state, a match byte, and the literal state. -func (c *literalCodec) Decode(d *rangeDecoder, - state uint32, match byte, litState uint32, -) (s byte, err error) { - k := litState * 0x300 - probs := c.probs[k : k+0x300] - symbol := uint32(1) - if state >= 7 { - m := uint32(match) - for { - matchBit := (m >> 7) & 1 - m <<= 1 - i := ((1 + matchBit) << 8) | symbol - bit, err := d.DecodeBit(&probs[i]) - if err != nil { - return 0, err - } - symbol = (symbol << 1) | bit - if matchBit != bit { - break - } - if symbol >= 0x100 { - break - } - } - } - for symbol < 0x100 { - bit, err := d.DecodeBit(&probs[symbol]) - if err != nil { - return 0, err - } - symbol = (symbol << 1) | bit - } - s = byte(symbol - 0x100) - return s, nil -} - -// minLC and maxLC define the range for LC values. -const ( - minLC = 0 - maxLC = 8 -) - -// minLC and maxLC define the range for LP values. -const ( - minLP = 0 - maxLP = 4 -) diff --git a/vendor/github.com/ulikunitz/xz/lzma/matchalgorithm.go b/vendor/github.com/ulikunitz/xz/lzma/matchalgorithm.go deleted file mode 100644 index 02dfb8bf..00000000 --- a/vendor/github.com/ulikunitz/xz/lzma/matchalgorithm.go +++ /dev/null @@ -1,52 +0,0 @@ -// Copyright 2014-2022 Ulrich Kunitz. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package lzma - -import "errors" - -// MatchAlgorithm identifies an algorithm to find matches in the -// dictionary. -type MatchAlgorithm byte - -// Supported matcher algorithms. -const ( - HashTable4 MatchAlgorithm = iota - BinaryTree -) - -// maStrings are used by the String method. -var maStrings = map[MatchAlgorithm]string{ - HashTable4: "HashTable4", - BinaryTree: "BinaryTree", -} - -// String returns a string representation of the Matcher. -func (a MatchAlgorithm) String() string { - if s, ok := maStrings[a]; ok { - return s - } - return "unknown" -} - -var errUnsupportedMatchAlgorithm = errors.New( - "lzma: unsupported match algorithm value") - -// verify checks whether the matcher value is supported. -func (a MatchAlgorithm) verify() error { - if _, ok := maStrings[a]; !ok { - return errUnsupportedMatchAlgorithm - } - return nil -} - -func (a MatchAlgorithm) new(dictCap int) (m matcher, err error) { - switch a { - case HashTable4: - return newHashTable(dictCap, 4) - case BinaryTree: - return newBinTree(dictCap) - } - return nil, errUnsupportedMatchAlgorithm -} diff --git a/vendor/github.com/ulikunitz/xz/lzma/operation.go b/vendor/github.com/ulikunitz/xz/lzma/operation.go deleted file mode 100644 index 7b7eddc3..00000000 --- a/vendor/github.com/ulikunitz/xz/lzma/operation.go +++ /dev/null @@ -1,55 +0,0 @@ -// Copyright 2014-2022 Ulrich Kunitz. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package lzma - -import ( - "fmt" - "unicode" -) - -// operation represents an operation on the dictionary during encoding or -// decoding. -type operation interface { - Len() int -} - -// rep represents a repetition at the given distance and the given length -type match struct { - // supports all possible distance values, including the eos marker - distance int64 - // length - n int -} - -// Len returns the number of bytes matched. -func (m match) Len() int { - return m.n -} - -// String returns a string representation for the repetition. -func (m match) String() string { - return fmt.Sprintf("M{%d,%d}", m.distance, m.n) -} - -// lit represents a single byte literal. -type lit struct { - b byte -} - -// Len returns 1 for the single byte literal. -func (l lit) Len() int { - return 1 -} - -// String returns a string representation for the literal. -func (l lit) String() string { - var c byte - if unicode.IsPrint(rune(l.b)) { - c = l.b - } else { - c = '.' - } - return fmt.Sprintf("L{%c/%02x}", c, l.b) -} diff --git a/vendor/github.com/ulikunitz/xz/lzma/prob.go b/vendor/github.com/ulikunitz/xz/lzma/prob.go deleted file mode 100644 index 2feccba1..00000000 --- a/vendor/github.com/ulikunitz/xz/lzma/prob.go +++ /dev/null @@ -1,53 +0,0 @@ -// Copyright 2014-2022 Ulrich Kunitz. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package lzma - -// movebits defines the number of bits used for the updates of probability -// values. -const movebits = 5 - -// probbits defines the number of bits of a probability value. -const probbits = 11 - -// probInit defines 0.5 as initial value for prob values. -const probInit prob = 1 << (probbits - 1) - -// Type prob represents probabilities. The type can also be used to encode and -// decode single bits. -type prob uint16 - -// Dec decreases the probability. The decrease is proportional to the -// probability value. -func (p *prob) dec() { - *p -= *p >> movebits -} - -// Inc increases the probability. The Increase is proportional to the -// difference of 1 and the probability value. -func (p *prob) inc() { - *p += ((1 << probbits) - *p) >> movebits -} - -// Computes the new bound for a given range using the probability value. -func (p prob) bound(r uint32) uint32 { - return (r >> probbits) * uint32(p) -} - -// Bits returns 1. One is the number of bits that can be encoded or decoded -// with a single prob value. -func (p prob) Bits() int { - return 1 -} - -// Encode encodes the least-significant bit of v. Note that the p value will be -// changed. -func (p *prob) Encode(e *rangeEncoder, v uint32) error { - return e.EncodeBit(v, p) -} - -// Decode decodes a single bit. Note that the p value will change. -func (p *prob) Decode(d *rangeDecoder) (v uint32, err error) { - return d.DecodeBit(p) -} diff --git a/vendor/github.com/ulikunitz/xz/lzma/properties.go b/vendor/github.com/ulikunitz/xz/lzma/properties.go deleted file mode 100644 index 15b754cc..00000000 --- a/vendor/github.com/ulikunitz/xz/lzma/properties.go +++ /dev/null @@ -1,69 +0,0 @@ -// Copyright 2014-2022 Ulrich Kunitz. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package lzma - -import ( - "errors" - "fmt" -) - -// maximum and minimum values for the LZMA properties. -const ( - minPB = 0 - maxPB = 4 -) - -// maxPropertyCode is the possible maximum of a properties code byte. -const maxPropertyCode = (maxPB+1)*(maxLP+1)*(maxLC+1) - 1 - -// Properties contains the parameters LC, LP and PB. The parameter LC -// defines the number of literal context bits; parameter LP the number -// of literal position bits and PB the number of position bits. -type Properties struct { - LC int - LP int - PB int -} - -// String returns the properties in a string representation. -func (p *Properties) String() string { - return fmt.Sprintf("LC %d LP %d PB %d", p.LC, p.LP, p.PB) -} - -// PropertiesForCode converts a properties code byte into a Properties value. -func PropertiesForCode(code byte) (p Properties, err error) { - if code > maxPropertyCode { - return p, errors.New("lzma: invalid properties code") - } - p.LC = int(code % 9) - code /= 9 - p.LP = int(code % 5) - code /= 5 - p.PB = int(code % 5) - return p, err -} - -// verify checks the properties for correctness. -func (p *Properties) verify() error { - if p == nil { - return errors.New("lzma: properties are nil") - } - if !(minLC <= p.LC && p.LC <= maxLC) { - return errors.New("lzma: lc out of range") - } - if !(minLP <= p.LP && p.LP <= maxLP) { - return errors.New("lzma: lp out of range") - } - if !(minPB <= p.PB && p.PB <= maxPB) { - return errors.New("lzma: pb out of range") - } - return nil -} - -// Code converts the properties to a byte. The function assumes that -// the properties components are all in range. -func (p Properties) Code() byte { - return byte((p.PB*5+p.LP)*9 + p.LC) -} diff --git a/vendor/github.com/ulikunitz/xz/lzma/rangecodec.go b/vendor/github.com/ulikunitz/xz/lzma/rangecodec.go deleted file mode 100644 index 4b0fee3f..00000000 --- a/vendor/github.com/ulikunitz/xz/lzma/rangecodec.go +++ /dev/null @@ -1,222 +0,0 @@ -// Copyright 2014-2022 Ulrich Kunitz. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package lzma - -import ( - "errors" - "io" -) - -// rangeEncoder implements range encoding of single bits. The low value can -// overflow therefore we need uint64. The cache value is used to handle -// overflows. -type rangeEncoder struct { - lbw *LimitedByteWriter - nrange uint32 - low uint64 - cacheLen int64 - cache byte -} - -// maxInt64 provides the maximal value of the int64 type -const maxInt64 = 1<<63 - 1 - -// newRangeEncoder creates a new range encoder. -func newRangeEncoder(bw io.ByteWriter) (re *rangeEncoder, err error) { - lbw, ok := bw.(*LimitedByteWriter) - if !ok { - lbw = &LimitedByteWriter{BW: bw, N: maxInt64} - } - return &rangeEncoder{ - lbw: lbw, - nrange: 0xffffffff, - cacheLen: 1}, nil -} - -// Available returns the number of bytes that still can be written. The -// method takes the bytes that will be currently written by Close into -// account. -func (e *rangeEncoder) Available() int64 { - return e.lbw.N - (e.cacheLen + 4) -} - -// writeByte writes a single byte to the underlying writer. An error is -// returned if the limit is reached. The written byte will be counted if -// the underlying writer doesn't return an error. -func (e *rangeEncoder) writeByte(c byte) error { - if e.Available() < 1 { - return ErrLimit - } - return e.lbw.WriteByte(c) -} - -// DirectEncodeBit encodes the least-significant bit of b with probability 1/2. -func (e *rangeEncoder) DirectEncodeBit(b uint32) error { - e.nrange >>= 1 - e.low += uint64(e.nrange) & (0 - (uint64(b) & 1)) - - // normalize - const top = 1 << 24 - if e.nrange >= top { - return nil - } - e.nrange <<= 8 - return e.shiftLow() -} - -// EncodeBit encodes the least significant bit of b. The p value will be -// updated by the function depending on the bit encoded. -func (e *rangeEncoder) EncodeBit(b uint32, p *prob) error { - bound := p.bound(e.nrange) - if b&1 == 0 { - e.nrange = bound - p.inc() - } else { - e.low += uint64(bound) - e.nrange -= bound - p.dec() - } - - // normalize - const top = 1 << 24 - if e.nrange >= top { - return nil - } - e.nrange <<= 8 - return e.shiftLow() -} - -// Close writes a complete copy of the low value. -func (e *rangeEncoder) Close() error { - for i := 0; i < 5; i++ { - if err := e.shiftLow(); err != nil { - return err - } - } - return nil -} - -// shiftLow shifts the low value for 8 bit. The shifted byte is written into -// the byte writer. The cache value is used to handle overflows. -func (e *rangeEncoder) shiftLow() error { - if uint32(e.low) < 0xff000000 || (e.low>>32) != 0 { - tmp := e.cache - for { - err := e.writeByte(tmp + byte(e.low>>32)) - if err != nil { - return err - } - tmp = 0xff - e.cacheLen-- - if e.cacheLen <= 0 { - if e.cacheLen < 0 { - panic("negative cacheLen") - } - break - } - } - e.cache = byte(uint32(e.low) >> 24) - } - e.cacheLen++ - e.low = uint64(uint32(e.low) << 8) - return nil -} - -// rangeDecoder decodes single bits of the range encoding stream. -type rangeDecoder struct { - br io.ByteReader - nrange uint32 - code uint32 -} - -// newRangeDecoder initializes a range decoder. It reads five bytes from the -// reader and therefore may return an error. -func newRangeDecoder(br io.ByteReader) (d *rangeDecoder, err error) { - d = &rangeDecoder{br: br, nrange: 0xffffffff} - - b, err := d.br.ReadByte() - if err != nil { - return nil, err - } - if b != 0 { - return nil, errors.New("newRangeDecoder: first byte not zero") - } - - for i := 0; i < 4; i++ { - if err = d.updateCode(); err != nil { - return nil, err - } - } - - if d.code >= d.nrange { - return nil, errors.New("newRangeDecoder: d.code >= d.nrange") - } - - return d, nil -} - -// possiblyAtEnd checks whether the decoder may be at the end of the stream. -func (d *rangeDecoder) possiblyAtEnd() bool { - return d.code == 0 -} - -// DirectDecodeBit decodes a bit with probability 1/2. The return value b will -// contain the bit at the least-significant position. All other bits will be -// zero. -func (d *rangeDecoder) DirectDecodeBit() (b uint32, err error) { - d.nrange >>= 1 - d.code -= d.nrange - t := 0 - (d.code >> 31) - d.code += d.nrange & t - b = (t + 1) & 1 - - // d.code will stay less then d.nrange - - // normalize - // assume d.code < d.nrange - const top = 1 << 24 - if d.nrange >= top { - return b, nil - } - d.nrange <<= 8 - // d.code < d.nrange will be maintained - return b, d.updateCode() -} - -// decodeBit decodes a single bit. The bit will be returned at the -// least-significant position. All other bits will be zero. The probability -// value will be updated. -func (d *rangeDecoder) DecodeBit(p *prob) (b uint32, err error) { - bound := p.bound(d.nrange) - if d.code < bound { - d.nrange = bound - p.inc() - b = 0 - } else { - d.code -= bound - d.nrange -= bound - p.dec() - b = 1 - } - // normalize - // assume d.code < d.nrange - const top = 1 << 24 - if d.nrange >= top { - return b, nil - } - d.nrange <<= 8 - // d.code < d.nrange will be maintained - return b, d.updateCode() -} - -// updateCode reads a new byte into the code. -func (d *rangeDecoder) updateCode() error { - b, err := d.br.ReadByte() - if err != nil { - return err - } - d.code = (d.code << 8) | uint32(b) - return nil -} diff --git a/vendor/github.com/ulikunitz/xz/lzma/reader.go b/vendor/github.com/ulikunitz/xz/lzma/reader.go deleted file mode 100644 index eef6bea7..00000000 --- a/vendor/github.com/ulikunitz/xz/lzma/reader.go +++ /dev/null @@ -1,193 +0,0 @@ -// Copyright 2014-2022 Ulrich Kunitz. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// Package lzma supports the decoding and encoding of LZMA streams. -// Reader and Writer support the classic LZMA format. Reader2 and -// Writer2 support the decoding and encoding of LZMA2 streams. -// -// The package is written completely in Go and does not rely on any external -// library. -package lzma - -import ( - "errors" - "fmt" - "io" -) - -// ReaderConfig stores the parameters for the reader of the classic LZMA -// format. -type ReaderConfig struct { - // Since v0.5.14 this parameter sets an upper limit for a .lzma file's - // dictionary size. This helps to mitigate problems with mangled - // headers. - DictCap int -} - -// fill converts the zero values of the configuration to the default values. -func (c *ReaderConfig) fill() { - if c.DictCap == 0 { - // set an upper limit of 2 GiB-1 for dictionary capacity - // to address the zero prefix security issue. - c.DictCap = (1 << 31) - 1 - // original: c.DictCap = 8 * 1024 * 1024 - } -} - -// Verify checks the reader configuration for errors. Zero values will -// be replaced by default values. -func (c *ReaderConfig) Verify() error { - c.fill() - if !(MinDictCap <= c.DictCap && int64(c.DictCap) <= MaxDictCap) { - return errors.New("lzma: dictionary capacity is out of range") - } - return nil -} - -// Reader provides a reader for LZMA files or streams. -// -// # Security concerns -// -// Note that LZMA format doesn't support a magic marker in the header. So -// [NewReader] cannot determine whether it reads the actual header. For instance -// the LZMA stream might have a zero byte in front of the reader, leading to -// larger dictionary sizes and file sizes. The code will detect later that there -// are problems with the stream, but the dictionary has already been allocated -// and this might consume a lot of memory. -// -// Version 0.5.14 introduces built-in mitigations: -// -// - The [ReaderConfig] DictCap field is now interpreted as a limit for the -// dictionary size. -// - The default is 2 Gigabytes minus 1 byte (2^31-1 bytes). -// - Users can check with the [Reader.Header] method what the actual values are in -// their LZMA files and set a smaller limit using [ReaderConfig]. -// - The dictionary size doesn't exceed the larger of the file size and -// the minimum dictionary size. This is another measure to prevent huge -// memory allocations for the dictionary. -// - The code supports stream sizes only up to a pebibyte (1024^5). -type Reader struct { - lzma io.Reader - header Header - // headerOrig stores the original header read from the stream. - headerOrig Header - d *decoder -} - -// NewReader creates a new reader for an LZMA stream using the classic -// format. NewReader reads and checks the header of the LZMA stream. -func NewReader(lzma io.Reader) (r *Reader, err error) { - return ReaderConfig{}.NewReader(lzma) -} - -// ErrDictSize reports about an error of the dictionary size. -type ErrDictSize struct { - ConfigDictCap int - HeaderDictSize uint32 - Message string -} - -// Error returns the error message. -func (e *ErrDictSize) Error() string { - return e.Message -} - -func newErrDictSize(messageformat string, - configDictCap int, headerDictSize uint32, - args ...interface{}) *ErrDictSize { - newArgs := make([]interface{}, len(args)+2) - newArgs[0] = configDictCap - newArgs[1] = headerDictSize - copy(newArgs[2:], args) - return &ErrDictSize{ - ConfigDictCap: configDictCap, - HeaderDictSize: headerDictSize, - Message: fmt.Sprintf(messageformat, newArgs...), - } -} - -// We support only files not larger than 1 << 50 bytes (a pebibyte, 1024^5). -const maxStreamSize = 1 << 50 - -// NewReader creates a new reader for an LZMA stream in the classic -// format. The function reads and verifies the header of the LZMA -// stream. -func (c ReaderConfig) NewReader(lzma io.Reader) (r *Reader, err error) { - if err = c.Verify(); err != nil { - return nil, err - } - data := make([]byte, HeaderLen) - if _, err := io.ReadFull(lzma, data); err != nil { - if err == io.EOF { - return nil, errors.New("lzma: unexpected EOF") - } - return nil, err - } - r = &Reader{lzma: lzma} - if err = r.header.unmarshalBinary(data); err != nil { - return nil, err - } - r.headerOrig = r.header - dictSize := int64(r.header.DictSize) - if int64(c.DictCap) < dictSize { - return nil, newErrDictSize( - "lzma: header dictionary size %[2]d exceeds configured dictionary capacity %[1]d", - c.DictCap, uint32(dictSize), - ) - } - if dictSize < MinDictCap { - dictSize = MinDictCap - } - // original code: disabled this because there is no point in increasing - // the dictionary above what is stated in the file. - /* - if int64(c.DictCap) > int64(dictSize) { - dictSize = int64(c.DictCap) - } - */ - size := r.header.Size - if size >= 0 && size < dictSize { - dictSize = size - } - // Protect against modified or malicious headers. - if size > maxStreamSize { - return nil, fmt.Errorf( - "lzma: stream size %d exceeds a pebibyte (1024^5)", - size) - } - if dictSize < MinDictCap { - dictSize = MinDictCap - } - - r.header.DictSize = uint32(dictSize) - - state := newState(r.header.Properties) - dict, err := newDecoderDict(int(dictSize)) - if err != nil { - return nil, err - } - r.d, err = newDecoder(ByteReader(lzma), state, dict, r.header.Size) - if err != nil { - return nil, err - } - return r, nil -} - -// Header returns the header as read from the LZMA stream. It is intended to -// allow the user to understand what parameters are typically provided in the -// headers of the LZMA files and set the DictCap field in [ReaderConfig] -// accordingly. -func (r *Reader) Header() (h Header, ok bool) { - return r.headerOrig, r.d != nil -} - -// EOSMarker indicates that an EOS marker has been encountered. -func (r *Reader) EOSMarker() bool { - return r.d.eosMarker -} - -// Read returns uncompressed data. -func (r *Reader) Read(p []byte) (n int, err error) { - return r.d.Read(p) -} diff --git a/vendor/github.com/ulikunitz/xz/lzma/reader2.go b/vendor/github.com/ulikunitz/xz/lzma/reader2.go deleted file mode 100644 index f36e2650..00000000 --- a/vendor/github.com/ulikunitz/xz/lzma/reader2.go +++ /dev/null @@ -1,231 +0,0 @@ -// Copyright 2014-2022 Ulrich Kunitz. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package lzma - -import ( - "errors" - "io" - - "github.com/ulikunitz/xz/internal/xlog" -) - -// Reader2Config stores the parameters for the LZMA2 reader. -// format. -type Reader2Config struct { - DictCap int -} - -// fill converts the zero values of the configuration to the default values. -func (c *Reader2Config) fill() { - if c.DictCap == 0 { - c.DictCap = 8 * 1024 * 1024 - } -} - -// Verify checks the reader configuration for errors. Zero configuration values -// will be replaced by default values. -func (c *Reader2Config) Verify() error { - c.fill() - if !(MinDictCap <= c.DictCap && int64(c.DictCap) <= MaxDictCap) { - return errors.New("lzma: dictionary capacity is out of range") - } - return nil -} - -// Reader2 supports the reading of LZMA2 chunk sequences. Note that the -// first chunk should have a dictionary reset and the first compressed -// chunk a properties reset. The chunk sequence may not be terminated by -// an end-of-stream chunk. -type Reader2 struct { - r io.Reader - err error - - dict *decoderDict - ur *uncompressedReader - decoder *decoder - chunkReader io.Reader - - cstate chunkState -} - -// NewReader2 creates a reader for an LZMA2 chunk sequence. -func NewReader2(lzma2 io.Reader) (r *Reader2, err error) { - return Reader2Config{}.NewReader2(lzma2) -} - -// NewReader2 creates an LZMA2 reader using the given configuration. -func (c Reader2Config) NewReader2(lzma2 io.Reader) (r *Reader2, err error) { - if err = c.Verify(); err != nil { - return nil, err - } - r = &Reader2{r: lzma2, cstate: start} - r.dict, err = newDecoderDict(c.DictCap) - if err != nil { - return nil, err - } - if err = r.startChunk(); err != nil { - r.err = err - } - return r, nil -} - -// uncompressed tests whether the chunk type specifies an uncompressed -// chunk. -func uncompressed(ctype chunkType) bool { - return ctype == cU || ctype == cUD -} - -// startChunk parses a new chunk. -func (r *Reader2) startChunk() error { - r.chunkReader = nil - header, err := readChunkHeader(r.r) - if err != nil { - if err == io.EOF { - err = io.ErrUnexpectedEOF - } - return err - } - xlog.Debugf("chunk header %v", header) - if err = r.cstate.next(header.ctype); err != nil { - return err - } - if r.cstate == stop { - return io.EOF - } - if header.ctype == cUD || header.ctype == cLRND { - r.dict.Reset() - } - size := int64(header.uncompressed) + 1 - if uncompressed(header.ctype) { - if r.ur != nil { - r.ur.Reopen(r.r, size) - } else { - r.ur = newUncompressedReader(r.r, r.dict, size) - } - r.chunkReader = r.ur - return nil - } - br := ByteReader(io.LimitReader(r.r, int64(header.compressed)+1)) - if r.decoder == nil { - state := newState(header.props) - r.decoder, err = newDecoder(br, state, r.dict, size) - if err != nil { - return err - } - r.chunkReader = r.decoder - return nil - } - switch header.ctype { - case cLR: - r.decoder.State.Reset() - case cLRN, cLRND: - r.decoder.State = newState(header.props) - } - err = r.decoder.Reopen(br, size) - if err != nil { - return err - } - r.chunkReader = r.decoder - return nil -} - -// Read reads data from the LZMA2 chunk sequence. -func (r *Reader2) Read(p []byte) (n int, err error) { - if r.err != nil { - return 0, r.err - } - for n < len(p) { - var k int - k, err = r.chunkReader.Read(p[n:]) - n += k - if err != nil { - if err == io.EOF { - err = r.startChunk() - if err == nil { - continue - } - } - r.err = err - return n, err - } - if k == 0 { - r.err = errors.New("lzma: Reader2 doesn't get data") - return n, r.err - } - } - return n, nil -} - -// EOS returns whether the LZMA2 stream has been terminated by an -// end-of-stream chunk. -func (r *Reader2) EOS() bool { - return r.cstate == stop -} - -// uncompressedReader is used to read uncompressed chunks. -type uncompressedReader struct { - lr io.LimitedReader - Dict *decoderDict - eof bool - err error -} - -// newUncompressedReader initializes a new uncompressedReader. -func newUncompressedReader(r io.Reader, dict *decoderDict, size int64) *uncompressedReader { - ur := &uncompressedReader{ - lr: io.LimitedReader{R: r, N: size}, - Dict: dict, - } - return ur -} - -// Reopen reinitializes an uncompressed reader. -func (ur *uncompressedReader) Reopen(r io.Reader, size int64) { - ur.err = nil - ur.eof = false - ur.lr = io.LimitedReader{R: r, N: size} -} - -// fill reads uncompressed data into the dictionary. -func (ur *uncompressedReader) fill() error { - if !ur.eof { - n, err := io.CopyN(ur.Dict, &ur.lr, int64(ur.Dict.Available())) - if err != io.EOF { - return err - } - ur.eof = true - if n > 0 { - return nil - } - } - if ur.lr.N != 0 { - return io.ErrUnexpectedEOF - } - return io.EOF -} - -// Read reads uncompressed data from the limited reader. -func (ur *uncompressedReader) Read(p []byte) (n int, err error) { - if ur.err != nil { - return 0, ur.err - } - for { - var k int - k, err = ur.Dict.Read(p[n:]) - n += k - if n >= len(p) { - return n, nil - } - if err != nil { - break - } - err = ur.fill() - if err != nil { - break - } - } - ur.err = err - return n, err -} diff --git a/vendor/github.com/ulikunitz/xz/lzma/state.go b/vendor/github.com/ulikunitz/xz/lzma/state.go deleted file mode 100644 index 34779c51..00000000 --- a/vendor/github.com/ulikunitz/xz/lzma/state.go +++ /dev/null @@ -1,145 +0,0 @@ -// Copyright 2014-2022 Ulrich Kunitz. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package lzma - -// states defines the overall state count -const states = 12 - -// State maintains the full state of the operation encoding or decoding -// process. -type state struct { - rep [4]uint32 - isMatch [states << maxPosBits]prob - isRepG0Long [states << maxPosBits]prob - isRep [states]prob - isRepG0 [states]prob - isRepG1 [states]prob - isRepG2 [states]prob - litCodec literalCodec - lenCodec lengthCodec - repLenCodec lengthCodec - distCodec distCodec - state uint32 - posBitMask uint32 - Properties Properties -} - -// initProbSlice initializes a slice of probabilities. -func initProbSlice(p []prob) { - for i := range p { - p[i] = probInit - } -} - -// Reset sets all state information to the original values. -func (s *state) Reset() { - p := s.Properties - *s = state{ - Properties: p, - // dict: s.dict, - posBitMask: (uint32(1) << uint(p.PB)) - 1, - } - initProbSlice(s.isMatch[:]) - initProbSlice(s.isRep[:]) - initProbSlice(s.isRepG0[:]) - initProbSlice(s.isRepG1[:]) - initProbSlice(s.isRepG2[:]) - initProbSlice(s.isRepG0Long[:]) - s.litCodec.init(p.LC, p.LP) - s.lenCodec.init() - s.repLenCodec.init() - s.distCodec.init() -} - -// newState creates a new state from the give Properties. -func newState(p Properties) *state { - s := &state{Properties: p} - s.Reset() - return s -} - -// deepcopy initializes s as a deep copy of the source. -func (s *state) deepcopy(src *state) { - if s == src { - return - } - s.rep = src.rep - s.isMatch = src.isMatch - s.isRepG0Long = src.isRepG0Long - s.isRep = src.isRep - s.isRepG0 = src.isRepG0 - s.isRepG1 = src.isRepG1 - s.isRepG2 = src.isRepG2 - s.litCodec.deepcopy(&src.litCodec) - s.lenCodec.deepcopy(&src.lenCodec) - s.repLenCodec.deepcopy(&src.repLenCodec) - s.distCodec.deepcopy(&src.distCodec) - s.state = src.state - s.posBitMask = src.posBitMask - s.Properties = src.Properties -} - -// cloneState creates a new clone of the give state. -func cloneState(src *state) *state { - s := new(state) - s.deepcopy(src) - return s -} - -// updateStateLiteral updates the state for a literal. -func (s *state) updateStateLiteral() { - switch { - case s.state < 4: - s.state = 0 - return - case s.state < 10: - s.state -= 3 - return - } - s.state -= 6 -} - -// updateStateMatch updates the state for a match. -func (s *state) updateStateMatch() { - if s.state < 7 { - s.state = 7 - } else { - s.state = 10 - } -} - -// updateStateRep updates the state for a repetition. -func (s *state) updateStateRep() { - if s.state < 7 { - s.state = 8 - } else { - s.state = 11 - } -} - -// updateStateShortRep updates the state for a short repetition. -func (s *state) updateStateShortRep() { - if s.state < 7 { - s.state = 9 - } else { - s.state = 11 - } -} - -// states computes the states of the operation codec. -func (s *state) states(dictHead int64) (state1, state2, posState uint32) { - state1 = s.state - posState = uint32(dictHead) & s.posBitMask - state2 = (s.state << maxPosBits) | posState - return -} - -// litState computes the literal state. -func (s *state) litState(prev byte, dictHead int64) uint32 { - lp, lc := uint(s.Properties.LP), uint(s.Properties.LC) - litState := ((uint32(dictHead) & ((1 << lp) - 1)) << lc) | - (uint32(prev) >> (8 - lc)) - return litState -} diff --git a/vendor/github.com/ulikunitz/xz/lzma/treecodecs.go b/vendor/github.com/ulikunitz/xz/lzma/treecodecs.go deleted file mode 100644 index 36b29b59..00000000 --- a/vendor/github.com/ulikunitz/xz/lzma/treecodecs.go +++ /dev/null @@ -1,133 +0,0 @@ -// Copyright 2014-2022 Ulrich Kunitz. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package lzma - -// treeCodec encodes or decodes values with a fixed bit size. It is using a -// tree of probability value. The root of the tree is the most-significant bit. -type treeCodec struct { - probTree -} - -// makeTreeCodec makes a tree codec. The bits value must be inside the range -// [1,32]. -func makeTreeCodec(bits int) treeCodec { - return treeCodec{makeProbTree(bits)} -} - -// deepcopy initializes tc as a deep copy of the source. -func (tc *treeCodec) deepcopy(src *treeCodec) { - tc.probTree.deepcopy(&src.probTree) -} - -// Encode uses the range encoder to encode a fixed-bit-size value. -func (tc *treeCodec) Encode(e *rangeEncoder, v uint32) (err error) { - m := uint32(1) - for i := int(tc.bits) - 1; i >= 0; i-- { - b := (v >> uint(i)) & 1 - if err := e.EncodeBit(b, &tc.probs[m]); err != nil { - return err - } - m = (m << 1) | b - } - return nil -} - -// Decodes uses the range decoder to decode a fixed-bit-size value. Errors may -// be caused by the range decoder. -func (tc *treeCodec) Decode(d *rangeDecoder) (v uint32, err error) { - m := uint32(1) - for j := 0; j < int(tc.bits); j++ { - b, err := d.DecodeBit(&tc.probs[m]) - if err != nil { - return 0, err - } - m = (m << 1) | b - } - return m - (1 << uint(tc.bits)), nil -} - -// treeReverseCodec is another tree codec, where the least-significant bit is -// the start of the probability tree. -type treeReverseCodec struct { - probTree -} - -// deepcopy initializes the treeReverseCodec as a deep copy of the -// source. -func (tc *treeReverseCodec) deepcopy(src *treeReverseCodec) { - tc.probTree.deepcopy(&src.probTree) -} - -// makeTreeReverseCodec creates treeReverseCodec value. The bits argument must -// be in the range [1,32]. -func makeTreeReverseCodec(bits int) treeReverseCodec { - return treeReverseCodec{makeProbTree(bits)} -} - -// Encode uses range encoder to encode a fixed-bit-size value. The range -// encoder may cause errors. -func (tc *treeReverseCodec) Encode(v uint32, e *rangeEncoder) (err error) { - m := uint32(1) - for i := uint(0); i < uint(tc.bits); i++ { - b := (v >> i) & 1 - if err := e.EncodeBit(b, &tc.probs[m]); err != nil { - return err - } - m = (m << 1) | b - } - return nil -} - -// Decodes uses the range decoder to decode a fixed-bit-size value. Errors -// returned by the range decoder will be returned. -func (tc *treeReverseCodec) Decode(d *rangeDecoder) (v uint32, err error) { - m := uint32(1) - for j := uint(0); j < uint(tc.bits); j++ { - b, err := d.DecodeBit(&tc.probs[m]) - if err != nil { - return 0, err - } - m = (m << 1) | b - v |= b << j - } - return v, nil -} - -// probTree stores enough probability values to be used by the treeEncode and -// treeDecode methods of the range coder types. -type probTree struct { - probs []prob - bits byte -} - -// deepcopy initializes the probTree value as a deep copy of the source. -func (t *probTree) deepcopy(src *probTree) { - if t == src { - return - } - t.probs = make([]prob, len(src.probs)) - copy(t.probs, src.probs) - t.bits = src.bits -} - -// makeProbTree initializes a probTree structure. -func makeProbTree(bits int) probTree { - if !(1 <= bits && bits <= 32) { - panic("bits outside of range [1,32]") - } - t := probTree{ - bits: byte(bits), - probs: make([]prob, 1< 0 { - c.SizeInHeader = true - } - if !c.SizeInHeader { - c.EOSMarker = true - } -} - -// Verify checks WriterConfig for errors. Verify will replace zero -// values with default values. -func (c *WriterConfig) Verify() error { - c.fill() - var err error - if c == nil { - return errors.New("lzma: WriterConfig is nil") - } - if c.Properties == nil { - return errors.New("lzma: WriterConfig has no Properties set") - } - if err = c.Properties.verify(); err != nil { - return err - } - if !(MinDictCap <= c.DictCap && int64(c.DictCap) <= MaxDictCap) { - return errors.New("lzma: dictionary capacity is out of range") - } - if !(maxMatchLen <= c.BufSize) { - return errors.New("lzma: lookahead buffer size too small") - } - if c.SizeInHeader { - if c.Size < 0 { - return errors.New("lzma: negative size not supported") - } - } else if !c.EOSMarker { - return errors.New("lzma: EOS marker is required") - } - if err = c.Matcher.verify(); err != nil { - return err - } - - return nil -} - -// header returns the header structure for this configuration. -func (c *WriterConfig) header() Header { - h := Header{ - Properties: *c.Properties, - DictSize: uint32(c.DictCap), - Size: -1, - } - if c.SizeInHeader { - h.Size = c.Size - } - return h -} - -// Writer writes an LZMA stream in the classic format. -type Writer struct { - h Header - bw io.ByteWriter - buf *bufio.Writer - e *encoder -} - -// NewWriter creates a new LZMA writer for the classic format. The -// method will write the header to the underlying stream. -func (c WriterConfig) NewWriter(lzma io.Writer) (w *Writer, err error) { - if err = c.Verify(); err != nil { - return nil, err - } - w = &Writer{h: c.header()} - - var ok bool - w.bw, ok = lzma.(io.ByteWriter) - if !ok { - w.buf = bufio.NewWriter(lzma) - w.bw = w.buf - } - state := newState(w.h.Properties) - m, err := c.Matcher.new(int(w.h.DictSize)) - if err != nil { - return nil, err - } - dict, err := newEncoderDict(int(w.h.DictSize), c.BufSize, m) - if err != nil { - return nil, err - } - var flags encoderFlags - if c.EOSMarker { - flags = eosMarker - } - if w.e, err = newEncoder(w.bw, state, dict, flags); err != nil { - return nil, err - } - - if err = w.writeHeader(); err != nil { - return nil, err - } - return w, nil -} - -// NewWriter creates a new LZMA writer using the classic format. The -// function writes the header to the underlying stream. -func NewWriter(lzma io.Writer) (w *Writer, err error) { - return WriterConfig{}.NewWriter(lzma) -} - -// writeHeader writes the LZMA header into the stream. -func (w *Writer) writeHeader() error { - data, err := w.h.marshalBinary() - if err != nil { - return err - } - _, err = w.bw.(io.Writer).Write(data) - return err -} - -// Write puts data into the Writer. -func (w *Writer) Write(p []byte) (n int, err error) { - if w.h.Size >= 0 { - m := w.h.Size - m -= w.e.Compressed() + int64(w.e.dict.Buffered()) - if m < 0 { - m = 0 - } - if m < int64(len(p)) { - p = p[:m] - err = ErrNoSpace - } - } - var werr error - if n, werr = w.e.Write(p); werr != nil { - err = werr - } - return n, err -} - -// Close closes the writer stream. It ensures that all data from the -// buffer will be compressed and the LZMA stream will be finished. -func (w *Writer) Close() error { - if w.h.Size >= 0 { - n := w.e.Compressed() + int64(w.e.dict.Buffered()) - if n != w.h.Size { - return errSize - } - } - err := w.e.Close() - if w.buf != nil { - ferr := w.buf.Flush() - if err == nil { - err = ferr - } - } - return err -} diff --git a/vendor/github.com/ulikunitz/xz/lzma/writer2.go b/vendor/github.com/ulikunitz/xz/lzma/writer2.go deleted file mode 100644 index 97bbafa1..00000000 --- a/vendor/github.com/ulikunitz/xz/lzma/writer2.go +++ /dev/null @@ -1,305 +0,0 @@ -// Copyright 2014-2022 Ulrich Kunitz. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package lzma - -import ( - "bytes" - "errors" - "io" -) - -// Writer2Config is used to create a Writer2 using parameters. -type Writer2Config struct { - // The properties for the encoding. If the it is nil the value - // {LC: 3, LP: 0, PB: 2} will be chosen. - Properties *Properties - // The capacity of the dictionary. If DictCap is zero, the value - // 8 MiB will be chosen. - DictCap int - // Size of the lookahead buffer; value 0 indicates default size - // 4096 - BufSize int - // Match algorithm - Matcher MatchAlgorithm -} - -// fill replaces zero values with default values. -func (c *Writer2Config) fill() { - if c.Properties == nil { - c.Properties = &Properties{LC: 3, LP: 0, PB: 2} - } - if c.DictCap == 0 { - c.DictCap = 8 * 1024 * 1024 - } - if c.BufSize == 0 { - c.BufSize = 4096 - } -} - -// Verify checks the Writer2Config for correctness. Zero values will be -// replaced by default values. -func (c *Writer2Config) Verify() error { - c.fill() - var err error - if c == nil { - return errors.New("lzma: WriterConfig is nil") - } - if c.Properties == nil { - return errors.New("lzma: WriterConfig has no Properties set") - } - if err = c.Properties.verify(); err != nil { - return err - } - if !(MinDictCap <= c.DictCap && int64(c.DictCap) <= MaxDictCap) { - return errors.New("lzma: dictionary capacity is out of range") - } - if !(maxMatchLen <= c.BufSize) { - return errors.New("lzma: lookahead buffer size too small") - } - if c.Properties.LC+c.Properties.LP > 4 { - return errors.New("lzma: sum of lc and lp exceeds 4") - } - if err = c.Matcher.verify(); err != nil { - return err - } - return nil -} - -// Writer2 supports the creation of an LZMA2 stream. But note that -// written data is buffered, so call Flush or Close to write data to the -// underlying writer. The Close method writes the end-of-stream marker -// to the stream. So you may be able to concatenate the output of two -// writers as long the output of the first writer has only been flushed -// but not closed. -// -// Any change to the fields Properties, DictCap must be done before the -// first call to Write, Flush or Close. -type Writer2 struct { - w io.Writer - - start *state - encoder *encoder - - cstate chunkState - ctype chunkType - - buf bytes.Buffer - lbw LimitedByteWriter -} - -// NewWriter2 creates an LZMA2 chunk sequence writer with the default -// parameters and options. -func NewWriter2(lzma2 io.Writer) (w *Writer2, err error) { - return Writer2Config{}.NewWriter2(lzma2) -} - -// NewWriter2 creates a new LZMA2 writer using the given configuration. -func (c Writer2Config) NewWriter2(lzma2 io.Writer) (w *Writer2, err error) { - if err = c.Verify(); err != nil { - return nil, err - } - w = &Writer2{ - w: lzma2, - start: newState(*c.Properties), - cstate: start, - ctype: start.defaultChunkType(), - } - w.buf.Grow(maxCompressed) - w.lbw = LimitedByteWriter{BW: &w.buf, N: maxCompressed} - m, err := c.Matcher.new(c.DictCap) - if err != nil { - return nil, err - } - d, err := newEncoderDict(c.DictCap, c.BufSize, m) - if err != nil { - return nil, err - } - w.encoder, err = newEncoder(&w.lbw, cloneState(w.start), d, 0) - if err != nil { - return nil, err - } - return w, nil -} - -// written returns the number of bytes written to the current chunk -func (w *Writer2) written() int { - if w.encoder == nil { - return 0 - } - return int(w.encoder.Compressed()) + w.encoder.dict.Buffered() -} - -// errClosed indicates that the writer is closed. -var errClosed = errors.New("lzma: writer closed") - -// Writes data to LZMA2 stream. Note that written data will be buffered. -// Use Flush or Close to ensure that data is written to the underlying -// writer. -func (w *Writer2) Write(p []byte) (n int, err error) { - if w.cstate == stop { - return 0, errClosed - } - for n < len(p) { - m := maxUncompressed - w.written() - if m <= 0 { - panic("lzma: maxUncompressed reached") - } - var q []byte - if n+m < len(p) { - q = p[n : n+m] - } else { - q = p[n:] - } - k, err := w.encoder.Write(q) - n += k - if err != nil && err != ErrLimit { - return n, err - } - if err == ErrLimit || k == m { - if err = w.flushChunk(); err != nil { - return n, err - } - } - } - return n, nil -} - -// writeUncompressedChunk writes an uncompressed chunk to the LZMA2 -// stream. -func (w *Writer2) writeUncompressedChunk() error { - u := w.encoder.Compressed() - if u <= 0 { - return errors.New("lzma: can't write empty uncompressed chunk") - } - if u > maxUncompressed { - panic("overrun of uncompressed data limit") - } - switch w.ctype { - case cLRND: - w.ctype = cUD - default: - w.ctype = cU - } - w.encoder.state = w.start - - header := chunkHeader{ - ctype: w.ctype, - uncompressed: uint32(u - 1), - } - hdata, err := header.MarshalBinary() - if err != nil { - return err - } - if _, err = w.w.Write(hdata); err != nil { - return err - } - _, err = w.encoder.dict.CopyN(w.w, int(u)) - return err -} - -// writeCompressedChunk writes a compressed chunk to the underlying -// writer. -func (w *Writer2) writeCompressedChunk() error { - if w.ctype == cU || w.ctype == cUD { - panic("chunk type uncompressed") - } - - u := w.encoder.Compressed() - if u <= 0 { - return errors.New("writeCompressedChunk: empty chunk") - } - if u > maxUncompressed { - panic("overrun of uncompressed data limit") - } - c := w.buf.Len() - if c <= 0 { - panic("no compressed data") - } - if c > maxCompressed { - panic("overrun of compressed data limit") - } - header := chunkHeader{ - ctype: w.ctype, - uncompressed: uint32(u - 1), - compressed: uint16(c - 1), - props: w.encoder.state.Properties, - } - hdata, err := header.MarshalBinary() - if err != nil { - return err - } - if _, err = w.w.Write(hdata); err != nil { - return err - } - _, err = io.Copy(w.w, &w.buf) - return err -} - -// writes a single chunk to the underlying writer. -func (w *Writer2) writeChunk() error { - u := int(uncompressedHeaderLen + w.encoder.Compressed()) - c := headerLen(w.ctype) + w.buf.Len() - if u < c { - return w.writeUncompressedChunk() - } - return w.writeCompressedChunk() -} - -// flushChunk terminates the current chunk. The encoder will be reset -// to support the next chunk. -func (w *Writer2) flushChunk() error { - if w.written() == 0 { - return nil - } - var err error - if err = w.encoder.Close(); err != nil { - return err - } - if err = w.writeChunk(); err != nil { - return err - } - w.buf.Reset() - w.lbw.N = maxCompressed - if err = w.encoder.Reopen(&w.lbw); err != nil { - return err - } - if err = w.cstate.next(w.ctype); err != nil { - return err - } - w.ctype = w.cstate.defaultChunkType() - w.start = cloneState(w.encoder.state) - return nil -} - -// Flush writes all buffered data out to the underlying stream. This -// could result in multiple chunks to be created. -func (w *Writer2) Flush() error { - if w.cstate == stop { - return errClosed - } - for w.written() > 0 { - if err := w.flushChunk(); err != nil { - return err - } - } - return nil -} - -// Close terminates the LZMA2 stream with an EOS chunk. -func (w *Writer2) Close() error { - if w.cstate == stop { - return errClosed - } - if err := w.Flush(); err != nil { - return nil - } - // write zero byte EOS chunk - _, err := w.w.Write([]byte{0}) - if err != nil { - return err - } - w.cstate = stop - return nil -} diff --git a/vendor/github.com/ulikunitz/xz/lzmafilter.go b/vendor/github.com/ulikunitz/xz/lzmafilter.go deleted file mode 100644 index bd5f42ee..00000000 --- a/vendor/github.com/ulikunitz/xz/lzmafilter.go +++ /dev/null @@ -1,117 +0,0 @@ -// Copyright 2014-2022 Ulrich Kunitz. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package xz - -import ( - "errors" - "fmt" - "io" - - "github.com/ulikunitz/xz/lzma" -) - -// LZMA filter constants. -const ( - lzmaFilterID = 0x21 - lzmaFilterLen = 3 -) - -// lzmaFilter declares the LZMA2 filter information stored in an xz -// block header. -type lzmaFilter struct { - dictCap int64 -} - -// String returns a representation of the LZMA filter. -func (f lzmaFilter) String() string { - return fmt.Sprintf("LZMA dict cap %#x", f.dictCap) -} - -// id returns the ID for the LZMA2 filter. -func (f lzmaFilter) id() uint64 { return lzmaFilterID } - -// MarshalBinary converts the lzmaFilter in its encoded representation. -func (f lzmaFilter) MarshalBinary() (data []byte, err error) { - c := lzma.EncodeDictCap(f.dictCap) - return []byte{lzmaFilterID, 1, c}, nil -} - -// UnmarshalBinary unmarshals the given data representation of the LZMA2 -// filter. -func (f *lzmaFilter) UnmarshalBinary(data []byte) error { - if len(data) != lzmaFilterLen { - return errors.New("xz: data for LZMA2 filter has wrong length") - } - if data[0] != lzmaFilterID { - return errors.New("xz: wrong LZMA2 filter id") - } - if data[1] != 1 { - return errors.New("xz: wrong LZMA2 filter size") - } - dc, err := lzma.DecodeDictCap(data[2]) - if err != nil { - return errors.New("xz: wrong LZMA2 dictionary size property") - } - - f.dictCap = dc - return nil -} - -// reader creates a new reader for the LZMA2 filter. -func (f lzmaFilter) reader(r io.Reader, c *ReaderConfig) (fr io.Reader, - err error) { - - config := new(lzma.Reader2Config) - if c != nil { - config.DictCap = c.DictCap - } - dc := int(f.dictCap) - if dc < 1 { - return nil, errors.New("xz: LZMA2 filter parameter " + - "dictionary capacity overflow") - } - if dc > config.DictCap { - config.DictCap = dc - } - - fr, err = config.NewReader2(r) - if err != nil { - return nil, err - } - return fr, nil -} - -// writeCloser creates a io.WriteCloser for the LZMA2 filter. -func (f lzmaFilter) writeCloser(w io.WriteCloser, c *WriterConfig, -) (fw io.WriteCloser, err error) { - config := new(lzma.Writer2Config) - if c != nil { - *config = lzma.Writer2Config{ - Properties: c.Properties, - DictCap: c.DictCap, - BufSize: c.BufSize, - Matcher: c.Matcher, - } - } - - dc := int(f.dictCap) - if dc < 1 { - return nil, errors.New("xz: LZMA2 filter parameter " + - "dictionary capacity overflow") - } - if dc > config.DictCap { - config.DictCap = dc - } - - fw, err = config.NewWriter2(w) - if err != nil { - return nil, err - } - return fw, nil -} - -// last returns true, because an LZMA2 filter must be the last filter in -// the filter list. -func (f lzmaFilter) last() bool { return true } diff --git a/vendor/github.com/ulikunitz/xz/make-docs b/vendor/github.com/ulikunitz/xz/make-docs deleted file mode 100644 index a8c612ce..00000000 --- a/vendor/github.com/ulikunitz/xz/make-docs +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/sh - -set -x -pandoc -t html5 -f markdown -s --css=doc/md.css -o README.html README.md -pandoc -t html5 -f markdown -s --css=doc/md.css -o TODO.html TODO.md diff --git a/vendor/github.com/ulikunitz/xz/none-check.go b/vendor/github.com/ulikunitz/xz/none-check.go deleted file mode 100644 index 6a56a261..00000000 --- a/vendor/github.com/ulikunitz/xz/none-check.go +++ /dev/null @@ -1,23 +0,0 @@ -// Copyright 2014-2022 Ulrich Kunitz. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package xz - -import "hash" - -type noneHash struct{} - -func (h noneHash) Write(p []byte) (n int, err error) { return len(p), nil } - -func (h noneHash) Sum(b []byte) []byte { return b } - -func (h noneHash) Reset() {} - -func (h noneHash) Size() int { return 0 } - -func (h noneHash) BlockSize() int { return 0 } - -func newNoneHash() hash.Hash { - return &noneHash{} -} diff --git a/vendor/github.com/ulikunitz/xz/reader.go b/vendor/github.com/ulikunitz/xz/reader.go deleted file mode 100644 index bde1412c..00000000 --- a/vendor/github.com/ulikunitz/xz/reader.go +++ /dev/null @@ -1,359 +0,0 @@ -// Copyright 2014-2022 Ulrich Kunitz. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// Package xz supports the compression and decompression of xz files. It -// supports version 1.0.4 of the specification without the non-LZMA2 -// filters. See http://tukaani.org/xz/xz-file-format-1.0.4.txt -package xz - -import ( - "bytes" - "errors" - "fmt" - "hash" - "io" - - "github.com/ulikunitz/xz/internal/xlog" - "github.com/ulikunitz/xz/lzma" -) - -// ReaderConfig defines the parameters for the xz reader. The -// SingleStream parameter requests the reader to assume that the -// underlying stream contains only a single stream. -type ReaderConfig struct { - DictCap int - SingleStream bool -} - -// Verify checks the reader parameters for Validity. Zero values will be -// replaced by default values. -func (c *ReaderConfig) Verify() error { - if c == nil { - return errors.New("xz: reader parameters are nil") - } - lc := lzma.Reader2Config{DictCap: c.DictCap} - if err := lc.Verify(); err != nil { - return err - } - return nil -} - -// Reader supports the reading of one or multiple xz streams. -type Reader struct { - ReaderConfig - - xz io.Reader - sr *streamReader -} - -// streamReader decodes a single xz stream -type streamReader struct { - ReaderConfig - - xz io.Reader - br *blockReader - newHash func() hash.Hash - h header - index []record -} - -// NewReader creates a new xz reader using the default parameters. -// The function reads and checks the header of the first XZ stream. The -// reader will process multiple streams including padding. -func NewReader(xz io.Reader) (r *Reader, err error) { - return ReaderConfig{}.NewReader(xz) -} - -// NewReader creates an xz stream reader. The created reader will be -// able to process multiple streams and padding unless a SingleStream -// has been set in the reader configuration c. -func (c ReaderConfig) NewReader(xz io.Reader) (r *Reader, err error) { - if err = c.Verify(); err != nil { - return nil, err - } - r = &Reader{ - ReaderConfig: c, - xz: xz, - } - if r.sr, err = c.newStreamReader(xz); err != nil { - if err == io.EOF { - err = io.ErrUnexpectedEOF - } - return nil, err - } - return r, nil -} - -var errUnexpectedData = errors.New("xz: unexpected data after stream") - -// Read reads uncompressed data from the stream. -func (r *Reader) Read(p []byte) (n int, err error) { - for n < len(p) { - if r.sr == nil { - if r.SingleStream { - data := make([]byte, 1) - _, err = io.ReadFull(r.xz, data) - if err != io.EOF { - return n, errUnexpectedData - } - return n, io.EOF - } - for { - r.sr, err = r.ReaderConfig.newStreamReader(r.xz) - if err != errPadding { - break - } - } - if err != nil { - return n, err - } - } - k, err := r.sr.Read(p[n:]) - n += k - if err != nil { - if err == io.EOF { - r.sr = nil - continue - } - return n, err - } - } - return n, nil -} - -var errPadding = errors.New("xz: padding (4 zero bytes) encountered") - -// newStreamReader creates a new xz stream reader using the given configuration -// parameters. NewReader reads and checks the header of the xz stream. -func (c ReaderConfig) newStreamReader(xz io.Reader) (r *streamReader, err error) { - if err = c.Verify(); err != nil { - return nil, err - } - data := make([]byte, HeaderLen) - if _, err := io.ReadFull(xz, data[:4]); err != nil { - return nil, err - } - if bytes.Equal(data[:4], []byte{0, 0, 0, 0}) { - return nil, errPadding - } - if _, err = io.ReadFull(xz, data[4:]); err != nil { - if err == io.EOF { - err = io.ErrUnexpectedEOF - } - return nil, err - } - r = &streamReader{ - ReaderConfig: c, - xz: xz, - index: make([]record, 0, 4), - } - if err = r.h.UnmarshalBinary(data); err != nil { - return nil, err - } - xlog.Debugf("xz header %s", r.h) - if r.newHash, err = newHashFunc(r.h.flags); err != nil { - return nil, err - } - return r, nil -} - -// readTail reads the index body and the xz footer. -func (r *streamReader) readTail() error { - index, n, err := readIndexBody(r.xz, len(r.index)) - if err != nil { - if err == io.EOF { - err = io.ErrUnexpectedEOF - } - return err - } - - for i, rec := range r.index { - if rec != index[i] { - return fmt.Errorf("xz: record %d is %v; want %v", - i, rec, index[i]) - } - } - - p := make([]byte, footerLen) - if _, err = io.ReadFull(r.xz, p); err != nil { - if err == io.EOF { - err = io.ErrUnexpectedEOF - } - return err - } - var f footer - if err = f.UnmarshalBinary(p); err != nil { - return err - } - xlog.Debugf("xz footer %s", f) - if f.flags != r.h.flags { - return errors.New("xz: footer flags incorrect") - } - if f.indexSize != int64(n)+1 { - return errors.New("xz: index size in footer wrong") - } - return nil -} - -// Read reads actual data from the xz stream. -func (r *streamReader) Read(p []byte) (n int, err error) { - for n < len(p) { - if r.br == nil { - bh, hlen, err := readBlockHeader(r.xz) - if err != nil { - if err == errIndexIndicator { - if err = r.readTail(); err != nil { - return n, err - } - return n, io.EOF - } - return n, err - } - xlog.Debugf("block %v", *bh) - r.br, err = r.ReaderConfig.newBlockReader(r.xz, bh, - hlen, r.newHash()) - if err != nil { - return n, err - } - } - k, err := r.br.Read(p[n:]) - n += k - if err != nil { - if err == io.EOF { - r.index = append(r.index, r.br.record()) - r.br = nil - } else { - return n, err - } - } - } - return n, nil -} - -// countingReader is a reader that counts the bytes read. -type countingReader struct { - r io.Reader - n int64 -} - -// Read reads data from the wrapped reader and adds it to the n field. -func (lr *countingReader) Read(p []byte) (n int, err error) { - n, err = lr.r.Read(p) - lr.n += int64(n) - return n, err -} - -// blockReader supports the reading of a block. -type blockReader struct { - lxz countingReader - header *blockHeader - headerLen int - n int64 - hash hash.Hash - r io.Reader -} - -// newBlockReader creates a new block reader. -func (c *ReaderConfig) newBlockReader(xz io.Reader, h *blockHeader, - hlen int, hash hash.Hash) (br *blockReader, err error) { - - br = &blockReader{ - lxz: countingReader{r: xz}, - header: h, - headerLen: hlen, - hash: hash, - } - - fr, err := c.newFilterReader(&br.lxz, h.filters) - if err != nil { - return nil, err - } - if br.hash.Size() != 0 { - br.r = io.TeeReader(fr, br.hash) - } else { - br.r = fr - } - - return br, nil -} - -// uncompressedSize returns the uncompressed size of the block. -func (br *blockReader) uncompressedSize() int64 { - return br.n -} - -// compressedSize returns the compressed size of the block. -func (br *blockReader) compressedSize() int64 { - return br.lxz.n -} - -// unpaddedSize computes the unpadded size for the block. -func (br *blockReader) unpaddedSize() int64 { - n := int64(br.headerLen) - n += br.compressedSize() - n += int64(br.hash.Size()) - return n -} - -// record returns the index record for the current block. -func (br *blockReader) record() record { - return record{br.unpaddedSize(), br.uncompressedSize()} -} - -// Read reads data from the block. -func (br *blockReader) Read(p []byte) (n int, err error) { - n, err = br.r.Read(p) - br.n += int64(n) - - u := br.header.uncompressedSize - if u >= 0 && br.uncompressedSize() > u { - return n, errors.New("xz: wrong uncompressed size for block") - } - c := br.header.compressedSize - if c >= 0 && br.compressedSize() > c { - return n, errors.New("xz: wrong compressed size for block") - } - if err != io.EOF { - return n, err - } - if br.uncompressedSize() < u || br.compressedSize() < c { - return n, io.ErrUnexpectedEOF - } - - s := br.hash.Size() - k := padLen(br.lxz.n) - q := make([]byte, k+s, k+2*s) - if _, err = io.ReadFull(br.lxz.r, q); err != nil { - if err == io.EOF { - err = io.ErrUnexpectedEOF - } - return n, err - } - if !allZeros(q[:k]) { - return n, errors.New("xz: non-zero block padding") - } - checkSum := q[k:] - computedSum := br.hash.Sum(checkSum[s:]) - if !bytes.Equal(checkSum, computedSum) { - return n, errors.New("xz: checksum error for block") - } - return n, io.EOF -} - -func (c *ReaderConfig) newFilterReader(r io.Reader, f []filter) (fr io.Reader, - err error) { - - if err = verifyFilters(f); err != nil { - return nil, err - } - - fr = r - for i := len(f) - 1; i >= 0; i-- { - fr, err = f[i].reader(fr, c) - if err != nil { - return nil, err - } - } - return fr, nil -} diff --git a/vendor/github.com/ulikunitz/xz/writer.go b/vendor/github.com/ulikunitz/xz/writer.go deleted file mode 100644 index f693e0ae..00000000 --- a/vendor/github.com/ulikunitz/xz/writer.go +++ /dev/null @@ -1,399 +0,0 @@ -// Copyright 2014-2022 Ulrich Kunitz. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package xz - -import ( - "errors" - "fmt" - "hash" - "io" - - "github.com/ulikunitz/xz/lzma" -) - -// WriterConfig describe the parameters for an xz writer. -type WriterConfig struct { - Properties *lzma.Properties - DictCap int - BufSize int - BlockSize int64 - // checksum method: CRC32, CRC64 or SHA256 (default: CRC64) - CheckSum byte - // Forces NoChecksum (default: false) - NoCheckSum bool - // match algorithm - Matcher lzma.MatchAlgorithm -} - -// fill replaces zero values with default values. -func (c *WriterConfig) fill() { - if c.Properties == nil { - c.Properties = &lzma.Properties{LC: 3, LP: 0, PB: 2} - } - if c.DictCap == 0 { - c.DictCap = 8 * 1024 * 1024 - } - if c.BufSize == 0 { - c.BufSize = 4096 - } - if c.BlockSize == 0 { - c.BlockSize = maxInt64 - } - if c.CheckSum == 0 { - c.CheckSum = CRC64 - } - if c.NoCheckSum { - c.CheckSum = None - } -} - -// Verify checks the configuration for errors. Zero values will be -// replaced by default values. -func (c *WriterConfig) Verify() error { - if c == nil { - return errors.New("xz: writer configuration is nil") - } - c.fill() - lc := lzma.Writer2Config{ - Properties: c.Properties, - DictCap: c.DictCap, - BufSize: c.BufSize, - Matcher: c.Matcher, - } - if err := lc.Verify(); err != nil { - return err - } - if c.BlockSize <= 0 { - return errors.New("xz: block size out of range") - } - if err := verifyFlags(c.CheckSum); err != nil { - return err - } - return nil -} - -// filters creates the filter list for the given parameters. -func (c *WriterConfig) filters() []filter { - return []filter{&lzmaFilter{int64(c.DictCap)}} -} - -// maxInt64 defines the maximum 64-bit signed integer. -const maxInt64 = 1<<63 - 1 - -// verifyFilters checks the filter list for the length and the right -// sequence of filters. -func verifyFilters(f []filter) error { - if len(f) == 0 { - return errors.New("xz: no filters") - } - if len(f) > 4 { - return errors.New("xz: more than four filters") - } - for _, g := range f[:len(f)-1] { - if g.last() { - return errors.New("xz: last filter is not last") - } - } - if !f[len(f)-1].last() { - return errors.New("xz: wrong last filter") - } - return nil -} - -// newFilterWriteCloser converts a filter list into a WriteCloser that -// can be used by a blockWriter. -func (c *WriterConfig) newFilterWriteCloser(w io.Writer, f []filter) (fw io.WriteCloser, err error) { - if err = verifyFilters(f); err != nil { - return nil, err - } - fw = nopWriteCloser(w) - for i := len(f) - 1; i >= 0; i-- { - fw, err = f[i].writeCloser(fw, c) - if err != nil { - return nil, err - } - } - return fw, nil -} - -// nopWCloser implements a WriteCloser with a Close method not doing -// anything. -type nopWCloser struct { - io.Writer -} - -// Close returns nil and doesn't do anything else. -func (c nopWCloser) Close() error { - return nil -} - -// nopWriteCloser converts the Writer into a WriteCloser with a Close -// function that does nothing beside returning nil. -func nopWriteCloser(w io.Writer) io.WriteCloser { - return nopWCloser{w} -} - -// Writer compresses data written to it. It is an io.WriteCloser. -type Writer struct { - WriterConfig - - xz io.Writer - bw *blockWriter - newHash func() hash.Hash - h header - index []record - closed bool -} - -// newBlockWriter creates a new block writer writes the header out. -func (w *Writer) newBlockWriter() error { - var err error - w.bw, err = w.WriterConfig.newBlockWriter(w.xz, w.newHash()) - if err != nil { - return err - } - if err = w.bw.writeHeader(w.xz); err != nil { - return err - } - return nil -} - -// closeBlockWriter closes a block writer and records the sizes in the -// index. -func (w *Writer) closeBlockWriter() error { - var err error - if err = w.bw.Close(); err != nil { - return err - } - w.index = append(w.index, w.bw.record()) - return nil -} - -// NewWriter creates a new xz writer using default parameters. -func NewWriter(xz io.Writer) (w *Writer, err error) { - return WriterConfig{}.NewWriter(xz) -} - -// NewWriter creates a new Writer using the given configuration parameters. -func (c WriterConfig) NewWriter(xz io.Writer) (w *Writer, err error) { - if err = c.Verify(); err != nil { - return nil, err - } - w = &Writer{ - WriterConfig: c, - xz: xz, - h: header{c.CheckSum}, - index: make([]record, 0, 4), - } - if w.newHash, err = newHashFunc(c.CheckSum); err != nil { - return nil, err - } - data, err := w.h.MarshalBinary() - if err != nil { - return nil, fmt.Errorf("w.h.MarshalBinary(): error %w", err) - } - if _, err = xz.Write(data); err != nil { - return nil, err - } - if err = w.newBlockWriter(); err != nil { - return nil, err - } - return w, nil - -} - -// Write compresses the uncompressed data provided. -func (w *Writer) Write(p []byte) (n int, err error) { - if w.closed { - return 0, errClosed - } - for { - k, err := w.bw.Write(p[n:]) - n += k - if err != errNoSpace { - return n, err - } - if err = w.closeBlockWriter(); err != nil { - return n, err - } - if err = w.newBlockWriter(); err != nil { - return n, err - } - } -} - -// Close closes the writer and adds the footer to the Writer. Close -// doesn't close the underlying writer. -func (w *Writer) Close() error { - if w.closed { - return errClosed - } - w.closed = true - var err error - if err = w.closeBlockWriter(); err != nil { - return err - } - - f := footer{flags: w.h.flags} - if f.indexSize, err = writeIndex(w.xz, w.index); err != nil { - return err - } - data, err := f.MarshalBinary() - if err != nil { - return err - } - if _, err = w.xz.Write(data); err != nil { - return err - } - return nil -} - -// countingWriter is a writer that counts all data written to it. -type countingWriter struct { - w io.Writer - n int64 -} - -// Write writes data to the countingWriter. -func (cw *countingWriter) Write(p []byte) (n int, err error) { - n, err = cw.w.Write(p) - cw.n += int64(n) - if err == nil && cw.n < 0 { - return n, errors.New("xz: counter overflow") - } - return -} - -// blockWriter is writes a single block. -type blockWriter struct { - cxz countingWriter - // mw combines io.WriteCloser w and the hash. - mw io.Writer - w io.WriteCloser - n int64 - blockSize int64 - closed bool - headerLen int - - filters []filter - hash hash.Hash -} - -// newBlockWriter creates a new block writer. -func (c *WriterConfig) newBlockWriter(xz io.Writer, hash hash.Hash) (bw *blockWriter, err error) { - bw = &blockWriter{ - cxz: countingWriter{w: xz}, - blockSize: c.BlockSize, - filters: c.filters(), - hash: hash, - } - bw.w, err = c.newFilterWriteCloser(&bw.cxz, bw.filters) - if err != nil { - return nil, err - } - if bw.hash.Size() != 0 { - bw.mw = io.MultiWriter(bw.w, bw.hash) - } else { - bw.mw = bw.w - } - return bw, nil -} - -// writeHeader writes the header. If the function is called after Close -// the commpressedSize and uncompressedSize fields will be filled. -func (bw *blockWriter) writeHeader(w io.Writer) error { - h := blockHeader{ - compressedSize: -1, - uncompressedSize: -1, - filters: bw.filters, - } - if bw.closed { - h.compressedSize = bw.compressedSize() - h.uncompressedSize = bw.uncompressedSize() - } - data, err := h.MarshalBinary() - if err != nil { - return err - } - if _, err = w.Write(data); err != nil { - return err - } - bw.headerLen = len(data) - return nil -} - -// compressed size returns the amount of data written to the underlying -// stream. -func (bw *blockWriter) compressedSize() int64 { - return bw.cxz.n -} - -// uncompressedSize returns the number of data written to the -// blockWriter -func (bw *blockWriter) uncompressedSize() int64 { - return bw.n -} - -// unpaddedSize returns the sum of the header length, the uncompressed -// size of the block and the hash size. -func (bw *blockWriter) unpaddedSize() int64 { - if bw.headerLen <= 0 { - panic("xz: block header not written") - } - n := int64(bw.headerLen) - n += bw.compressedSize() - n += int64(bw.hash.Size()) - return n -} - -// record returns the record for the current stream. Call Close before -// calling this method. -func (bw *blockWriter) record() record { - return record{bw.unpaddedSize(), bw.uncompressedSize()} -} - -var errClosed = errors.New("xz: writer already closed") - -var errNoSpace = errors.New("xz: no space") - -// Write writes uncompressed data to the block writer. -func (bw *blockWriter) Write(p []byte) (n int, err error) { - if bw.closed { - return 0, errClosed - } - - t := bw.blockSize - bw.n - if int64(len(p)) > t { - err = errNoSpace - p = p[:t] - } - - var werr error - n, werr = bw.mw.Write(p) - bw.n += int64(n) - if werr != nil { - return n, werr - } - return n, err -} - -// Close closes the writer. -func (bw *blockWriter) Close() error { - if bw.closed { - return errClosed - } - bw.closed = true - if err := bw.w.Close(); err != nil { - return err - } - s := bw.hash.Size() - k := padLen(bw.cxz.n) - p := make([]byte, k+s) - bw.hash.Sum(p[k:k]) - if _, err := bw.cxz.w.Write(p); err != nil { - return err - } - return nil -} diff --git a/vendor/modules.txt b/vendor/modules.txt index 30d04aac..4a86d3fb 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -83,12 +83,9 @@ github.com/nats-io/nuid github.com/novox/mesh-host/internal/declaration github.com/novox/mesh-host/rootsearch github.com/novox/mesh-host/validate -# github.com/ulikunitz/xz v0.5.17 -## explicit; go 1.20 -github.com/ulikunitz/xz -github.com/ulikunitz/xz/internal/hash -github.com/ulikunitz/xz/internal/xlog -github.com/ulikunitz/xz/lzma +# github.com/therootcompany/xz v1.0.1 +## explicit; go 1.15 +github.com/therootcompany/xz # golang.org/x/crypto v0.57.0 ## explicit; go 1.26.0 golang.org/x/crypto/acme From 539a1c16cd4e11feb49511e1b8b859c03d478076 Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 12 Oct 2026 16:08:33 +0200 Subject: [PATCH 10/26] Refuse an xz file of more than 1,024 blocks before decoding it (review 68, hq issue 345) The xz reader keeps one delta or BCJ decoder for every block of a stream and frees them only when the stream ends, so a file of many tiny blocks grows the heap with its length: about 67 MiB live for a 6 MB `xz --block-size=1 --delta` file, about 5.6 GiB at the 512 MiB download cap. The file's framing is now walked first, without decompressing: stream header, each block header, the LZMA2 chunk headers (each carries its compressed size, so a chunk is skipped unread), padding, check, index and footer. A file of more than 1,024 blocks or 16 streams is refused, as is an index whose block count is not the count walked. The index alone is not trusted because the reader compares it with the blocks only after decoding them all, so a lying index would still be decoded. Refusing delta and BCJ outright was not chosen: it needs the same walk to see each block header, and would refuse sound releases filtered with BCJ. 1,024 blocks keep at most about 0.35 MiB of decoders. Betterbird's archive is 1 stream with 1 block; multi-threaded xz cuts blocks of 3 times the dictionary, so 512 MiB in blocks of 512 KiB still fits. The library stays unforked. --- internal/builder/upstream_file.go | 7 +- internal/builder/xz_framing.go | 290 ++++++++++++++++++++++++++++ internal/builder/xz_framing_test.go | 207 ++++++++++++++++++++ 3 files changed, 503 insertions(+), 1 deletion(-) create mode 100644 internal/builder/xz_framing.go create mode 100644 internal/builder/xz_framing_test.go diff --git a/internal/builder/upstream_file.go b/internal/builder/upstream_file.go index 38d374e7..c4d3a730 100644 --- a/internal/builder/upstream_file.go +++ b/internal/builder/upstream_file.go @@ -198,7 +198,12 @@ func unpackUpstream(ctx context.Context, file, into, within string) error { // may not carry: the seat's image is the catalogue's build-agent recipe, which installs git and docker-cli // alone, and a release pinned as .tar.xz failed there on 2026-10-12 (hq issue 345). xzDictMax is a real // maximum: a block whose LZMA2 dictionary is larger is refused before its dictionary is allocated - // (ErrMemlimit). What the stream unpacks to is bounded by untar (maxUpstream, maxEntries). + // (ErrMemlimit). The reader keeps a filter decoder for every block of a stream, so the file's framing is + // walked first and a file of more blocks than xzBlocksMax is refused before anything is decoded + // (xz_framing.go). What the stream unpacks to is bounded by untar (maxUpstream, maxEntries). + if err := xzWithinLimits(file); err != nil { + return err + } zipped, err := xz.NewReader(buffered, xzDictMax) if err != nil { return fmt.Errorf("it is not an xz stream this build seat reads (a dictionary of %d MiB at most): %v", diff --git a/internal/builder/xz_framing.go b/internal/builder/xz_framing.go new file mode 100644 index 00000000..d750c1ef --- /dev/null +++ b/internal/builder/xz_framing.go @@ -0,0 +1,290 @@ +package builder + +import ( + "bufio" + "bytes" + "encoding/binary" + "errors" + "fmt" + "hash/crc32" + "io" + "os" +) + +// xzBlocksMax and xzStreamsMax are the most blocks and the most concatenated streams an xz file may hold for +// the build seat to read it. +// +// **Why a block count is limited at all.** The xz reader (github.com/therootcompany/xz v1.0.1) keeps one +// filter decoder for every block that names a delta or BCJ filter, and frees them only when a stream ends: +// its block counters are reset per stream, not per block. A stream of many tiny blocks therefore grows the +// heap with its length: an `xz --block-size=1 --delta` file of 6 MB kept about 67 MiB live, which comes to +// about 5.6 GiB at the 512 MiB download cap (review 68 on mesh-controller pull request 264, hq issue 345). +// +// **Why 1,024.** At about 340 bytes of decoder a block, 1,024 blocks keep at most about 0.35 MiB live. A +// real release is far below it: Betterbird's archive is 1 stream with 1 block, and multi-threaded xz cuts +// blocks of three times the dictionary (24 MiB at the default preset), so even 512 MiB unpacked in blocks +// of 512 KiB stays within it. Streams are limited to 16 because a release is one stream; the decoders are +// reused from one stream to the next, so the stream limit only bounds the work of this walk. +const ( + xzBlocksMax = 1024 + xzStreamsMax = 16 +) + +// xzWithinLimits walks an xz file's framing, without decompressing anything, and refuses one holding more +// blocks or streams than the build seat reads. +// +// **The framing, not the index.** Each stream's index at its end lists its blocks, but the reader compares +// the index with the blocks only after it has decoded them all, so a file whose index lies would still be +// decoded block by block. This walk follows the same framing the reader follows: stream header, each block +// header, the block's LZMA2 chunks (each chunk header carries its own compressed size, so a chunk is skipped +// without being decoded), its padding and check, then the index and the footer. It counts the blocks the +// reader would decode, and it also refuses an index whose count of blocks is not the count walked. +func xzWithinLimits(file string) error { + f, err := os.Open(file) + if err != nil { + return err + } + defer f.Close() + _, _, err = xzFraming(bufio.NewReader(f)) + if errors.Is(err, errXzTooManyBlocks) || errors.Is(err, errXzTooManyStreams) { + return err + } + if err != nil { + return fmt.Errorf("it is not an xz file this build seat reads: %v", err) + } + return nil +} + +var ( + errXzTooManyBlocks = fmt.Errorf("it holds more than %d xz blocks, the most this build seat reads: "+ + "its xz reader keeps a filter decoder for every block (hq issue 345)", xzBlocksMax) + errXzTooManyStreams = fmt.Errorf("it holds more than %d concatenated xz streams, the most this build "+ + "seat reads", xzStreamsMax) +) + +// xzFraming counts the streams and blocks of an xz file, checking its framing as it goes. It stops with +// errXzTooManyBlocks or errXzTooManyStreams at the first block or stream over the limit. +func xzFraming(r *bufio.Reader) (streams, blocks int, err error) { + full := func(n int) ([]byte, error) { + b := make([]byte, n) + if _, err := io.ReadFull(r, b); err != nil { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + return nil, err + } + return b, nil + } + skip := func(n int) error { + if got, err := r.Discard(n); err != nil || got != n { + return io.ErrUnexpectedEOF + } + return nil + } + for { + header, err := full(12) + if err != nil { + return streams, blocks, fmt.Errorf("stream %d: header: %v", streams+1, err) + } + if !bytes.Equal(header[:6], []byte{0xfd, '7', 'z', 'X', 'Z', 0x00}) { + return streams, blocks, fmt.Errorf("stream %d: no xz stream header", streams+1) + } + if header[6] != 0 || header[7]&0xf0 != 0 || + crc32.ChecksumIEEE(header[6:8]) != binary.LittleEndian.Uint32(header[8:12]) { + return streams, blocks, fmt.Errorf("stream %d: a corrupt stream header", streams+1) + } + if streams++; streams > xzStreamsMax { + return streams, blocks, errXzTooManyStreams + } + checkSize := 0 + if check := int(header[7]); check != 0 { + checkSize = 4 << ((check - 1) / 3) + } + inStream := 0 + for { + first, err := r.ReadByte() + if err != nil { + return streams, blocks, fmt.Errorf("stream %d: no index: %v", streams, io.ErrUnexpectedEOF) + } + if first == 0 { + break // the index indicator + } + rest, err := full(int(first)*4 + 3) + if err != nil { + return streams, blocks, fmt.Errorf("stream %d, block %d: header: %v", streams, inStream+1, err) + } + if err := xzBlockHeader(append([]byte{first}, rest...)); err != nil { + return streams, blocks, fmt.Errorf("stream %d, block %d: %v", streams, inStream+1, err) + } + inStream++ + if blocks++; blocks > xzBlocksMax { + return streams, blocks, errXzTooManyBlocks + } + // The block's LZMA2 chunks, up to the end marker. + size := 0 + for { + control, err := r.ReadByte() + if err != nil { + return streams, blocks, fmt.Errorf("stream %d, block %d: %v", streams, inStream, io.ErrUnexpectedEOF) + } + size++ + if control == 0 { + break + } + switch { + case control == 1 || control == 2: // stored: a 16-bit size less one, then the bytes + b, err := full(2) + if err != nil { + return streams, blocks, err + } + n := int(binary.BigEndian.Uint16(b)) + 1 + if err := skip(n); err != nil { + return streams, blocks, err + } + size += 2 + n + case control >= 0x80: // LZMA: unpacked and packed sizes, the properties byte on a reset + b, err := full(4) + if err != nil { + return streams, blocks, err + } + n := int(binary.BigEndian.Uint16(b[2:])) + 1 + size += 4 + if control >= 0xc0 { + if err := skip(1); err != nil { + return streams, blocks, err + } + size++ + } + if err := skip(n); err != nil { + return streams, blocks, err + } + size += n + default: + return streams, blocks, fmt.Errorf("stream %d, block %d: an LZMA2 chunk of kind %#x", + streams, inStream, control) + } + } + if pad := (4 - size%4) % 4; pad > 0 { + b, err := full(pad) + if err != nil || !bytes.Equal(b, make([]byte, pad)) { + return streams, blocks, fmt.Errorf("stream %d, block %d: corrupt block padding", streams, inStream) + } + } + if err := skip(checkSize); err != nil { + return streams, blocks, err + } + } + // The index: its count of records, two numbers each, padding, CRC32. + used := 1 + count, n, err := xzVLI(r) + if err != nil { + return streams, blocks, fmt.Errorf("stream %d: index: %v", streams, err) + } + used += n + if count != uint64(inStream) { + return streams, blocks, fmt.Errorf("stream %d: its index lists %d blocks, but it holds %d", + streams, count, inStream) + } + for i := 0; i < 2*inStream; i++ { + _, n, err := xzVLI(r) + if err != nil { + return streams, blocks, fmt.Errorf("stream %d: index: %v", streams, err) + } + used += n + } + if err := skip((4-used%4)%4 + 4); err != nil { + return streams, blocks, fmt.Errorf("stream %d: index: %v", streams, err) + } + footer, err := full(12) + if err != nil || footer[10] != 'Y' || footer[11] != 'Z' { + return streams, blocks, fmt.Errorf("stream %d: no stream footer", streams) + } + // Stream padding: zero bytes, a multiple of four, then the end or another stream. + zeros := 0 + for { + b, err := r.ReadByte() + if err == io.EOF { + if zeros%4 != 0 { + return streams, blocks, errors.New("stream padding not a multiple of four bytes") + } + return streams, blocks, nil + } + if err != nil { + return streams, blocks, err + } + if b != 0 { + _ = r.UnreadByte() + break + } + zeros++ + } + if zeros%4 != 0 { + return streams, blocks, errors.New("stream padding not a multiple of four bytes") + } + } +} + +// xzBlockHeader checks one block header: its CRC32, its flags, and that its last filter is LZMA2, the only +// one whose chunks this walk can skip (and the only last filter the reader decodes). +func xzBlockHeader(h []byte) error { + size := len(h) + if crc32.ChecksumIEEE(h[:size-4]) != binary.LittleEndian.Uint32(h[size-4:]) { + return errors.New("a corrupt block header") + } + flags := h[1] + if flags&0x3c != 0 { + return errors.New("a block header with reserved flags") + } + body := bytes.NewReader(h[2 : size-4]) + if flags&0x40 != 0 { + if _, _, err := xzVLI(body); err != nil { + return err + } + } + if flags&0x80 != 0 { + if _, _, err := xzVLI(body); err != nil { + return err + } + } + var last uint64 + for i := 0; i <= int(flags&3); i++ { + id, _, err := xzVLI(body) + if err != nil { + return err + } + props, _, err := xzVLI(body) + if err != nil { + return err + } + if props > uint64(body.Len()) { + return errors.New("a corrupt block header") + } + if _, err := body.Seek(int64(props), io.SeekCurrent); err != nil { + return err + } + last = id + } + if last != 0x21 { + return fmt.Errorf("a block whose last filter is %#x, not LZMA2", last) + } + return nil +} + +// xzVLI reads one of xz's variable-length integers (at most 9 bytes, 7 bits each), and how many bytes it took. +func xzVLI(r io.ByteReader) (uint64, int, error) { + var v uint64 + for i := 0; i < 9; i++ { + b, err := r.ReadByte() + if err != nil { + return 0, i, io.ErrUnexpectedEOF + } + v |= uint64(b&0x7f) << (7 * i) + if b&0x80 == 0 { + if i > 0 && b == 0 { + return 0, i + 1, errors.New("a number not in its shortest form") + } + return v, i + 1, nil + } + } + return 0, 9, errors.New("a number longer than nine bytes") +} diff --git a/internal/builder/xz_framing_test.go b/internal/builder/xz_framing_test.go new file mode 100644 index 00000000..a672cb80 --- /dev/null +++ b/internal/builder/xz_framing_test.go @@ -0,0 +1,207 @@ +package builder + +import ( + "archive/tar" + "bufio" + "bytes" + "context" + "encoding/binary" + "errors" + "hash/crc32" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + "testing" +) + +// xzStream is one xz stream (no integrity check) of data cut into blocks of blockSize bytes, each block's +// LZMA2 data stored uncompressed, each block a delta filter before LZMA2 when delta is set: the shape of +// `xz --block-size=N --filters=delta,lzma2`, built without any xz tool. listed is how many blocks its index +// lists (-1: as many as it holds), so a stream whose index lies can be built too. +func xzStream(data []byte, blockSize int, delta bool, listed int) []byte { + var out bytes.Buffer + flags := []byte{0, 0} + out.Write([]byte{0xfd, '7', 'z', 'X', 'Z', 0x00}) + out.Write(flags) + out.Write(binary.LittleEndian.AppendUint32(nil, crc32.ChecksumIEEE(flags))) + type record struct{ unpadded, uncompressed uint64 } + var records []record + for at := 0; at < len(data); at += blockSize { + block := data[at:min(at+blockSize, len(data))] + if delta { + // Delta-encoded with a distance of one; the decoder's history starts empty at each block. + encoded := make([]byte, len(block)) + for i := range block { + encoded[i] = block[i] + if i > 0 { + encoded[i] -= block[i-1] + } + } + block = encoded + } + header := []byte{2, 0} // 12 bytes + if delta { + header[1] = 1 // two filters + header = append(header, 0x03, 0x01, 0x00) + } + header = append(header, 0x21, 0x01, 0x00) // LZMA2, a 4 KiB dictionary + for len(header) < 8 { + header = append(header, 0) + } + header = binary.LittleEndian.AppendUint32(header, crc32.ChecksumIEEE(header)) + out.Write(header) + size := 0 + for i, chunk := 0, block; len(chunk) > 0; i++ { + n := min(len(chunk), 1<<16) + control := byte(2) + if i == 0 { + control = 1 // the first chunk of a block resets the dictionary + } + out.WriteByte(control) + out.Write(binary.BigEndian.AppendUint16(nil, uint16(n-1))) + out.Write(chunk[:n]) + size += 3 + n + chunk = chunk[n:] + } + out.WriteByte(0) + size++ + out.Write(make([]byte, (4-size%4)%4)) + records = append(records, record{uint64(len(header) + size), uint64(len(block))}) + } + if listed >= 0 { + for len(records) < listed { + records = append(records, records[0]) + } + records = records[:listed] + } + vli := func(b []byte, v uint64) []byte { + for v >= 0x80 { + b = append(b, byte(v)|0x80) + v >>= 7 + } + return append(b, byte(v)) + } + index := vli([]byte{0}, uint64(len(records))) + for _, r := range records { + index = vli(vli(index, r.unpadded), r.uncompressed) + } + for len(index)%4 != 0 { + index = append(index, 0) + } + index = binary.LittleEndian.AppendUint32(index, crc32.ChecksumIEEE(index)) + out.Write(index) + footer := binary.LittleEndian.AppendUint32(nil, uint32(len(index)/4-1)) + footer = append(footer, flags...) + out.Write(binary.LittleEndian.AppendUint32(nil, crc32.ChecksumIEEE(footer))) + out.Write(footer) + out.WriteString("YZ") + return out.Bytes() +} + +// tarHolding is a tar holding betterbird/data with body. +func tarHolding(t *testing.T, body []byte) []byte { + t.Helper() + var out bytes.Buffer + w := tar.NewWriter(&out) + if err := w.WriteHeader(&tar.Header{Name: "betterbird/data", Mode: 0o644, Size: int64(len(body)), + Typeflag: tar.TypeReg}); err != nil { + t.Fatal(err) + } + w.Write(body) + if err := w.Close(); err != nil { + t.Fatal(err) + } + return out.Bytes() +} + +func writeTemp(t *testing.T, body []byte) string { + t.Helper() + file := filepath.Join(t.TempDir(), "r.tar.xz") + if err := os.WriteFile(file, body, 0o644); err != nil { + t.Fatal(err) + } + return file +} + +func TestAnXzFileOfAFewDeltaBlocksAndTwoStreamsIsRead(t *testing.T) { + // The streams this test builds are ones the reader decodes, so what the walk refuses below is the + // same framing the reader follows. + body := bytes.Repeat([]byte("betterbird "), 300) + whole := tarHolding(t, body) + half := len(whole) / 2 + for name, file := range map[string][]byte{ + "one stream of delta blocks": xzStream(whole, 512, true, -1), + "two streams with padding": append(append(xzStream(whole[:half], 700, true, -1), 0, 0, 0, 0), + xzStream(whole[half:], 100, false, -1)...), + } { + path := writeTemp(t, file) + if xzPath, err := exec.LookPath("xz"); err == nil { + if out, err := exec.Command(xzPath, "-t", path).CombinedOutput(); err != nil { + t.Fatalf("%s: xz itself refuses the built stream: %v %s", name, err, out) + } + } + into := t.TempDir() + if err := unpackUpstream(context.Background(), path, into, "betterbird"); err != nil { + t.Fatalf("%s: %v", name, err) + } + got, err := os.ReadFile(filepath.Join(into, "data")) + if err != nil || !bytes.Equal(got, body) { + t.Fatalf("%s: unpacked %d bytes, want %d: %v", name, len(got), len(body), err) + } + } +} + +func TestAnXzFileOfManyDeltaBlocksIsRefusedBeforeItIsDecoded(t *testing.T) { + // Review 68 on pull request 264: `xz --block-size=1 --delta` of a 200 kB tar kept about 67 MiB live in + // the reader, one delta decoder per block. Its index here lists one block, as a file crafted to pass a + // check of the index alone would; the walk counts the blocks the reader would decode. + body := make([]byte, 200_000) + for i := range body { + body[i] = byte(i * 7) + } + crafted := xzStream(tarHolding(t, body), 1, true, 1) + path := writeTemp(t, crafted) + var before, after runtime.MemStats + runtime.GC() + runtime.ReadMemStats(&before) + err := unpackUpstream(context.Background(), path, t.TempDir(), "betterbird") + runtime.ReadMemStats(&after) + if err == nil || !strings.Contains(err.Error(), "more than 1024 xz blocks") { + t.Fatalf("a file of %d bytes in blocks of one byte was not refused for its blocks: %v", len(crafted), err) + } + if grew := after.TotalAlloc - before.TotalAlloc; grew > 4<<20 { + t.Fatalf("%d bytes were allocated for a refused file", grew) + } +} + +func TestTheXzBlockLimitIsExact(t *testing.T) { + data := make([]byte, xzBlocksMax+1) + streams, blocks, err := xzFraming(bufio.NewReader(bytes.NewReader(xzStream(data[:xzBlocksMax], 1, true, -1)))) + if err != nil || streams != 1 || blocks != xzBlocksMax { + t.Fatalf("%d blocks: %d streams, %d blocks, %v", xzBlocksMax, streams, blocks, err) + } + if _, _, err := xzFraming(bufio.NewReader(bytes.NewReader(xzStream(data, 1, true, -1)))); !errors.Is(err, errXzTooManyBlocks) { + t.Fatalf("%d blocks: %v", xzBlocksMax+1, err) + } +} + +func TestAnXzFileOfTooManyStreamsIsRefused(t *testing.T) { + var file []byte + for i := 0; i <= xzStreamsMax; i++ { + file = append(file, xzStream([]byte("x"), 1, false, -1)...) + } + err := unpackUpstream(context.Background(), writeTemp(t, file), t.TempDir(), "") + if err == nil || !strings.Contains(err.Error(), "more than 16 concatenated xz streams") { + t.Fatalf("%v", err) + } +} + +func TestAnXzIndexThatDoesNotListItsBlocksIsRefused(t *testing.T) { + file := xzStream(tarHolding(t, []byte("x")), 512, false, 1) + err := unpackUpstream(context.Background(), writeTemp(t, file), t.TempDir(), "betterbird") + if err == nil || !strings.Contains(err.Error(), "its index lists 1 blocks, but it holds 4") { + t.Fatalf("%v", err) + } +} From d20b52cb196b0fbbeb885e830e50d22e1b68b8cc Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 12 Oct 2026 16:18:54 +0200 Subject: [PATCH 11/26] Let plans wait for a plan to move and answer it as data (issue 229) An agent following a rollout had plans answer once, in prose, and polled the console with a curl loop of its own to be told a plan moved. plans now takes wait (a plan's id), until (moved, tier or done) and bound, and answers when the plan moves or the bound passes, saying which, with the plan as data and its line as the output. A call's one answer has to reach the console inside its thirty seconds, so link.AnswersWithin lets a handler that waits on purpose hold its answer up to LongestAnswer (25s) instead of AnswerWithin; the bound is at most 20s, and a caller following a plan further asks again. --- cmd/mesh-controller/plans_follow.go | 156 +++++++++++++++++++ cmd/mesh-controller/plans_follow_test.go | 181 +++++++++++++++++++++++ cmd/mesh-controller/release_plan.go | 7 + cmd/mesh-controller/seatverbs.go | 31 +++- internal/catalogue/verbs.go | 10 +- internal/link/calls.go | 57 +++++-- internal/link/calls_wait_test.go | 55 +++++++ testdata/store-tests | 1 + 8 files changed, 484 insertions(+), 14 deletions(-) create mode 100644 cmd/mesh-controller/plans_follow.go create mode 100644 cmd/mesh-controller/plans_follow_test.go create mode 100644 internal/link/calls_wait_test.go diff --git a/cmd/mesh-controller/plans_follow.go b/cmd/mesh-controller/plans_follow.go new file mode 100644 index 00000000..c13dedc7 --- /dev/null +++ b/cmd/mesh-controller/plans_follow.go @@ -0,0 +1,156 @@ +package main + +import ( + "context" + "encoding/json" + "fmt" + "os" + "strconv" + "strings" + "time" + + "github.com/novox/mesh-controller/internal/inventory" + "github.com/novox/mesh-controller/internal/link" +) + +// Following a plan (novox/hq issue 229): `plans wait ` holds its answer until the plan moves, reaches its +// next tier or ends, or until its bound passes, and says which. Before it, an agent following a rollout had +// `plans` answer once, in prose, and wrote a loop of its own round the mesh's tools to be told the plan moved. +// +// **No wait is longer than a call is answered within** (link.LongestAnswer, inside the console's thirty +// seconds): a longer one would answer "still running" and leave its caller to ask `calls` — the loop again. +// So a caller following a plan to its end asks again; each call answers the moment the plan moves. + +// maxPlanWait is the longest bound a wait takes, and the bound when none is given: what a call is answered +// within, less the time a verb's command takes to start and read the store. +var maxPlanWait = link.LongestAnswer - planWaitStart + +// planWaitStart is what the verb's window allows beyond the bound: starting the command and opening the stores. +const planWaitStart = 5 * time.Second + +// planWaitEvery is how often a wait reads the plan again. +const planWaitEvery = time.Second + +// The things a wait waits until. +var planWaitUntil = []string{"moved", "tier", "done"} + +// planWaitAsked checks what a wait is asked for: until (moved when empty) and a bound (a duration, or a number +// of seconds; maxPlanWait when empty). +func planWaitAsked(until, bound string) (string, time.Duration, error) { + if until == "" { + until = "moved" + } + known := false + for _, u := range planWaitUntil { + known = known || u == until + } + if !known { + return "", 0, fmt.Errorf("a wait is until %s, not %q", strings.Join(planWaitUntil, ", "), until) + } + d := maxPlanWait + if bound = strings.TrimSpace(bound); bound != "" { + var err error + if secs, isNumber := strconv.Atoi(bound); isNumber == nil { + d = time.Duration(secs) * time.Second + } else if d, err = time.ParseDuration(bound); err != nil { + return "", 0, fmt.Errorf("the bound %q is not a duration (such as 20s) or a number of seconds", bound) + } + if d <= 0 || d > maxPlanWait { + return "", 0, fmt.Errorf("the bound is at most %s, the longest a call is answered within; %s was asked "+ + "— to follow a plan longer, ask again: each call answers the moment the plan moves", maxPlanWait, d) + } + } + return until, d, nil +} + +// planEnded is a plan that will not move again: done, failed or superseded. A batch being assembled is not one. +func planEnded(p inventory.Plan) bool { return !p.Open() && !p.Batch() } + +// waitOnPlan reads the plan until what it waits for happens or the bound passes, and answers the plan as last +// read and what ended the wait: moved (any save of it), tier (its tier changed), done (it ended — which ends +// every wait, since nothing comes after it), or bound. Moved and tier are judged against the plan as first read. +func waitOnPlan(ctx context.Context, read func(context.Context) (inventory.Plan, error), until string, + bound, every time.Duration) (inventory.Plan, string, error) { + first, err := read(ctx) + if err != nil { + return inventory.Plan{}, "", err + } + if planEnded(first) { + return first, "done", nil + } + deadline := time.Now().Add(bound) + tick := time.NewTicker(every) + defer tick.Stop() + last := first + for { + select { + case <-ctx.Done(): + return last, "", ctx.Err() + case <-tick.C: + } + if !time.Now().Before(deadline) { + return last, "bound", nil + } + p, err := read(ctx) + if err != nil { + return last, "", err + } + last = p + switch { + case planEnded(p): + return p, "done", nil + case until == "moved" && p.Revision != first.Revision: + return p, "moved", nil + case until == "tier" && p.Tier != first.Tier: + return p, "tier", nil + } + } +} + +// planWaited is what `plans wait` answers: the plan, what ended the wait, and the line a person reads (`text`, +// which the verb answers as its output and not again in the data). +type planWaited struct { + // Came is what ended the wait: moved, tier, done, or bound when nothing did. + Came string `json:"came"` + Until string `json:"until"` + Waited string `json:"waited"` + Plan planStatus `json:"plan"` + // Revision is the plan's as read: a caller asking again with the same plan sees whether it moved since. + Revision int64 `json:"revision"` + Text string `json:"text"` +} + +func planWaitedDoc(p inventory.Plan, st planStatus, until, came string, waited time.Duration, line string) planWaited { + said := map[string]string{ + "moved": "moved", "tier": "reached another tier", "done": "ended", + "bound": "did not " + map[string]string{"moved": "move", "tier": "reach another tier", "done": "end"}[until], + }[came] + text := fmt.Sprintf("%s %s in %s (waited until %s)\n%s — %s\n", p.ID, said, waited.Round(time.Second), until, + p.ID, line) + return planWaited{Came: came, Until: until, Waited: waited.Round(time.Millisecond).String(), Plan: st, + Revision: p.Revision, Text: text} +} + +// planWaitCommand is `plans wait `: the plan followed until it moves, or the bound passes. +func planWaitCommand(ctx context.Context, inv *inventory.Inventory, id, untilAsked, boundAsked string) error { + until, bound, err := planWaitAsked(untilAsked, boundAsked) + if err != nil { + return err + } + began := time.Now() + p, came, err := waitOnPlan(ctx, func(ctx context.Context) (inventory.Plan, error) { + return inv.PlanByID(ctx, id) + }, until, bound, planWaitEvery) + if err != nil { + return err + } + // Said as `plans ` says its first line, and as `status` says the plan. + now, one := time.Now(), []inventory.Plan{p} + bounds, pause := readTierBounds(ctx, inv, now), buildSeatPause(ctx, inv, one) + line := planLineSaid(p, now, pause, bounds.of(p.Repository), func(module string) []string { + on, _ := inv.Running(ctx, module) + return on + }, waitsOf(ctx, inv, one)) + doc := planWaitedDoc(p, planStatuses(one, now, pause, bounds)[0], until, came, now.Sub(began), line) + return json.NewEncoder(os.Stdout).Encode(doc) +} diff --git a/cmd/mesh-controller/plans_follow_test.go b/cmd/mesh-controller/plans_follow_test.go new file mode 100644 index 00000000..844b99e7 --- /dev/null +++ b/cmd/mesh-controller/plans_follow_test.go @@ -0,0 +1,181 @@ +package main + +import ( + "context" + "encoding/json" + "strings" + "testing" + "time" + + "github.com/novox/mesh-controller/internal/inventory" +) + +// A plan as the store would answer it at each read: the reads are counted, and each answers the next. +type planReads struct { + plans []inventory.Plan + n int +} + +func (r *planReads) read(context.Context) (inventory.Plan, error) { + p := r.plans[min(r.n, len(r.plans)-1)] + r.n++ + return p, nil +} + +func followedPlan(rev int64, state string, tier int) inventory.Plan { + return inventory.Plan{ID: "plan-1", Repository: "novox/x", Commit: "abc", State: state, Tier: tier, + Tiers: [][]string{{"a"}, {"b"}}, Revision: rev} +} + +// **`plans` waits for a plan, and says what ended the wait** (novox/hq issue 229): an agent following a +// rollout is told when the plan moved, reached its next tier or ended, or that the bound passed — in one +// call, without a loop of its own. +func TestAWaitForAPlanEndsWhenWhatItWaitsForHappens(t *testing.T) { + cases := []struct { + name string + until string + reads []inventory.Plan + came string + times int + }{ + {"moved: any save", "moved", []inventory.Plan{ + followedPlan(1, inventory.PlanBuilding, 0), followedPlan(1, inventory.PlanBuilding, 0), + followedPlan(2, inventory.PlanBuilding, 0)}, "moved", 3}, + {"tier: a save inside the tier is not enough", "tier", []inventory.Plan{ + followedPlan(1, inventory.PlanBuilding, 0), followedPlan(2, inventory.PlanBuilding, 0), + followedPlan(3, inventory.PlanRolling, 1)}, "tier", 3}, + {"tier: a plan that ended has no next tier to wait for", "tier", []inventory.Plan{ + followedPlan(1, inventory.PlanBuilding, 0), followedPlan(2, inventory.PlanFailed, 0)}, "done", 2}, + {"done: a new tier is not the end", "done", []inventory.Plan{ + followedPlan(1, inventory.PlanBuilding, 0), followedPlan(2, inventory.PlanRolling, 1), + followedPlan(3, inventory.PlanDone, 1)}, "done", 3}, + {"done: a plan already ended answers at once", "done", []inventory.Plan{ + followedPlan(4, inventory.PlanSuperseded, 1)}, "done", 1}, + {"the bound passes", "moved", []inventory.Plan{followedPlan(1, inventory.PlanBuilding, 0)}, "bound", -1}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + r := &planReads{plans: c.reads} + p, came, err := waitOnPlan(context.Background(), r.read, c.until, 50*time.Millisecond, time.Millisecond) + if err != nil { + t.Fatal(err) + } + if came != c.came { + t.Fatalf("the wait ended as %q, want %q", came, c.came) + } + if c.times > 0 && r.n != c.times { + t.Errorf("read the plan %d times, want %d: it answered before or after what it waited for", r.n, c.times) + } + if want := c.reads[min(r.n, len(c.reads))-1]; p.Revision != want.Revision { + t.Errorf("answered revision %d, not the plan as it was last read (%d)", p.Revision, want.Revision) + } + }) + } +} + +// What a wait is asked for is checked before anything is read: what it waits until, and a bound no longer than +// a call is answered within. +func TestAWaitIsRefusedWhatItCannotDo(t *testing.T) { + for _, bad := range []struct{ until, bound string }{{"landed", ""}, {"", "1h"}, {"", "-3s"}, {"", "soon"}} { + if _, _, err := planWaitAsked(bad.until, bad.bound); err == nil { + t.Errorf("until %q, bound %q was taken", bad.until, bad.bound) + } + } + until, bound, err := planWaitAsked("", "") + if err != nil || until != "moved" || bound != maxPlanWait { + t.Fatalf("the defaults are %q %s %v, want moved and %s", until, bound, err, maxPlanWait) + } + if _, bound, _ := planWaitAsked("tier", "12"); bound != 12*time.Second { + t.Errorf("a bare number is seconds; read %s", bound) + } +} + +// **The answer is data, with its prose as a field beside it** (novox/hq issue 229): the verb's `answer` carries +// the plan and what ended the wait, and its output is the line a person reads — never JSON inside a string. +func TestAWaitForAPlanAnswersAsData(t *testing.T) { + p := followedPlan(7, inventory.PlanRolling, 1) + st := planStatuses([]inventory.Plan{p}, time.Now(), pauseView{}, tierBounds{})[0] + doc := planWaitedDoc(p, st, "tier", "tier", 3*time.Second, "rolling, tier 2 of 2") + stdout, _ := json.Marshal(doc) + a := answerOf([]string{"plans", "wait", "plan-1"}, stdout, "", true) + data, ok := a.Answer.(map[string]any) + if !ok { + t.Fatalf("the answer is not data: %v", a.Answer) + } + if data["came"] != "tier" || data["revision"] != float64(7) { + t.Errorf("the data does not say what ended the wait: %v", data) + } + if plan, _ := data["plan"].(map[string]any); plan["state"] != "rolling" || plan["tier"] != float64(1) { + t.Errorf("the data does not carry the plan: %v", data["plan"]) + } + if _, text := data["text"]; text { + t.Errorf("the prose is in the data too: %v", data) + } + if strings.Contains(a.Output, "{") || !strings.Contains(a.Output, "plan-1") || !strings.Contains(a.Output, "tier") { + t.Errorf("the output is not the line a person reads: %q", a.Output) + } +} + +// The verb composes the wait, and asks for a window as long as its bound. +func TestThePlansVerbComposesAWait(t *testing.T) { + argv, err := argvFor("plans", map[string]any{"wait": "plan-1", "until": "done", "bound": "15s"}) + if err != nil { + t.Fatal(err) + } + if got := strings.Join(argv, " "); got != "plans wait plan-1 --until done --bound 15s" { + t.Fatalf("composed %q", got) + } + if _, err := argvFor("plans", map[string]any{"until": "done"}); err == nil { + t.Error("until without wait was taken") + } + if _, err := argvFor("plans", map[string]any{"wait": "plan-1", "bound": "1h"}); err == nil { + t.Error("a bound past what a call can wait was taken") + } +} + +// Live, with the store: `plans wait` run as the verb runs it answers when the plan reaches its next tier, as +// data, and says the bound passed when nothing moved (novox/hq issue 229). +func TestAPlanFollowedFromTheStoreAnswersWhenItMoves(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + now := time.Now().UTC() + walk := inventory.Plan{ID: "plan-229", Repository: "novox/mesh-catalog", Branch: "main", Commit: "c229c229", + Created: now, State: inventory.PlanBuilding, Tiers: [][]string{{"a"}, {"b"}}, + Modules: map[string]*inventory.PlanModule{"a": {}, "b": {}}} + if err := open.inventory.SavePlan(ctx, &walk); err != nil { + t.Fatal(err) + } + moved := make(chan error, 1) + go func() { + time.Sleep(1500 * time.Millisecond) + p, err := open.inventory.PlanByID(ctx, walk.ID) + if err == nil { + p.Tier, p.State = 1, inventory.PlanRolling + err = open.inventory.SavePlan(ctx, &p) + } + moved <- err + }() + began := time.Now() + var got planWaited + out := printed(t, func() error { + return plansCommand(ctx, []string{"wait", walk.ID, "--until", "tier", "--bound", "15s"}) + }) + if err := <-moved; err != nil { + t.Fatal(err) + } + if err := json.Unmarshal([]byte(out), &got); err != nil { + t.Fatalf("plans wait did not answer data: %v: %q", err, out) + } + if got.Came != "tier" || got.Plan.Tier != 1 || got.Plan.State != inventory.PlanRolling { + t.Fatalf("the wait answered %+v", got) + } + if waited := time.Since(began); waited > 10*time.Second { + t.Errorf("it answered after %s, not when the plan moved", waited) + } + out = printed(t, func() error { + return plansCommand(ctx, []string{"wait", walk.ID, "--bound", "2"}) + }) + if err := json.Unmarshal([]byte(out), &got); err != nil || got.Came != "bound" || got.Until != "moved" { + t.Fatalf("a plan that did not move answered %+v (%v)", got, err) + } +} diff --git a/cmd/mesh-controller/release_plan.go b/cmd/mesh-controller/release_plan.go index 1710c59a..452ae757 100644 --- a/cmd/mesh-controller/release_plan.go +++ b/cmd/mesh-controller/release_plan.go @@ -1463,6 +1463,10 @@ func plansCommand(ctx context.Context, args []string) error { modules := set.String("modules", "", "or the modules it would change, comma-separated") moduleDirs := set.String("module-dirs", "", "with --paths: the directories holding a module.json at the commit, "+ "comma-separated, as the forge's announcer says them (novox/hq issue 278)") + // Following a plan (novox/hq issue 229): `plans wait `. + until := set.String("until", "", "with wait: what to wait for — moved (any change), tier (its next tier) or done (its end)") + bound := set.String("bound", "", "with wait: how long to wait at most, as 15s or seconds (default and at most the "+ + "longest a call is answered within)") // Ending a plan by hand is a repair, and says why (novox/hq to-be 45 §7). why := addHandActFlags(set) positionals, err := parseAround(set, args) @@ -1482,6 +1486,9 @@ func plansCommand(ctx context.Context, args []string) error { defer open.Close() inv := open.inventory now := time.Now() + if len(positionals) == 2 && positionals[0] == "wait" { + return planWaitCommand(ctx, inv, positionals[1], *until, *bound) + } if len(positionals) == 1 { p, err := inv.PlanByID(ctx, positionals[0]) if err != nil { diff --git a/cmd/mesh-controller/seatverbs.go b/cmd/mesh-controller/seatverbs.go index 3e4d1d18..44598bbf 100644 --- a/cmd/mesh-controller/seatverbs.go +++ b/cmd/mesh-controller/seatverbs.go @@ -365,6 +365,21 @@ func (a *verbArguments) commandLine() ([]string, error) { return argv, nil } } + if id := str("wait"); id != "" { + // Following a plan (novox/hq issue 229): checked here too, so a call asking what no wait can do is + // refused before a command is started. + if _, _, err := planWaitAsked(a.given["until"], a.given["bound"]); err != nil { + return nil, err + } + argv := []string{"plans", "wait", id} + if u := str("until"); u != "" { + argv = append(argv, "--until", u) + } + if b := str("bound"); b != "" { + argv = append(argv, "--bound", b) + } + return argv, nil + } if id := str("id"); id != "" { return []string{"plans", id}, nil } @@ -985,12 +1000,14 @@ var jsonVerbs = map[string]bool{"status": true, "seats": true, "plan": true, "co // The delivery's owner's verbs answer JSON where they read (plan, order, check, walks) — novox/hq ADR 0239. "delivery": true, // The build queue, each ask with its position among the waiting (novox/hq issue 455). - "queue": true} + "queue": true, + // A plan followed (novox/hq issue 229): `plans wait` answers JSON; its other forms say text, and so answer no data. + "plans": true} // tableBesideVerbs are the JSON verbs whose document carries the table a person reads as its `text`: the // verb answers that table as its output, as it did before it answered data, and the rest as `answer` — // both from the one read, so they never disagree (novox/hq issue 455). -var tableBesideVerbs = map[string]bool{"queue": true} +var tableBesideVerbs = map[string]bool{"queue": true, "plans": true} // overviewVerbs are the JSON verbs whose answer is said once, as data, and not again as the text the // command printed: the overviews, which grow with the mesh (novox/hq issue 314). Every reader of theirs @@ -1264,6 +1281,16 @@ func seatToolHandlers() (map[string]link.ToolHandler, []string, error) { // Whatever it did, `status` is composed again once it has. defer statusFrom.nudge() } + if verb == "plans" && len(argv) > 1 && argv[1] == "wait" { + // A wait holds its answer up to its bound (novox/hq issue 229), and is answered in full inside it. + asked := "" + if i := slices.Index(argv, "--bound"); i > 0 && i+1 < len(argv) { + asked = argv[i+1] + } + if _, bound, err := planWaitAsked("", asked); err == nil { + link.AnswersWithin(ctx, bound+planWaitStart) + } + } if answer, read := readHere(ctx, argv); read { return answer, nil } diff --git a/internal/catalogue/verbs.go b/internal/catalogue/verbs.go index 93a62ce8..af7cc055 100644 --- a/internal/catalogue/verbs.go +++ b/internal/catalogue/verbs.go @@ -116,9 +116,15 @@ var ControllerVerbs = []Verb{ "limit": "how many builds to list (default 20); not with log", }, nil)}, {Name: "plans", Description: "What the last merges produced and where each stands (novox/hq ADR 0162): " + - "the tiers, the tier a plan is at, what it waits for and since when; one plan whole, given its id.", + "the tiers, the tier a plan is at, what it waits for and since when; one plan whole, given its id; " + + "or, given wait, one plan followed until it moves.", Input: schema(map[string]string{ - "id": "a plan's id (as `plans` lists them): that plan, tier by tier", + "id": "a plan's id (as `plans` lists them): that plan, tier by tier", + "wait": "a plan's id: wait until it moves, then answer it as data — the plan, and what ended the wait " + + "(moved, tier, done, or bound); ask again to follow it further (novox/hq issue 229)", + "until": "with wait: moved (any change, the default), tier (it reaches another tier) or done (it ends)", + "bound": "with wait: how long to wait at most, as 15s or seconds — 20s, the default, is the longest, " + + "inside what a call is answered within", "stop": "a plan's id: stop it — what was asked still builds, nothing further is asked", "close": "a plan's id: close a plan that will not move again, as failed by hand (novox/hq issue 254)", "retry": "a failed plan's id: ask its failed builds again under new ids, and carry the plan on from that tier (novox/hq ADR 0219)", diff --git a/internal/link/calls.go b/internal/link/calls.go index fa1d4740..262defee 100644 --- a/internal/link/calls.go +++ b/internal/link/calls.go @@ -601,6 +601,24 @@ func Acknowledge(ctx context.Context) { } } +// LongestAnswer is the longest a handler that waits on purpose may hold its caller's one answer +// (AnswersWithin, novox/hq issue 229): inside the console's thirty seconds with room for the trip back, +// and inside the bus's window for an answer (broker.ResponseTTL). A variable so a test need not wait. +var LongestAnswer = 25 * time.Second + +type answerWithinKey struct{} + +// AnswersWithin asks that the call's caller be answered in full if the handler ends within d of the +// call's start, rather than within AnswerWithin: for a handler that waits on purpose, with a bound — +// `plans` waiting for a plan to move (novox/hq issue 229). Past d, or past LongestAnswer whichever is +// shorter, its caller is told it is running, as for any call. Never shortens the window; without effect +// outside a served call, and once the caller was answered. +func AnswersWithin(ctx context.Context, d time.Duration) { + if f, ok := ctx.Value(answerWithinKey{}).(func(time.Duration)); ok { + f(d) + } +} + // running is the interim answer: what a caller reads when the call has not finished. func running(c *Call, within time.Duration, acknowledged bool, follow string) []byte { why := fmt.Sprintf("it has not finished in %s", within) @@ -645,6 +663,13 @@ func (l *CallLog) serveCallWithin(seat, verb string, args json.RawMessage, reply acknowledged := make(chan struct{}) var once sync.Once ctx = context.WithValue(ctx, answerNowKey{}, func() { once.Do(func() { close(acknowledged) }) }) + longer := make(chan time.Duration, 1) + ctx = context.WithValue(ctx, answerWithinKey{}, func(d time.Duration) { + select { + case longer <- d: + default: + } + }) type outcome struct { body []byte @@ -703,17 +728,29 @@ func (l *CallLog) serveCallWithin(seat, verb string, args json.RawMessage, reply } } } - timer := time.NewTimer(AnswerWithin) + within, start := AnswerWithin, time.Now() + timer := time.NewTimer(within) defer timer.Stop() - select { - case o := <-done: - l.finish(c, o.body, o.failed, false) - say(o.body) - return - case <-acknowledged: - say(running(c, AnswerWithin, true, l.Follow)) - case <-timer.C: - say(running(c, AnswerWithin, false, l.Follow)) +waiting: + for { + select { + case o := <-done: + l.finish(c, o.body, o.failed, false) + say(o.body) + return + case <-acknowledged: + say(running(c, within, true, l.Follow)) + break waiting + case d := <-longer: + // A handler waiting on purpose (AnswersWithin): its window is stretched, never past LongestAnswer. + if d = min(d, LongestAnswer); d > within { + within = d + timer.Reset(time.Until(start.Add(d))) + } + case <-timer.C: + say(running(c, within, false, l.Follow)) + break waiting + } } o := <-done l.finish(c, o.body, o.failed, true) diff --git a/internal/link/calls_wait_test.go b/internal/link/calls_wait_test.go new file mode 100644 index 00000000..6c3ef638 --- /dev/null +++ b/internal/link/calls_wait_test.go @@ -0,0 +1,55 @@ +package link + +import ( + "context" + "encoding/json" + "testing" + "time" +) + +// **A handler that waits on purpose is answered in full when it ends inside the window it asked for** +// (novox/hq issue 229): `plans` asked to wait for a plan holds its caller's one answer until the plan +// moves or its bound passes, rather than being answered "still running" at AnswerWithin. +func TestAHandlerThatWaitsOnPurposeIsAnsweredInFullInsideItsWindow(t *testing.T) { + shortWindow(t, 20*time.Millisecond) + l, a := NewCallLog(), newAnswers(t) + l.serveCall("mesh-controller", "plans", json.RawMessage(`{"wait":"plan-1"}`), "_INBOX.x.3", + func(ctx context.Context, _ json.RawMessage) (any, error) { + AnswersWithin(ctx, 200*time.Millisecond) + time.Sleep(80 * time.Millisecond) // past AnswerWithin, inside what it asked for + return "plan-1 moved", nil + }, a.respond, nil) + if got := a.only()["result"]; got != "plan-1 moved" { + t.Fatalf("a waiting handler inside its window was answered %v, not in full", got) + } +} + +// What a handler may ask for is bounded: past LongestAnswer its caller is told it is running, as any +// call is, so no answer is ever later than the console waits. +func TestAWaitIsNeverLongerThanTheLongestAnswer(t *testing.T) { + shortWindow(t, 10*time.Millisecond) + was := LongestAnswer + LongestAnswer = 30 * time.Millisecond + t.Cleanup(func() { LongestAnswer = was }) + l, a := NewCallLog(), newAnswers(t) + release := make(chan struct{}) + finished := make(chan struct{}) + go func() { + l.serveCall("mesh-controller", "plans", nil, "_INBOX.x.4", func(ctx context.Context, _ json.RawMessage) (any, error) { + AnswersWithin(ctx, time.Hour) + <-release + return "late", nil + }, a.respond, nil) + close(finished) + }() + select { + case <-a.sent: + case <-time.After(2 * time.Second): + t.Fatal("a handler asking for an hour held its caller past LongestAnswer") + } + if got, _ := a.only()["result"].(map[string]any); got["running"] != true { + t.Fatalf("past LongestAnswer the caller was answered %v, not that it is running", a.only()) + } + close(release) + <-finished +} diff --git a/testdata/store-tests b/testdata/store-tests index 591ba44c..78c6a831 100644 --- a/testdata/store-tests +++ b/testdata/store-tests @@ -7,6 +7,7 @@ ./cmd/mesh-controller TestAStopNamesWhatStaysOnTheMachinesAndWhatStillCarriesIt ./cmd/mesh-controller TestAStopWithoutAMergeLeavesItOutOfABatch ./cmd/mesh-controller TestSavingPlansAndMovingMergesIsOneAct +./cmd/mesh-controller TestAPlanFollowedFromTheStoreAnswersWhenItMoves ./internal/inventory TestTheLastSendSaysWhatTheMachineAnswered ./internal/inventory TestAReportIsTheLastSendsAnswerOnlyWhenItIsAboutThatSend ./cmd/mesh-controller TestNodeShowAndStatusReadTheLastSendFromTheStore From e9f060e0bec3c2bda0365e40b8a90a56ce11aead Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 12 Oct 2026 16:33:23 +0200 Subject: [PATCH 12/26] Read a plan by its id as a parameter, never as SQL (issue 561) PlanByID pasted its id into the query, and the id comes from a caller of the plans verb: an id holding a quote changed the query (x' or '1'='1 answered another plan). The id is now $1, and RecentPlans's limit too, so no plan query is built from a value. --- internal/inventory/plan_id_test.go | 45 ++++++++++++++++++++++++++++++ internal/inventory/plans.go | 7 +++-- testdata/store-tests | 1 + 3 files changed, 50 insertions(+), 3 deletions(-) create mode 100644 internal/inventory/plan_id_test.go diff --git a/internal/inventory/plan_id_test.go b/internal/inventory/plan_id_test.go new file mode 100644 index 00000000..76323226 --- /dev/null +++ b/internal/inventory/plan_id_test.go @@ -0,0 +1,45 @@ +package inventory + +import ( + "strings" + "testing" + "time" +) + +// **A plan's id is a value, never SQL** (novox/hq issue 561): the id reaches PlanByID from a caller of the +// controller's plans verb, so an id holding a quote is read as a plain id — no error from the store, and no +// other plan answered for it. +func TestAPlanIdHoldingAQuoteIsReadAsAPlainId(t *testing.T) { + inv := ForTest(t) + ctx := t.Context() + p := Plan{ID: "plan-561", Repository: "novox/mesh-controller", Commit: "abc", Created: time.Now().UTC(), + State: PlanBuilding, Tiers: [][]string{{"a"}}, Modules: map[string]*PlanModule{"a": {}}} + if err := inv.SavePlan(ctx, &p); err != nil { + t.Fatal(err) + } + for _, id := range []string{ + "x' or '1'='1", // every plan, when pasted into the query + "plan-561' or id like '%", // the plan above, by a pattern + "it's", // a quote alone: a syntax error, when pasted + } { + got, err := inv.PlanByID(ctx, id) + if err == nil { + t.Fatalf("PlanByID(%q) answered %s, a plan of another id", id, got.ID) + } + if want := "no plan " + id; err.Error() != want { + t.Fatalf("PlanByID(%q) failed with %q, not as a plain id no plan has (%q)", id, err, want) + } + } + // A plan whose id holds a quote is found by it, and by nothing else. + quoted := p + quoted.ID, quoted.Revision = "plan-'561'", 0 + if err := inv.SavePlan(ctx, "ed); err != nil { + t.Fatal(err) + } + if got, err := inv.PlanByID(ctx, quoted.ID); err != nil || got.ID != quoted.ID { + t.Fatalf("the plan with a quote in its id was not read back: %v %+v", err, got.ID) + } + if recent, err := inv.RecentPlans(ctx, 1); err != nil || len(recent) != 1 || !strings.HasPrefix(recent[0].ID, "plan-") { + t.Fatalf("RecentPlans with a parameter: %v %+v", err, recent) + } +} diff --git a/internal/inventory/plans.go b/internal/inventory/plans.go index b6530854..d6933d04 100644 --- a/internal/inventory/plans.go +++ b/internal/inventory/plans.go @@ -623,7 +623,7 @@ func (i *Inventory) PlansSince(ctx context.Context, since time.Time) ([]Plan, er // RecentPlans is the last few plans, newest first, open or not — what the overview shows. func (i *Inventory) RecentPlans(ctx context.Context, limit int) ([]Plan, error) { - return i.plans(ctx, fmt.Sprintf(`order by created desc limit %d`, limit)) + return i.plans(ctx, `order by created desc limit $1`, limit) } // NewestMergeOf is the plan, in any state, of the newest merge into a repository's branch that the mesh @@ -638,9 +638,10 @@ func (i *Inventory) NewestMergeOf(ctx context.Context, repository, branch string return plans[0], true, nil } -// PlanByID is one plan. +// PlanByID is one plan. The id is a parameter, never part of the query (novox/hq issue 561): it comes from a +// caller of the plans verb. func (i *Inventory) PlanByID(ctx context.Context, id string) (Plan, error) { - plans, err := i.plans(ctx, `where id = '`+id+`'`) + plans, err := i.plans(ctx, `where id = $1`, id) if err != nil { return Plan{}, err } diff --git a/testdata/store-tests b/testdata/store-tests index 591ba44c..cdb56e70 100644 --- a/testdata/store-tests +++ b/testdata/store-tests @@ -13,6 +13,7 @@ ./internal/inventory TestAnIssuedValueIsSealedToEveryHolderOnceAndNotAgain ./internal/inventory TestAnIssuedValueIsSealedAgainAfterAKeyChangeOrAnAcceptance ./internal/inventory TestAConditionsLooksAreKeptBesideTheBucket +./internal/inventory TestAPlanIdHoldingAQuoteIsReadAsAPlainId ./cmd/mesh-controller TestTheConditionsVerbReadsTheLooksTheBucketNoLongerHolds ./internal/inventory TestAWalksStartIsKeptAndARefusedSaveLeavesItUnsaid ./cmd/mesh-controller TestAWalkLetGoAndStoppedByVerbsIsSaidStartedAndEnded From a1a6f6493d50c4afe40e09819e661305df8eed32 Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 12 Oct 2026 16:46:29 +0200 Subject: [PATCH 13/26] Refuse a plans argument that is not a plan's id (issue 561) Every argument of plans that names a plan (id, wait, stop, close, retry, go) is checked once where the call becomes a command line, against the ids the mesh gives plans (plan-, release-), so a quote from a caller never reaches a command or the store (review 75 on #266). --- cmd/mesh-controller/plan_ids.go | 26 +++++++++++++++++++ cmd/mesh-controller/plan_ids_test.go | 37 ++++++++++++++++++++++++++++ cmd/mesh-controller/seatverbs.go | 3 +++ 3 files changed, 66 insertions(+) create mode 100644 cmd/mesh-controller/plan_ids.go create mode 100644 cmd/mesh-controller/plan_ids_test.go diff --git a/cmd/mesh-controller/plan_ids.go b/cmd/mesh-controller/plan_ids.go new file mode 100644 index 00000000..d9dca230 --- /dev/null +++ b/cmd/mesh-controller/plan_ids.go @@ -0,0 +1,26 @@ +package main + +import ( + "fmt" + "regexp" +) + +// planID is every id the mesh gives a plan: a merge's walk or batch (plan-) or a release +// (release-). +var planID = regexp.MustCompile(`^(plan|release)-[0-9]+$`) + +// planIDArguments are the arguments of plans that name a plan. wait is among them for when it is declared +// (novox/hq issue 229); an argument not declared is refused before this is asked. +var planIDArguments = []string{"id", "wait", "stop", "close", "retry", "go"} + +// plainPlanIDs refuses a plans call naming a plan by anything but a plan's id (novox/hq issue 561): checked +// once, where the call becomes a command line, so nothing else a caller sent reaches a command or the store. +func plainPlanIDs(given map[string]string) error { + for _, arg := range planIDArguments { + if id, named := given[arg]; named && !planID.MatchString(id) { + return fmt.Errorf("plans %s: %q is not a plan's id (plan- or release-, as `plans` "+ + "lists them); nothing was done", arg, id) + } + } + return nil +} diff --git a/cmd/mesh-controller/plan_ids_test.go b/cmd/mesh-controller/plan_ids_test.go new file mode 100644 index 00000000..f49039f4 --- /dev/null +++ b/cmd/mesh-controller/plan_ids_test.go @@ -0,0 +1,37 @@ +package main + +import ( + "strings" + "testing" +) + +// **A plan's id is checked once, where a call becomes a command line** (novox/hq issue 561): every argument of +// plans that names a plan is refused unless it is an id the mesh gives a plan, so a quote never reaches a +// command, nor the store behind it. +func TestAPlanIdThatIsNoPlansIdIsRefusedBeforeACommandStarts(t *testing.T) { + for _, arg := range []string{"id", "stop", "close", "retry", "go"} { + for _, bad := range []string{"x' or '1'='1", "plan-1'", "plan-", "plan-1 2", "batch-1", "PLAN-1"} { + args := map[string]any{arg: bad} + if arg == "stop" || arg == "close" || arg == "go" { + args["why"] = "a test" + } + argv, err := argvFor("plans", args) + if err == nil { + t.Errorf("plans %s %q was composed: %v", arg, bad, argv) + continue + } + if !strings.Contains(err.Error(), "is not a plan's id") { + t.Errorf("plans %s %q was refused for another reason: %v", arg, bad, err) + } + } + for _, good := range []string{"plan-1791813626034291775", "release-1791811938267404408"} { + args := map[string]any{arg: good} + if arg == "stop" || arg == "close" || arg == "go" { + args["why"] = "a test" + } + if _, err := argvFor("plans", args); err != nil { + t.Errorf("plans %s %q was refused: %v", arg, good, err) + } + } + } +} diff --git a/cmd/mesh-controller/seatverbs.go b/cmd/mesh-controller/seatverbs.go index 3e4d1d18..017d9fea 100644 --- a/cmd/mesh-controller/seatverbs.go +++ b/cmd/mesh-controller/seatverbs.go @@ -336,6 +336,9 @@ func (a *verbArguments) commandLine() ([]string, error) { } return argv, nil case "plans": + if err := plainPlanIDs(a.given); err != nil { + return nil, err + } if r := str("repository"); r != "" { argv := []string{"plans", "--what-if", r} if p := str("paths"); p != "" { From 5ad36127ee8bf3a5c3c99dc4821a78a9a98f9b88 Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 12 Oct 2026 16:50:59 +0200 Subject: [PATCH 14/26] Hold the plans schema's longest bound to the value, and say what a slow store gives (review 75) --- cmd/mesh-controller/plans_follow.go | 2 ++ cmd/mesh-controller/plans_follow_test.go | 15 +++++++++++++++ 2 files changed, 17 insertions(+) diff --git a/cmd/mesh-controller/plans_follow.go b/cmd/mesh-controller/plans_follow.go index c13dedc7..daaaeed7 100644 --- a/cmd/mesh-controller/plans_follow.go +++ b/cmd/mesh-controller/plans_follow.go @@ -26,6 +26,8 @@ import ( var maxPlanWait = link.LongestAnswer - planWaitStart // planWaitStart is what the verb's window allows beyond the bound: starting the command and opening the stores. +// A store slow enough to push the read and the answer past it degrades as any long call does: the caller is +// answered that the call is still running, and `calls` with its id says what came of it. const planWaitStart = 5 * time.Second // planWaitEvery is how often a wait reads the plan again. diff --git a/cmd/mesh-controller/plans_follow_test.go b/cmd/mesh-controller/plans_follow_test.go index 844b99e7..b3932d59 100644 --- a/cmd/mesh-controller/plans_follow_test.go +++ b/cmd/mesh-controller/plans_follow_test.go @@ -179,3 +179,18 @@ func TestAPlanFollowedFromTheStoreAnswersWhenItMoves(t *testing.T) { t.Fatalf("a plan that did not move answered %+v (%v)", got, err) } } + +// The verb's schema says the longest bound in words, and the words are the value (review 75 on #266): the +// catalogue cannot read maxPlanWait, so this holds them together. +func TestThePlansSchemaSaysTheLongestBoundThereIs(t *testing.T) { + v, ok := controllerVerb("plans") + if !ok { + t.Fatal("no plans verb") + } + props, _ := v.Input["properties"].(map[string]any) + bound, _ := props["bound"].(map[string]any) + said, _ := bound["description"].(string) + if !strings.Contains(said, maxPlanWait.String()+", the default, is the longest") { + t.Fatalf("the schema says %q; the longest bound is %s", said, maxPlanWait) + } +} From edd33f21d2acea8d14c3732bb8671cb0cab1b106 Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 12 Oct 2026 14:41:28 +0200 Subject: [PATCH 15/26] Make no value for an own secret issued outside the mesh that nobody gave (issue 401) The send minted a random value for a plain own secret marked issued-by outside and placed it, so a module could not tell "not given" from "given and wrong" (ADR 0228). It now places only a value a person gave: not given is no file, an empty value where a module's file names it, and nothing to restart on, as ADR 0283 decided for a family's member. A value the mesh made for one earlier is not read as given. --- cmd/mesh-controller/outside_not_given_test.go | 120 ++++++++++++++++++ cmd/mesh-controller/plan.go | 38 +++++- internal/catalogue/declaration.go | 10 +- internal/catalogue/outside_not_given_test.go | 106 ++++++++++++++++ internal/catalogue/secrets_into_files.go | 27 +++- internal/inventory/secrets.go | 31 +++++ 6 files changed, 324 insertions(+), 8 deletions(-) create mode 100644 cmd/mesh-controller/outside_not_given_test.go create mode 100644 internal/catalogue/outside_not_given_test.go diff --git a/cmd/mesh-controller/outside_not_given_test.go b/cmd/mesh-controller/outside_not_given_test.go new file mode 100644 index 00000000..11a096c0 --- /dev/null +++ b/cmd/mesh-controller/outside_not_given_test.go @@ -0,0 +1,120 @@ +package main + +import ( + "testing" + + "github.com/novox/mesh-controller/internal/catalogue" +) + +// An own secret issued outside the mesh that nobody gave is never made (novox/hq issue 401, ADR 0228). Seen on +// 2026-10-10: the send made a random value for the mounts module's smb-credentials and placed it, and the module read +// it as a credential given and wrong. The send now makes none, places none, and places a value only once a person +// gave one — and a value the mesh made for it before is not placed either. + +// anAssistant is a module with a secret the mesh makes and one only an outside party issues. +func anAssistant() catalogue.Manifest { + return catalogue.Manifest{Module: "assistant", Version: "1", + OwnSecrets: catalogue.OwnSecrets{ + "password": {Path: "/var/lib/mesh/assistant/password", Taken: catalogue.TakenAtStart}, + "vendor-key": {Path: "/var/lib/mesh/assistant/vendor-key", Taken: catalogue.TakenAtStart, IssuedBy: catalogue.IssuedOutside}, + }, + Resources: []map[string]any{ + {"id": "state", "type": "directory", "path": "/var/lib/mesh/assistant", "mode": "0700"}, + }} +} + +// sentTo composes what a push sends to a machine, on the send path. +func sentTo(t *testing.T, open *stores, node string) []map[string]any { + t.Helper() + plan, settings, err := planFor(t.Context(), open, node) + if err != nil { + t.Fatal(err) + } + declared, err := declarationFor(t.Context(), open, node, plan, settings) + if err != nil { + t.Fatalf("the send to %s did not compose: %v", node, err) + } + return declared.Resources +} + +func placedAt(resources []map[string]any, path string) map[string]any { + for _, r := range resources { + if r["path"] == path && r["sealed"] != nil { + return r + } + } + return nil +} + +func TestTheSendMakesNoValueForAnOutsideSecretNobodyGave(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + register(t, open, anAssistant()) + if _, err := assign(ctx, open, "laptop", "assistant"); err != nil { + t.Fatal(err) + } + + sent := sentTo(t, open, "laptop") + if placedAt(sent, "/var/lib/mesh/assistant/password") == nil { + t.Fatal("the secret the mesh makes was not placed") + } + if got := placedAt(sent, "/var/lib/mesh/assistant/vendor-key"); got != nil { + t.Fatalf("a secret nobody gave was placed: %v", got) + } + if _, held, err := open.inventory.ModuleSecretIfIssued(ctx, "laptop", "assistant", "vendor-key"); err != nil || held { + t.Fatalf("the send made a value for a secret issued outside the mesh (held %v, %v)", held, err) + } + + // A value the mesh made before this rule is not one a person gave: it is not placed. + if _, err := open.inventory.SecretForModule(ctx, "laptop", "assistant", "vendor-key"); err != nil { + t.Fatal(err) + } + if got := placedAt(sentTo(t, open, "laptop"), "/var/lib/mesh/assistant/vendor-key"); got != nil { + t.Fatalf("a value the mesh made for a secret issued outside was placed: %v", got) + } + + // Given, it is placed. + if err := open.inventory.AcceptSecretForModule(ctx, "laptop", "assistant", "vendor-key", "the vendor's key"); err != nil { + t.Fatal(err) + } + given, ok, err := open.inventory.GivenOwnSecret(ctx, "laptop", "assistant", "vendor-key") + if err != nil || !ok || !given.Current { + t.Fatalf("the value given is not read as given: %+v %v %v", given, ok, err) + } + placed := placedAt(sentTo(t, open, "laptop"), "/var/lib/mesh/assistant/vendor-key") + if placed == nil || placed["sealed"] != given.Sealed { + t.Fatalf("the value given is placed as %v", placed) + } +} + +// The read paths answer as the send does: a question about the machine composes without the secret nobody gave, and +// asking writes nothing. +func TestAQuestionComposesWithoutAnOutsideSecretNobodyGave(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + register(t, open, anAssistant()) + if _, err := assign(ctx, open, "laptop", "assistant"); err != nil { + t.Fatal(err) + } + pushedBy(t, open, "laptop") // the mesh's own secret is made by the send + plan, settings, err := planFor(ctx, open, "laptop") + if err != nil { + t.Fatal(err) + } + gens, err := generators(ctx, open) + if err != nil { + t.Fatal(err) + } + for _, choosing := range []Choosing{Reading, Foreseeing} { + got, err := declarationWith(ctx, open, "laptop", plan, settings, gens, choosing) + if err != nil { + t.Fatalf("%v: %v", choosing, err) + } + if placedAt(got.Resources, "/var/lib/mesh/assistant/vendor-key") != nil || len(got.foreseen) != 0 { + t.Fatalf("%v: a secret nobody gave is composed (foreseen %v)", choosing, got.foreseen) + } + } + if _, held, err := open.inventory.ModuleSecretIfIssued(ctx, "laptop", "assistant", "vendor-key"); err != nil || held { + t.Fatalf("a question made a value (held %v, %v)", held, err) + } +} diff --git a/cmd/mesh-controller/plan.go b/cmd/mesh-controller/plan.go index 16aa5515..d4401408 100644 --- a/cmd/mesh-controller/plan.go +++ b/cmd/mesh-controller/plan.go @@ -590,6 +590,13 @@ func busCredentialIssued(ctx context.Context, inv *inventory.Inventory, node, mo return nil } +// givenUnderAnOldKey is the refusal for a value a person gave — a family's member, or an own secret issued outside +// the mesh — sealed to a key the machine no longer holds: the mesh cannot make another, so it is given again. +func givenUnderAnOldKey(module, node, name string) error { + return fmt.Errorf("%s on %s holds %q, which was given to the mesh rather than made by it, and %s has "+ + "since generated a new sealing key. The mesh cannot make another; give it again", module, node, name, node) +} + // renderingFor is everything a node's declaration is composed with, and the node's record. func renderingFor(ctx context.Context, open *stores, node string, plan catalogue.Resolution, settings catalogue.SettingsBy, @@ -708,10 +715,7 @@ func renderingFor(ctx context.Context, open *stores, node string, } if !g.Current { if choosing == Allocating { - return catalogue.Rendering{}, inventory.Node{}, fmt.Errorf( - "%s on %s holds %q, which was given to the mesh rather than made by it, and %s has "+ - "since generated a new sealing key. The mesh cannot make another; give it again", - m.Module, node, g.Name, node) + return catalogue.Rendering{}, inventory.Node{}, givenUnderAnOldKey(m.Module, node, g.Name) } continue } @@ -721,7 +725,31 @@ func renderingFor(ctx context.Context, open *stores, node string, needed[m.Module][g.Name] = g.Sealed } } - for name := range m.OwnSecrets.Plain() { + for name, own := range m.OwnSecrets.Plain() { + // **An own secret issued outside the mesh is never made** (novox/hq issue 401, ADR 0228; ADR 0283 for + // a family): only a value a person gave is placed. One not given is nothing — no file, and an empty + // value where a file of the module names it — so the module can tell it was not given, where a value + // the mesh made looked given and wrong. A value the mesh made for it before is not one either. + if own.IssuedBy == catalogue.IssuedOutside { + g, given, err := inv.GivenOwnSecret(ctx, node, m.Module, name) + if err != nil { + return catalogue.Rendering{}, inventory.Node{}, err + } + if !given { + continue + } + if !g.Current { + if choosing == Allocating { + return catalogue.Rendering{}, inventory.Node{}, givenUnderAnOldKey(m.Module, node, name) + } + continue + } + if needed[m.Module] == nil { + needed[m.Module] = map[string]string{} + } + needed[m.Module][name] = g.Sealed + continue + } // Minted on the send path and only read on every other. Making one is an insert, and // a question that writes is a question that can block against the machine it is about. var sealed string diff --git a/internal/catalogue/declaration.go b/internal/catalogue/declaration.go index facc148e..a73f873b 100644 --- a/internal/catalogue/declaration.go +++ b/internal/catalogue/declaration.go @@ -653,6 +653,11 @@ func (r Resolution) compose(with Rendering, owner map[string]string, sealed = ForeseenSealed *foreseen = append(*foreseen, m.Module+"/"+name) } + if sealed == "" && m.OwnSecrets[name].IssuedBy == IssuedOutside { + // **Not given, and the mesh never makes it** (novox/hq issue 401, ADR 0228): no file, as for a + // family's member (ADR 0283), so the module reads its absence as not given. + continue + } if sealed == "" { // Declared and not made. Refused rather than skipped: a module whose own // credential is silently absent starts, fails to authenticate, and the reason is @@ -1010,7 +1015,7 @@ func (r Resolution) compose(with Rendering, owner map[string]string, // content is where a setting lands, so a placeholder may only exist once the setting // has been put in — filling secrets first would look at content that is not yet what // the machine receives. - if err := intoFile(copied, sealed, m.Module); err != nil { + if err := intoFile(copied, sealed, outsideNotGiven(m, with.Needed[m.Module]), m.Module); err != nil { return nil, err } // **Substituted here, not on the machine.** A bound value is not secret — the mesh @@ -2453,6 +2458,9 @@ func secretsReadBy(resource map[string]any, m Manifest, given map[string]string) } paths := map[string]string{} for name, own := range m.OwnSecrets.Plain() { + if own.IssuedBy == IssuedOutside && given[name] == "" { + continue // not given, so not placed: nothing to restart on (novox/hq issue 401) + } paths[name] = own.Path } for name, sealed := range given { diff --git a/internal/catalogue/outside_not_given_test.go b/internal/catalogue/outside_not_given_test.go new file mode 100644 index 00000000..a63e3d8a --- /dev/null +++ b/internal/catalogue/outside_not_given_test.go @@ -0,0 +1,106 @@ +package catalogue + +import ( + "strings" + "testing" +) + +// An own secret issued outside the mesh that nobody gave (novox/hq issue 401, ADR 0228): the mesh makes no value for it, +// so the composition places no file, a file of the module that names it says empty there, and nothing restarts on it. +// Given, it is placed and filled like any own secret. + +func outsideManifest(t *testing.T) Manifest { + t.Helper() + m, err := ParseManifest([]byte(`{"module":"assistant","version":"1", + "own-secrets":{ + "password":{"path":"/var/lib/assistant/password.secret","taken":"at-start"}, + "vendor-key":{"path":"/var/lib/assistant/vendor-key.secret","taken":"at-start","issued-by":"outside"}}, + "resources":[ + {"id":"env","type":"file","path":"/var/lib/assistant/env","mode":"0600", + "content":"PASSWORD=${secret:password}\nVENDOR_KEY=${secret:vendor-key}\n"}, + {"id":"server","type":"container","name":"assistant","image":"assistant:1", + "env-file":["/var/lib/assistant/env"], + "volumes":["/var/lib/assistant/vendor-key.secret:/run/vendor-key:ro"], + "secrets-in-environment":"reads its settings from the environment only"}]}`)) + if err != nil { + t.Fatal(err) + } + return m +} + +func resourceAt(out []map[string]any, path string) map[string]any { + for _, r := range out { + if r["path"] == path { + return r + } + } + return nil +} + +func resourceNamed(out []map[string]any, name string) map[string]any { + for _, r := range out { + if r["name"] == name { + return r + } + } + return nil +} + +func TestAnOutsideSecretNotGivenIsNoFileAndAFileNamingItSaysEmpty(t *testing.T) { + r := Resolution{Node: "server", Modules: []Manifest{outsideManifest(t)}} + out, err := r.Declaration(Rendering{Needed: map[string]map[string]string{"assistant": {"password": "sealed-password"}}}) + if err != nil { + t.Fatalf("a module whose outside secret nobody gave did not compose: %v", err) + } + if got := resourceAt(out, "/var/lib/assistant/vendor-key.secret"); got != nil { + t.Fatalf("a secret nobody gave was placed: %v", got) + } + env := resourceAt(out, "/var/lib/assistant/env") + if env == nil { + t.Fatal("the module's own file is not composed") + } + if content := env["content"].(string); content != "PASSWORD=${secret:password}\nVENDOR_KEY=\n" { + t.Fatalf("the file names the secret nobody gave as %q", content) + } + secrets, _ := env["secrets"].(map[string]any) + if len(secrets) != 1 || secrets["password"] != "sealed-password" { + t.Fatalf("the file is filled from %v", secrets) + } + server := resourceNamed(out, "assistant") + if on := strings.Join(stringsIn(server["restart-on"]), " "); strings.Contains(on, "vendor-key") { + t.Fatalf("the container restarts on a secret nobody gave: %q", on) + } +} + +func TestAnOutsideSecretGivenIsPlacedAndFilled(t *testing.T) { + r := Resolution{Node: "server", Modules: []Manifest{outsideManifest(t)}} + out, err := r.Declaration(Rendering{Needed: map[string]map[string]string{"assistant": { + "password": "sealed-password", "vendor-key": "sealed-vendor-key"}}}) + if err != nil { + t.Fatal(err) + } + placed := resourceAt(out, "/var/lib/assistant/vendor-key.secret") + if placed == nil || placed["sealed"] != "sealed-vendor-key" { + t.Fatalf("the secret given is placed as %v", placed) + } + env := resourceAt(out, "/var/lib/assistant/env") + if content := env["content"].(string); content != "PASSWORD=${secret:password}\nVENDOR_KEY=${secret:vendor-key}\n" { + t.Fatalf("the file was changed: %q", content) + } + if secrets, _ := env["secrets"].(map[string]any); secrets["vendor-key"] != "sealed-vendor-key" { + t.Fatalf("the file is filled from %v", secrets) + } + server := resourceNamed(out, "assistant") + if on := strings.Join(stringsIn(server["restart-on"]), " "); !strings.Contains(on, "assistant."+NeedID("vendor-key")) { + t.Fatalf("the container does not restart on the secret given: %q", on) + } +} + +// A secret the mesh makes is still refused when it was not made: only an outside one is left out. +func TestAnOwnSecretTheMeshMakesIsStillRefusedWhenNotMade(t *testing.T) { + r := Resolution{Node: "server", Modules: []Manifest{outsideManifest(t)}} + _, err := r.Declaration(Rendering{}) + if nm, ok := err.(*NotMadeError); !ok || nm.Name != "password" { + t.Fatalf("a secret the mesh makes, not made: %v", err) + } +} diff --git a/internal/catalogue/secrets_into_files.go b/internal/catalogue/secrets_into_files.go index 106d7898..0ed07c37 100644 --- a/internal/catalogue/secrets_into_files.go +++ b/internal/catalogue/secrets_into_files.go @@ -156,7 +156,11 @@ func secretFilesOf(resources []map[string]any) map[string]bool { // A name the module never declared is refused here rather than on the machine. The host would // refuse it too — but it would do so having already been handed a declaration, which reads as the // mesh sending something broken, and the name it could not find is a manifest's typo. -func intoFile(resource map[string]any, sealed map[string]string, module string) error { +// +// **An own secret issued outside the mesh that nobody gave is filled with nothing** (novox/hq issue 401): the mesh +// never makes one (ADR 0228), and a file is where the module reads it, so the place says empty — the module sees it +// was not given, where a value the mesh made looked given and wrong. notGiven names those secrets. +func intoFile(resource map[string]any, sealed map[string]string, notGiven map[string]bool, module string) error { if fmt.Sprint(resource["type"]) != "file" { return nil } @@ -171,6 +175,10 @@ func intoFile(resource map[string]any, sealed map[string]string, module string) into := map[string]any{} for _, name := range used { value := sealed[name] + if value == "" && notGiven[name] { + content = strings.ReplaceAll(content, "${secret:"+name+"}", "") + continue + } if value == "" { return fmt.Errorf( "%s has a file that says ${secret:%s}, and %s has no secret of its own by that "+ @@ -179,10 +187,25 @@ func intoFile(resource map[string]any, sealed map[string]string, module string) } into[name] = value } - resource["secrets"] = into + resource["content"] = content + if len(into) > 0 { + resource["secrets"] = into + } return nil } +// outsideNotGiven is each own secret of a module issued outside the mesh that this composition holds no value for +// (novox/hq issue 401): what intoFile fills with nothing, and what nothing restarts on. +func outsideNotGiven(m Manifest, given map[string]string) map[string]bool { + out := map[string]bool{} + for name, own := range m.OwnSecrets.Plain() { + if own.IssuedBy == IssuedOutside && given[name] == "" { + out[name] = true + } + } + return out +} + // namesOr says what a module could have written, because the answer to "that name is wrong" is // almost always one of two or three right ones. func namesOr(sealed map[string]string) string { diff --git a/internal/inventory/secrets.go b/internal/inventory/secrets.go index 34ee9cb1..c51480a1 100644 --- a/internal/inventory/secrets.go +++ b/internal/inventory/secrets.go @@ -325,6 +325,8 @@ func (i *Inventory) ModuleSecretIfIssued( return sealed, true, nil } +// SecretForModule makes what is not held, so it is never asked for an own secret issued outside the mesh: the +// composition reads only a value a person gave for one (GivenOwnSecret; novox/hq issue 401, ADR 0228). func (i *Inventory) SecretForModule(ctx context.Context, node, module, name string) (string, error) { key, err := i.SealingKeyOf(ctx, node) if err != nil { @@ -655,6 +657,35 @@ func (i *Inventory) GivenMembers(ctx context.Context, node, module, family strin return out, rows.Err() } +// GivenOwnSecret is an own secret issued outside the mesh as a person gave it on a machine (novox/hq issue 401, ADR +// 0228): its value sealed to the machine and whether that is the key the machine holds now. Not given is false: a +// value the mesh made for it is not one a person gave, and is never read as one; a machine with no sealing key +// holds none either. +func (i *Inventory) GivenOwnSecret(ctx context.Context, node, module, name string) (GivenMember, bool, error) { + key, err := i.SealingKeyOf(ctx, node) + if err != nil || key == "" { + return GivenMember{}, false, err + } + record, err := i.NodeByName(ctx, node) + if err != nil { + return GivenMember{}, false, err + } + g := GivenMember{Name: name} + var against string + err = i.store.Pool().QueryRow(ctx, + `select sealed, node_key from module_secret + where node = $1 and module = $2 and name = $3 and origin = 'accepted'`, + record.ID, module, name).Scan(&g.Sealed, &against) + if errors.Is(err, pgx.ErrNoRows) { + return GivenMember{}, false, nil + } + if err != nil { + return GivenMember{}, false, err + } + g.Current = against == key + return g, true, nil +} + // GivenOwnSecrets is every own secret of a module a person gave on a machine, by name, with when (novox/hq ADR // 0283): what the controller checks a module's wait for a secret against. A value the mesh made is not one. func (i *Inventory) GivenOwnSecrets(ctx context.Context, node, module string) (map[string]time.Time, error) { From c3b4f187e195596e18e16177924d9d96cb7db1e6 Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 12 Oct 2026 15:16:07 +0200 Subject: [PATCH 16/26] Name issue 401's two store tests, so a skip without the store fails the check (issue 459's rule) --- testdata/store-tests | 2 ++ 1 file changed, 2 insertions(+) diff --git a/testdata/store-tests b/testdata/store-tests index 591ba44c..c510f6d9 100644 --- a/testdata/store-tests +++ b/testdata/store-tests @@ -27,3 +27,5 @@ ./cmd/mesh-controller TestAnUnassignThatLeavesItsNodeResolvedSaysNothingOfIt ./cmd/mesh-controller TestAnAssignThatLeavesItsNodeUnresolvedSaysOnlyThePlansRefusal ./cmd/mesh-controller TestADirectoryOwnersKeyIsRefusedThroughAVerb +./cmd/mesh-controller TestTheSendMakesNoValueForAnOutsideSecretNobodyGave +./cmd/mesh-controller TestAQuestionComposesWithoutAnOutsideSecretNobodyGave From 7b050abdcb65dc1748f6b4d6a97b50799a06c580 Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 12 Oct 2026 17:09:57 +0200 Subject: [PATCH 17/26] Stop composing ADR 0312's group grant, so deliveries to the media machine pass again Since #256 delivered, the node-engine judges every read-write access by the path's group alone and states six media modules unhealthy on ace, though they write their folders as the owner; every delivery there fails its gate (hq issue 567). ADR 0325 replaces ADR 0312; until it is built, the controller sends no grant, as before #256. This reverts commit 3cfc12167088e098eca7c58d3ff40d3029e1c3d1. --- cmd/mesh-controller/plan.go | 15 -- go.mod | 2 +- go.sum | 4 +- internal/catalogue/declaration.go | 22 --- internal/catalogue/grants.go | 134 -------------- internal/catalogue/grants_test.go | 174 ------------------ internal/link/protocol.go | 11 -- testdata/beside/CAPTURED | 2 +- .../internal/declaration/declaration.go | 78 -------- vendor/modules.txt | 4 +- 10 files changed, 6 insertions(+), 440 deletions(-) delete mode 100644 internal/catalogue/grants.go delete mode 100644 internal/catalogue/grants_test.go diff --git a/cmd/mesh-controller/plan.go b/cmd/mesh-controller/plan.go index 16aa5515..3fd360c7 100644 --- a/cmd/mesh-controller/plan.go +++ b/cmd/mesh-controller/plan.go @@ -947,14 +947,9 @@ func renderingFor(ctx context.Context, open *stores, node string, if err != nil { return catalogue.Rendering{}, inventory.Node{}, err } - grantsGroups, err := engineGrantsGroups(ctx, inv, node) - if err != nil { - return catalogue.Rendering{}, inventory.Node{}, err - } return catalogue.Rendering{ ReadsHealth: readsHealth, JudgesRoot: judgesRoot, - GrantsGroups: grantsGroups, BusMembership: memberships[node], Settings: settings, Generators: gens, Grants: grants, Needed: needed, Foreseen: foreseen, Ports: ports, Certificate: certificate, Authority: authority, Mesh: private, Names: names, @@ -987,16 +982,6 @@ func engineJudgesRoot(ctx context.Context, inv *inventory.Inventory, node string return had && stated.Contract >= link.RootContract, nil } -// engineGrantsGroups says whether a machine's node-engine reads an access's `grant` (novox/hq ADR 0312), by its own -// newest statement, for the same reason as engineReadsHealth: an older engine parses strictly. -func engineGrantsGroups(ctx context.Context, inv *inventory.Inventory, node string) (bool, error) { - stated, had, err := inv.HealthOf(ctx, node) - if err != nil { - return false, err - } - return had && stated.Contract >= link.GrantContract, nil -} - // zonesInTheMesh is every zone a module in the mesh declares, where the mesh placed it (novox/hq ADR // 0199): the zone settled from that node's settings, the node's private address, the port the // answering listen is published on there. diff --git a/go.mod b/go.mod index 527faf8e..cdf300d5 100644 --- a/go.mod +++ b/go.mod @@ -35,4 +35,4 @@ require ( // committed. Every build (the build agent's `go build`, the Dockerfile) compiles from vendor/ and // fetches nothing; go refuses to build when vendor/ and this file disagree, so a pin moved without // `go mod vendor` fails loudly, at once, everywhere. -replace github.com/novox/mesh-host => git.novox.be/novox/mesh-host v0.0.0-20261012112256-3f7f797168d3 +replace github.com/novox/mesh-host => git.novox.be/novox/mesh-host v0.0.0-20261011092230-20d5af9b2d5f diff --git a/go.sum b/go.sum index dfc79fdf..e7436cbb 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,5 @@ -git.novox.be/novox/mesh-host v0.0.0-20261012112256-3f7f797168d3 h1:VpH8CesuBOo7k8GcXZq1eRUfbvUoktfmcUynI5N3IFQ= -git.novox.be/novox/mesh-host v0.0.0-20261012112256-3f7f797168d3/go.mod h1:tcTK4LMs1d6JpZUwy3hSHMFG/MIuDr/XFs7/nbeaW/c= +git.novox.be/novox/mesh-host v0.0.0-20261011092230-20d5af9b2d5f h1:8N5OW2mdTNIck2pe4EciTYX5NsrPsHrTLENGNIWYNTU= +git.novox.be/novox/mesh-host v0.0.0-20261011092230-20d5af9b2d5f/go.mod h1:tcTK4LMs1d6JpZUwy3hSHMFG/MIuDr/XFs7/nbeaW/c= git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261010191001-33917f91ac3c h1:l5onJwoIeH8yE/PjVlEeoPyXBsHp2NQiWLllz2fwX7s= git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261010191001-33917f91ac3c/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= diff --git a/internal/catalogue/declaration.go b/internal/catalogue/declaration.go index facc148e..a74ca6dc 100644 --- a/internal/catalogue/declaration.go +++ b/internal/catalogue/declaration.go @@ -246,11 +246,6 @@ type Rendering struct { // statement's contract is link.RootContract or later). To an older, strict engine the field is not // sent, and the account it names is not judged — which the self-check says, as not judged. JudgesRoot bool - - // GrantsGroups says this machine's node-engine reads an access's `grant` (novox/hq ADR 0312: its statement's - // contract is link.GrantContract or later). To an older, strict engine the field is not sent, nor the - // runtime's restarts on it, and a read-write access is reached as before: by whoever owns the path. - GrantsGroups bool } // RootField is a user resource's field saying the account must never become root without a person @@ -548,9 +543,6 @@ func (r Resolution) compose(with Rendering, owner map[string]string, } var out []map[string]any - // The read-write accesses each module grants its tools' account, by module (novox/hq ADR 0312): the runtime - // that runs as that account is restarted on them. - grantedToTools := map[string][]string{} for _, m := range r.Modules { if with.Adopted && m.Filtering != nil { // Nothing of a module that loads a filter, on an adopted node: its table would drop @@ -1123,13 +1115,6 @@ func (r Resolution) compose(with Rendering, owner map[string]string, out = append(out, copied) } - // **A read-write access grants its path's group to what runs this module's code** (novox/hq ADR 0312): - // the containers composed above that mount it, and the account its tools run as. Only to an engine that - // reads the field. - if with.GrantsGroups { - grantedToTools[m.Module] = grantAccesses(out, owner, m.Module, r.toolsAccount(m, with)) - } - // **What only the mesh knows, where this module asked for it.** The graph is the control // plane's; making a name resolve is the module's software. Emitted as ordinary files under // this module's name, so they are applied, reported and removed exactly as anything else @@ -1156,12 +1141,6 @@ func (r Resolution) compose(with Rendering, owner map[string]string, } owner[fmt.Sprint(process["id"])] = RuntimeModule out = append(out, process) - // Restarted when an access moved its account's groups, so its process holds them (novox/hq ADR 0312 §6). - for _, m := range r.Modules { - if m.RunsAs == "" { - restartOnGrants(process, grantedToTools[m.Module]) - } - } // And a runtime of its own for each module of its own account, after it (novox/hq ADR 0259 §8). owns, err := r.ownRuntimes(with) if err != nil { @@ -1170,7 +1149,6 @@ func (r Resolution) compose(with Rendering, owner map[string]string, for _, p := range owns { id := fmt.Sprint(p["id"]) owner[id] = strings.TrimSuffix(id, "."+OwnRuntimeID()) - restartOnGrants(p, grantedToTools[owner[id]]) out = append(out, p) } // What each module's bundles are given is read as the account the runtime runs as — not what a diff --git a/internal/catalogue/grants.go b/internal/catalogue/grants.go deleted file mode 100644 index 2574bdb7..00000000 --- a/internal/catalogue/grants.go +++ /dev/null @@ -1,134 +0,0 @@ -package catalogue - -import ( - "fmt" - "path/filepath" - "sort" - "strings" -) - -// A read-write access grants the path's own group to what runs the module's code (novox/hq ADR 0312, issue 537). -// -// **The controller says who; the node-engine reads which group.** Which group owns the operator's path is a fact -// of the machine the controller cannot know, and the path is never changed (§2): so each read-write access -// reaches a machine naming the identities its module's code runs as, and the node-engine reads the path's group -// at apply and grants it to them. Two kinds of identity run a module's code here: -// -// - each of its containers that mounts the path — the source of one of its volumes is the path, or below it — -// which gains the group as a supplementary group of its process and is recreated with it; -// - the account its tools run as, where it has a tools bundle the machine's runtime loads: the operator's -// account the runtime runs as, or the module's own account for a module served by a runtime of its own -// (ADR 0259 §8). That runtime names the access under its `restart-on`, so it is restarted when the access -// moved the account's groups, and its new process holds the group with no new login (§6). -// -// A read access grants nothing (§3): it is sent as before. A read-write access whose module runs nothing here is -// sent an empty grant, so the node-engine still states it unhealthy while its group cannot write (§2). -// -// **Only to an engine that reads it.** An older node-engine parses strictly and refuses a whole declaration for a -// field it does not know; the field and the restarts it brings are composed only where the engine's statement -// says link.GrantContract or later (Rendering.GrantsGroups). - -// GrantField is an access's field naming who its path's group is granted to (novox/hq ADR 0312). -const GrantField = "grant" - -// toolsAccount is the account module m's tools run as on this machine, or empty where they run nowhere as one: -// no runtime here, no bundle it loads, a module left out of an adopted machine's declaration, or a runtime -// running as root on a machine with no account. -func (r Resolution) toolsAccount(m Manifest, with Rendering) string { - if !r.runtimeHere() || (with.Adopted && m.Filtering != nil) { - return "" - } - loads := false - for _, b := range m.Bundles { - if len(b.Loads) > 0 { - loads = true - } - } - if !loads { - return "" - } - if m.RunsAs != "" { - return m.RunsAs - } - return r.Account -} - -// grantAccesses writes the grant of every read-write access of module, among the resources composed so far: the -// module's containers that mount the access's path, and account when it is not empty. The ids of the accesses -// that grant to account, for the runtime that runs as it to be restarted on. -func grantAccesses(out []map[string]any, owner map[string]string, module, account string) []string { - var containers []map[string]any - for _, res := range out { - if res["type"] == "container" && owner[fmt.Sprint(res["id"])] == module { - containers = append(containers, res) - } - } - var toAccount []string - for _, res := range out { - if res["type"] != "access" || owner[fmt.Sprint(res["id"])] != module || res["mode"] != AccessReadWrite { - continue - } - path, _ := res["path"].(string) - grant := map[string]any{} - var mounting []any - for _, c := range containers { - if mounts(c, path) { - mounting = append(mounting, fmt.Sprint(c["id"])) - } - } - if len(mounting) > 0 { - grant["containers"] = mounting - } - if account != "" { - grant["account"] = account - toAccount = append(toAccount, fmt.Sprint(res["id"])) - } - res[GrantField] = grant - } - sort.Strings(toAccount) - return toAccount -} - -// mounts says whether a container mounts path, or something below it, from the machine. A path that merely -// begins with the same letters — /storage/media2 beside /storage/media — is another path. A read-only mount of a -// read-write access still counts: the module declared the access read-write, the engine grants the group to what -// mounts it, and a read-only mount only means that container chose not to write through it. -func mounts(container map[string]any, path string) bool { - volumes, _ := container["volumes"].([]any) - path = filepath.Clean(path) - for _, v := range volumes { - s, ok := v.(string) - if !ok { - continue - } - source, _, _ := strings.Cut(s, ":") - if !strings.HasPrefix(source, "/") { - continue // a named volume: the runtime's, not the operator's path - } - source = filepath.Clean(source) - if source == path || strings.HasPrefix(source, path+"/") { - return true - } - } - return false -} - -// restartOnGrants adds to a runtime process's restart-on the accesses that grant to the account it runs as, so it -// is restarted to hold a group the account was put in or taken out of (novox/hq ADR 0312 §6). -func restartOnGrants(process map[string]any, ids []string) { - if process == nil || len(ids) == 0 { - return - } - restarts, _ := process["restart-on"].([]any) - seen := map[string]bool{} - for _, id := range restarts { - seen[fmt.Sprint(id)] = true - } - for _, id := range ids { - if !seen[id] { - restarts = append(restarts, id) - seen[id] = true - } - } - process["restart-on"] = restarts -} diff --git a/internal/catalogue/grants_test.go b/internal/catalogue/grants_test.go deleted file mode 100644 index 382b8e68..00000000 --- a/internal/catalogue/grants_test.go +++ /dev/null @@ -1,174 +0,0 @@ -package catalogue - -import ( - "encoding/json" - "fmt" - "testing" - - "github.com/novox/mesh-host/validate" -) - -// Defends novox/hq ADR 0312 and issue 537 on the controller's side: each read-write access reaches the machine -// naming who its path's group is granted to — the module's containers that mount it, and the account its tools -// run as — and the tool runner is restarted on it; a read access grants nothing; and none of it is sent to a -// node-engine older than the field, which would refuse the whole declaration. - -const anImage = "x@sha256:1111111111111111111111111111111111111111111111111111111111111111" - -// anImporter files into one library it writes and reads another: a container mounting both, a step mounting -// neither, and tools served by the machine's runtime. -func anImporter(t *testing.T) Manifest { - t.Helper() - m := aToolsModule(t, "importer", "tools/index.js") - m.Accesses = []Access{ - {ID: "music", Path: "/storage/media/music", Mode: AccessReadWrite}, - {ID: "movies", Path: "/storage/media/movies", Mode: AccessRead}, - } - m.Resources = []map[string]any{ - {"id": "app", "type": "container", "name": "importer", "image": anImage, - "volumes": []any{"${access:music}/incoming:/music", "${access:movies}:/movies:ro"}}, - {"id": "sidecar", "type": "container", "name": "importer-sidecar", "image": anImage}, - } - return m -} - -func composeImporter(t *testing.T, r Resolution, with Rendering) map[string]map[string]any { - t.Helper() - with.Needed = map[string]map[string]string{RuntimeModule: {"broker": "sealed-credential"}} - with.ArtifactStore = "anchor.internal:5101" - out, err := r.Declaration(with) - if err != nil { - t.Fatal(err) - } - // What the node-engine this controller pins would make of it: nothing refused. - body, err := json.Marshal(map[string]any{"declaration": validate.Version, "resources": out}) - if err != nil { - t.Fatal(err) - } - if problems := validate.Declaration(body); len(problems) > 0 { - t.Fatalf("composed into something the node-engine refuses: %v", problems) - } - return byID(out) -} - -func TestAReadWriteAccessNamesTheContainersThatMountItAndTheToolsAccount(t *testing.T) { - r := Resolution{Node: "ace", Account: "jochen", Modules: []Manifest{anImporter(t), theRuntime(t)}} - got := composeImporter(t, r, Rendering{GrantsGroups: true}) - - music := got["importer."+AccessID("/storage/media/music")] - if music == nil { - t.Fatalf("no access resource for the library: %v", keysOf(got)) - } - grant, _ := music[GrantField].(map[string]any) - if fmt.Sprint(grant["containers"]) != "[importer.app]" || grant["account"] != "jochen" { - t.Errorf("the read-write access grants %v; want the container that mounts it and the tools' account", grant) - } - // A read access grants nothing (§3). - if movies := got["importer."+AccessID("/storage/media/movies")]; movies[GrantField] != nil { - t.Errorf("a read access names a grant: %v", movies[GrantField]) - } - // The tool runner is restarted when the access moved its account's groups (§6). - runtime := got[RuntimeModule+"."+RuntimeProcessID()] - found := false - for _, id := range runtime["restart-on"].([]any) { - if id == "importer."+AccessID("/storage/media/music") { - found = true - } - } - if !found { - t.Errorf("the tool runner is not restarted on the access that grants its account: %v", runtime["restart-on"]) - } -} - -func TestAModuleOfItsOwnAccountIsGrantedAsThatAccount(t *testing.T) { - m := anImporter(t) - m.RunsAs = "importer" - m.OwnSecrets = OwnSecrets{"broker": {Path: "/var/lib/mesh/importer/broker"}} - runtime := theRuntime(t) - runtime.Bundles[0].Binary = "node-tools" - r := Resolution{Node: "ace", Account: "jochen", Modules: []Manifest{m, runtime}} - with := Rendering{GrantsGroups: true} - with.Needed = map[string]map[string]string{RuntimeModule: {"broker": "sealed"}, "importer": {"broker": "sealed"}} - with.ArtifactStore = "anchor.internal:5101" - out, err := r.Declaration(with) - if err != nil { - t.Fatal(err) - } - got := byID(out) - grant, _ := got["importer."+AccessID("/storage/media/music")][GrantField].(map[string]any) - if grant["account"] != "importer" { - t.Errorf("the access grants %v; want the module's own account", grant) - } - own := got["importer."+OwnRuntimeID()] - if fmt.Sprint(own["restart-on"]) == "" || !restartsOnIt(own["restart-on"], "importer."+AccessID("/storage/media/music")) { - t.Errorf("the module's own runtime is not restarted on its access: %v", own["restart-on"]) - } - if restartsOnIt(got[RuntimeModule+"."+RuntimeProcessID()]["restart-on"], "importer."+AccessID("/storage/media/music")) { - t.Error("the machine's runtime is restarted on an access of a module it does not serve") - } -} - -func TestAModuleThatRunsNothingHereSendsAnEmptyGrant(t *testing.T) { - m := mod("spool", nil, nil, nil) - m.Accesses = []Access{{Path: "/storage/downloads", Mode: AccessReadWrite}} - out, err := Resolution{Node: "ace", Account: "jochen", Modules: []Manifest{m}}.Declaration(Rendering{GrantsGroups: true}) - if err != nil { - t.Fatal(err) - } - grant, has := byID(out)["spool."+AccessID("/storage/downloads")][GrantField].(map[string]any) - if !has || len(grant) != 0 { - t.Errorf("a read-write access of a module running nothing here is sent %v (sent %v); want an empty grant, "+ - "so its group is still judged", grant, has) - } -} - -func TestNoGrantIsSentToAnEngineOlderThanTheField(t *testing.T) { - r := Resolution{Node: "ace", Account: "jochen", Modules: []Manifest{anImporter(t), theRuntime(t)}} - got := composeImporter(t, r, Rendering{}) - for id, res := range got { - if _, has := res[GrantField]; has { - t.Errorf("%s names a grant to an engine that does not read it", id) - } - } - if restartsOnIt(got[RuntimeModule+"."+RuntimeProcessID()]["restart-on"], "importer."+AccessID("/storage/media/music")) { - t.Error("the tool runner is restarted on an access an older engine grants nothing for") - } -} - -func restartsOnIt(list any, want string) bool { - items, _ := list.([]any) - for _, i := range items { - if fmt.Sprint(i) == want { - return true - } - } - return false -} - -func keysOf(m map[string]map[string]any) []string { - var out []string - for k := range m { - out = append(out, k) - } - return out -} - -// Only a container that mounts the access's path, or below it, is named: a path that begins with the same letters is -// another path. A read-only mount of a read-write access is still named — the access is read-write, and the grant -// follows what mounts it, not how. -func TestOnlyAContainerMountingThePathIsNamedReadOnlyIncluded(t *testing.T) { - m := mod("library", nil, nil, nil) - m.Accesses = []Access{{ID: "media", Path: "/storage/media", Mode: AccessReadWrite}} - m.Resources = []map[string]any{ - {"id": "beside", "type": "container", "name": "beside", "image": anImage, "volumes": []any{"/storage/media2:/x"}}, - {"id": "reader", "type": "container", "name": "reader", "image": anImage, "volumes": []any{"${access:media}:/media:ro"}}, - } - out, err := Resolution{Node: "ace", Modules: []Manifest{m}}.Declaration(Rendering{GrantsGroups: true}) - if err != nil { - t.Fatal(err) - } - grant, _ := byID(out)["library."+AccessID("/storage/media")][GrantField].(map[string]any) - if fmt.Sprint(grant["containers"]) != "[library.reader]" { - t.Errorf("the access grants %v; want the read-only mount and not /storage/media2", grant) - } -} diff --git a/internal/link/protocol.go b/internal/link/protocol.go index 5b431422..63349ece 100644 --- a/internal/link/protocol.go +++ b/internal/link/protocol.go @@ -436,12 +436,6 @@ const ReadinessContract = 2 // parses strictly and would refuse the whole declaration for it. const RootContract = 3 -// GrantContract is the statement of an engine that also reads an access's `grant` (novox/hq ADR 0312): it grants a -// read-write access's path group to the containers and the account the grant names, takes it back, and states the -// access unhealthy while the group cannot write. Only to such an engine is the field sent: an older one parses -// strictly and would refuse the whole declaration for it. -const GrantContract = 4 - // ReasonRoot starts the reason of an account verdict that found a way to root (ADR 0266); the node-engine's // own words (mesh-host internal/accounts ReasonRoot). const ReasonRoot = "can become root without a person" @@ -493,11 +487,6 @@ const KindAccount = "account" // ReasonUsedAsFound. const KindDirectory = "directory" -// KindAccess is a read-write access that grants its path's group (novox/hq ADR 0312): healthy while the group may -// write the path, unhealthy while it cannot — the operator's to decide, since the mesh changes nothing on the path -// — or while the path is owned by a group that is never granted. The node-engine's own (mesh-host internal/link). -const KindAccess = "access" - // ReasonUsedAsFound starts the reason of a directory used as found. const ReasonUsedAsFound = "used as found:" diff --git a/testdata/beside/CAPTURED b/testdata/beside/CAPTURED index c110ebf7..a37baf3b 100644 --- a/testdata/beside/CAPTURED +++ b/testdata/beside/CAPTURED @@ -8,7 +8,7 @@ private addresses. That is acceptable while mesh-controller is private; if it ev a second place to clean besides the catalogue itself. mesh-catalog b9de001833b1b61b297182b8e3bcdae1cbdeace9 modules/*/module.json, modules/nats/Dockerfile -mesh-host 3f7f797168d33ec22241bdd6ef2c64de86eea914 examples/foundation-first-node-nats.lock +mesh-host a76baa4b47f07a8b2bab065603414530bd02b53a examples/foundation-first-node-nats.lock mesh-sdk 33917f91ac3cc213b879b32f3c70e2aa70395f98 conformance/events/module-event.json To move them, from this repository's root, with the two repositories checked out beside it: diff --git a/vendor/github.com/novox/mesh-host/internal/declaration/declaration.go b/vendor/github.com/novox/mesh-host/internal/declaration/declaration.go index e2f6d294..50db892e 100644 --- a/vendor/github.com/novox/mesh-host/internal/declaration/declaration.go +++ b/vendor/github.com/novox/mesh-host/internal/declaration/declaration.go @@ -464,35 +464,6 @@ type Access struct { Path string `json:"path"` // Mode is how this module reaches the path: read or read-write. Absent narrows to read. Mode string `json:"mode,omitempty"` - - // Grant is who the path's own group is granted to, for a read-write access (novox/hq ADR 0312 §1): the - // identities the module's code runs as. The controller says who; the node-engine reads which group owns - // the path when it applies, because only the machine knows, and never changes the path (§2). Absent is a - // controller older than the grant, or an access that grants nothing — and a read access always is one (§3). - Grant *AccessGrant `json:"grant,omitempty"` -} - -// AccessGrant is the identities a read-write access grants its path's group to (novox/hq ADR 0312 §1). -type AccessGrant struct { - // Containers are the module's containers that mount the path, by id: each is created with the group as a - // supplementary group of its process, and recreated when the group moves (§6). - Containers []string `json:"containers,omitempty"` - // Account is the account that runs the module's tools: put in the group as an account group, as ADR 0252 - // §1 puts one, and taken out again as §3 takes back (§5). Empty for a module whose code runs nowhere as - // an account. - Account string `json:"account,omitempty"` -} - -// UnmarshalJSON reads a grant as strictly as a resource is read: a field the engine does not know is a -// thing the controller believes it asked for, and is refused rather than ignored. -func (g *AccessGrant) UnmarshalJSON(raw []byte) error { - type plain AccessGrant - var p plain - if err := strictDecode(raw, &p); err != nil { - return fmt.Errorf("a grant: %w", err) - } - *g = AccessGrant(p) - return nil } func (a *Access) Identity() string { return a.ID } @@ -511,24 +482,6 @@ func (a *Access) validate(where string, _ bool) []string { problems = append(problems, fmt.Sprintf( "%s: an access is %q or %q, not %q", where, AccessRead, AccessReadWrite, a.Mode)) } - if a.Grant != nil && a.Mode != AccessReadWrite { - // A read access reads as any account may, and gains no group (novox/hq ADR 0312 §3). - problems = append(problems, where+": a read access grants nothing, and this one names a grant; "+ - "only a read-write access grants its path's group (novox/hq ADR 0312 §3)") - } - if a.Grant != nil { - for _, id := range a.Grant.Containers { - if id == "" { - problems = append(problems, where+": a grant names a container with no id") - } - } - if a.Grant.Account != "" && !accountName.MatchString(a.Grant.Account) { - // Handed to id, usermod and gpasswd as an argument: a name starting with a dash would be read as an - // option by each of them. - problems = append(problems, fmt.Sprintf("%s: a grant's account is an account's name — letters, digits, "+ - "dots, underscores and dashes, not starting with a dash — and %q is not one", where, a.Grant.Account)) - } - } return problems } @@ -1715,7 +1668,6 @@ func parse(raw []byte, allowActions bool) (*Declaration, error) { d.Resources = append(d.Resources, resource) } problems = append(problems, checkWhileStopped(d.Resources)...) - problems = append(problems, checkGrants(d.Resources)...) problems = append(problems, checkAdoption(env.Adoption, d.Resources, allowActions)...) problems = append(problems, checkReplaces(d.Resources)...) if env.Adoption == nil { @@ -1780,36 +1732,6 @@ func checkWhileStopped(resources []Resource) []string { return problems } -// accountName is what a grant's account may be: the portable account name the user tools take, optionally ending -// in `$`, never starting with a dash. -var accountName = regexp.MustCompile(`^[A-Za-z0-9_.][A-Za-z0-9_.-]*\$?$`) - -// checkGrants judges each access's grant against the whole declaration (novox/hq ADR 0312): a container it -// names is a container of this declaration, as a maintenance window's are. A grant to a container the machine -// is not told about would be a group given to nothing, said as granted. -func checkGrants(resources []Resource) []string { - containers := map[string]bool{} - for _, r := range resources { - if r.Kind() == TypeContainer { - containers[r.Identity()] = true - } - } - var problems []string - for _, r := range resources { - a, ok := r.(*Access) - if !ok || a.Grant == nil { - continue - } - for _, id := range a.Grant.Containers { - if id != "" && !containers[id] { - problems = append(problems, fmt.Sprintf("resource %q: its grant names %q, and this declaration "+ - "has no container by that id", a.ID, id)) - } - } - } - return problems -} - func strictDecode(raw []byte, into any) error { // DisallowUnknownFields is the whole point rather than strictness for its own sake: a // field the host does not know is a thing the control plane believes it asked for. diff --git a/vendor/modules.txt b/vendor/modules.txt index 470b8478..bb6a448e 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -78,7 +78,7 @@ github.com/nats-io/nkeys # github.com/nats-io/nuid v1.0.1 ## explicit github.com/nats-io/nuid -# github.com/novox/mesh-host v0.0.0 => git.novox.be/novox/mesh-host v0.0.0-20261012112256-3f7f797168d3 +# github.com/novox/mesh-host v0.0.0 => git.novox.be/novox/mesh-host v0.0.0-20261011092230-20d5af9b2d5f ## explicit; go 1.26.0 github.com/novox/mesh-host/internal/declaration github.com/novox/mesh-host/rootsearch @@ -135,4 +135,4 @@ golang.org/x/text/width # golang.org/x/time v0.15.0 ## explicit; go 1.25.0 golang.org/x/time/rate -# github.com/novox/mesh-host => git.novox.be/novox/mesh-host v0.0.0-20261012112256-3f7f797168d3 +# github.com/novox/mesh-host => git.novox.be/novox/mesh-host v0.0.0-20261011092230-20d5af9b2d5f From 83a9cf6e3ab2cb16ff583ba043c3cdfecd384a28 Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 12 Oct 2026 17:27:01 +0200 Subject: [PATCH 18/26] Judge a pull request's head merged into its trunk, and name the trunk commit A check judged the head alone, so two pull requests that each passed could break the trunk together (node-engine #66 on 2026-10-09), and a recheck after the trunk moved judged the same head again. The build seat now merges the head into the trunk's head in its own clone, never pushed, fails a head that does not merge cleanly naming the files, and the verdict names the head, the trunk commit and the commits read beside it, for the forge and mesh-delivery to weigh against the trunk's next move (novox/hq ADR 0323, issue 341). --- cmd/mesh-builder/main.go | 10 +- cmd/mesh-controller/checks.go | 35 ++++- cmd/mesh-controller/checks_test.go | 50 +++++++ cmd/mesh-controller/delivery.go | 2 +- internal/builder/check.go | 83 +++++++++++- internal/builder/check_merge.go | 114 ++++++++++++++++ internal/builder/check_merge_test.go | 188 +++++++++++++++++++++++++++ internal/builder/replay341_test.go | 139 ++++++++++++++++++++ internal/link/build.go | 11 ++ internal/link/events.go | 15 ++- 10 files changed, 633 insertions(+), 14 deletions(-) create mode 100644 internal/builder/check_merge.go create mode 100644 internal/builder/check_merge_test.go create mode 100644 internal/builder/replay341_test.go diff --git a/cmd/mesh-builder/main.go b/cmd/mesh-builder/main.go index c49fc8cb..3faccf9a 100644 --- a/cmd/mesh-builder/main.go +++ b/cmd/mesh-builder/main.go @@ -256,7 +256,13 @@ func answer(ctx context.Context, publisher builder.Publisher, on, workspace stri v, err = builder.Check(building, builder.Command, checkSpecOf(request), workspace, registry, forgeFrom(), say) if err == nil { result.Check = &link.CheckOutcome{Verdict: v.Verdict, Summary: v.Summary, Report: v.Report, - Took: v.Took.Round(time.Second).String(), Gate: layerOf(v.Gate), RepoCheck: layerOf(v.Repo)} + Took: v.Took.Round(time.Second).String(), Gate: layerOf(v.Gate), RepoCheck: layerOf(v.Repo), + Head: v.Head, Trunk: v.Trunk, Beside: v.Beside} + // What each of a group's heads was judged as (novox/hq ADR 0323 §6). + for _, m := range v.Members { + result.Check.Members = append(result.Check.Members, link.CheckedMember{Owner: m.Owner, Repo: m.Repo, + Commit: m.Head, Trunk: m.Trunk}) + } } } else if err == nil { // The package-registry credential is a build input, so it is resolved before the clone: a @@ -354,7 +360,7 @@ func checkSpecOf(request link.BuildRequest) builder.CheckSpec { } for _, m := range c.Members { spec.Group = append(spec.Group, builder.GroupHead{Owner: m.Owner, Repo: m.Repo, Repository: m.Repository, - Ref: m.Ref, Paths: m.Paths}) + Ref: m.Ref, Paths: m.Paths, Base: m.Base}) } return spec } diff --git a/cmd/mesh-controller/checks.go b/cmd/mesh-controller/checks.go index bb78c718..9faa4915 100644 --- a/cmd/mesh-controller/checks.go +++ b/cmd/mesh-controller/checks.go @@ -363,6 +363,9 @@ func checkedOf(result link.BuildResult) link.Checked { case result.Check != nil: c.Verdict, c.Summary, c.Report = result.Check.Verdict, result.Check.Summary, result.Check.Report c.Gate, c.RepoCheck = result.Check.Gate, result.Check.RepoCheck + // What was judged (novox/hq ADR 0323 §3): the head merged into the trunk commit, and the commits of the + // repositories read beside it. + c.Trunk, c.Beside, c.Took = result.Check.Trunk, result.Check.Beside, result.Check.Took if c.Gate == nil { // A build seat from before the layers: its verdict is the gate's. c.Gate = &link.CheckLayer{Verdict: c.Verdict, Summary: c.Summary} @@ -388,14 +391,20 @@ func checkedOf(result link.BuildResult) link.Checked { } if result.Checked != nil { c.Plan = result.Checked.Plan + if c.Plan != nil && c.Trunk != "" { + // The kept delivery plan records the trunk commit with its other inputs (ADR 0323 §3). + plan := *c.Plan + plan.Trunk = c.Trunk + c.Plan = &plan + } // A delivery group's composed check (novox/hq ADR 0239): every head it judged, this one first. if g := result.Checked; g.Group != "" { c.Group = g.Group c.Members = append(c.Members, link.CheckedMember{Owner: g.Owner, Repo: g.Repo, Number: g.Number, - Commit: result.Ref}) + Commit: result.Ref, Trunk: c.Trunk}) for _, m := range g.Members { c.Members = append(c.Members, link.CheckedMember{Owner: m.Owner, Repo: m.Repo, Number: m.Number, - Commit: m.Ref}) + Commit: m.Ref, Trunk: memberTrunk(result.Check, m)}) } } } @@ -410,6 +419,20 @@ func checkedOf(result link.BuildResult) link.Checked { return c } +// memberTrunk is the trunk commit the build seat merged one of a group's heads into (novox/hq ADR 0323 §6), empty +// when it said none. +func memberTrunk(outcome *link.CheckOutcome, m link.GroupMember) string { + if outcome == nil { + return "" + } + for _, j := range outcome.Members { + if strings.EqualFold(j.Owner, m.Owner) && strings.EqualFold(j.Repo, m.Repo) && strings.HasPrefix(m.Ref, j.Commit) { + return j.Trunk + } + } + return "" +} + func prefixedAll(prefix string, items []string) []string { out := make([]string, 0, len(items)) for _, i := range items { @@ -424,8 +447,12 @@ func sayChecked(ctx context.Context, c link.Checked) { if c.RepoCheck != nil { repo = strings.ToUpper(c.RepoCheck.Verdict) + " — " + c.RepoCheck.Summary } - fmt.Printf("%s: %s/%s#%d at %.8s checked on %s: gate %s — %s; repository %s\n", c.ID, c.Owner, c.Repo, c.Number, - c.Commit, orSomewhere(c.On), strings.ToUpper(c.Verdict), c.Summary, repo) + judged := "" + if c.Trunk != "" { + judged = " merged into trunk " + c.Trunk + } + fmt.Printf("%s: %s/%s#%d at %.8s%s checked on %s: gate %s — %s; repository %s\n", c.ID, c.Owner, c.Repo, c.Number, + c.Commit, judged, orSomewhere(c.On), strings.ToUpper(c.Verdict), c.Summary, repo) if checkEvents == nil { return } diff --git a/cmd/mesh-controller/checks_test.go b/cmd/mesh-controller/checks_test.go index 7e0bc946..65a8314f 100644 --- a/cmd/mesh-controller/checks_test.go +++ b/cmd/mesh-controller/checks_test.go @@ -210,3 +210,53 @@ func TestABuildOffTheTrunkIsNeverPublished(t *testing.T) { t.Errorf("a build seat that said nothing was refused: %v", err) } } + +// **A verdict names the trunk commit it judged** (novox/hq ADR 0323 §3, issue 341): the controller's `checked` +// carries the head and the trunk commit the build seat merged it into, the commits of the repositories read beside +// it, how long it took, and the kept delivery plan records the trunk commit with its other inputs. A group's +// composed check names, per member, the trunk commit that head was merged into (§6). +func TestAVerdictNamesTheTrunkCommitItJudged(t *testing.T) { + plan := &link.ChangePlan{Repository: "novox/mesh-host", Base: "main", Head: "dfb1415aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + Summary: "builds mesh-host"} + asked := &link.CheckRequest{Owner: "novox", Repo: "mesh-host", Number: 66, Base: "main", Plan: plan} + c := checkedOf(link.BuildResult{ID: "b", Ref: "dfb1415aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", Checked: asked, + Check: &link.CheckOutcome{Verdict: "pass", Took: "4m12s", Head: "dfb1415aaaaa", Trunk: "d28ca95bbbbb", + Gate: &link.CheckLayer{Verdict: "pass"}, RepoCheck: &link.CheckLayer{Verdict: "pass"}, + Beside: map[string]string{"mesh-catalog": "0123456789ab", "mesh-lab": "ba9876543210"}}}) + if c.Trunk != "d28ca95bbbbb" || c.Commit != "dfb1415aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" { + t.Fatalf("the verdict names head %s and trunk %q", c.Commit, c.Trunk) + } + if c.Beside["mesh-catalog"] != "0123456789ab" || c.Beside["mesh-lab"] != "ba9876543210" { + t.Fatalf("the verdict names the repositories read beside it as %v", c.Beside) + } + if c.Took != "4m12s" { + t.Fatalf("the verdict says it took %q", c.Took) + } + if c.Plan == nil || c.Plan.Trunk != "d28ca95bbbbb" || c.Plan.Head != plan.Head { + t.Fatalf("the kept plan records %+v", c.Plan) + } + if plan.Trunk != "" { + t.Fatal("the plan the controller asked with was changed: the verdict's is a copy") + } + + group := &link.CheckRequest{Owner: "novox", Repo: "mesh-controller", Number: 7, Base: "main", Group: "feat/x", + Members: []link.GroupMember{{Owner: "novox", Repo: "mesh-host", Number: 8, Ref: "cccccccccccc0000", Base: "main"}}} + c = checkedOf(link.BuildResult{ID: "g", Ref: "aaaaaaaaaaaa0000", Checked: group, Check: &link.CheckOutcome{ + Verdict: "pass", Head: "aaaaaaaaaaaa", Trunk: "111111111111", Gate: &link.CheckLayer{Verdict: "pass"}, + Members: []link.CheckedMember{{Owner: "novox", Repo: "mesh-host", Commit: "cccccccccccc", Trunk: "222222222222"}}}}) + if len(c.Members) != 2 || c.Members[0].Trunk != "111111111111" || c.Members[1].Trunk != "222222222222" || + c.Members[1].Commit != "cccccccccccc0000" { + t.Fatalf("the composed check names its members %+v", c.Members) + } +} + +// **The merge made for a check is never registered** (novox/hq ADR 0323 §2): a build of it — a commit on no branch +// of the forge — is refused, naming the trunk. The rule of ADR 0238 §1 stands unchanged; this holds it for the +// merge commit a check makes. +func TestABuildOfACheckMergeIsRefusedNamingTheTrunk(t *testing.T) { + merge := link.BuildResult{ID: "b", Commit: "4b5c6d7e8f90aabbccddeeff0011223344556677", Trunk: "main"} + err := publishable(merge, "") + if !errors.Is(err, errOffTheTrunk) || !strings.Contains(err.Error(), "not on main") { + t.Fatalf("a build of a check's merge was answered %v", err) + } +} diff --git a/cmd/mesh-controller/delivery.go b/cmd/mesh-controller/delivery.go index 7f95d589..235f6a51 100644 --- a/cmd/mesh-controller/delivery.go +++ b/cmd/mesh-controller/delivery.go @@ -1185,7 +1185,7 @@ func askGroupCheck(ctx context.Context, open *stores, group string, members []or return "", fmt.Errorf("%s cannot be cloned for the group's check: %w", m.ID, err) } request.Check.Members = append(request.Check.Members, link.GroupMember{Owner: mp.Owner, Repo: mp.Repo, - Number: mp.Number, Repository: url, Ref: mp.Commit, Paths: mp.Paths}) + Number: mp.Number, Repository: url, Ref: mp.Commit, Paths: mp.Paths, Base: mp.Base}) } seat := buildSeatHeld(ctx) ask, err := askOverOn(seat) diff --git a/internal/builder/check.go b/internal/builder/check.go index 50b3a476..0a08d40c 100644 --- a/internal/builder/check.go +++ b/internal/builder/check.go @@ -49,8 +49,10 @@ import ( // // One check: // -// 1. clones the repository at the pull request's head, and beside it the repositories its check -// reads — the controller the mesh runs, the catalogue, the host, the lab — each at the ref asked; +// 1. clones the repository at the pull request's head and **makes the merge of that head into the trunk's +// head**, the check's trunk commit, in its own clone (novox/hq ADR 0323, issue 341; check_merge.go): what +// is judged is what merging makes, never pushed. Beside it the repositories its check reads — the +// controller the mesh runs, the catalogue, the host, the lab — each at the ref asked; // 2. reads the facts snapshot the controller keeps, and with it **the versions the mesh runs**: the // store a check's tests stand on is the store's own release, and the bus the bus's; // 3. raises a throwaway PostgreSQL and a throwaway bus of those versions, labelled with the ask so a @@ -79,7 +81,8 @@ type CheckSpec struct { Modules []string New []string Manifests []string - // Base is the branch the pull request merges into: what the gate compares the change against. + // Base is the branch the pull request merges into: what the gate compares the change against, and the trunk + // whose head the change is merged into before it is judged (novox/hq ADR 0323). Empty judges the head alone. Base string // Judge is who judges the gate: "" the controller the mesh runs, "self" the change's own, "validator" // the running one with the change's validator. @@ -104,6 +107,9 @@ type GroupHead struct { Repository string Ref string Paths []string + // Base is the branch it merges into: the head is merged into that trunk's head before it is composed + // (novox/hq ADR 0323 §6). Empty composes the head alone, as a controller from before it asks. + Base string } // Gated is whether the change touches the mesh's graph, and so whether the gate runs. @@ -139,6 +145,16 @@ type CheckVerdict struct { Took time.Duration Gate *Layer Repo *Layer + // Head and Trunk are what was judged (novox/hq ADR 0323 §3): the pull request's head merged into the trunk + // commit, each as twelve characters. Trunk is empty for a check asked with no trunk, which judged the head + // alone. Merge is the merge commit made for the check, whole: only checked, never pushed or registered. + Head, Trunk, Merge string + // Beside are the commits of the repositories the check read beside its own, by the directory each was + // cloned under, as twelve characters: named, not judged against (ADR 0323 §3, issue 568). + Beside map[string]string + // Members are a delivery group's other heads as the composed check judged them, each with the trunk + // commit it was merged into (ADR 0323 §6). + Members []JudgedHead } // Layer is one layer of a check, judged. @@ -281,6 +297,34 @@ func Check(ctx context.Context, run Runner, spec CheckSpec, workspace, registry return CheckVerdict{}, err } tree := filepath.Join(root, name) + // **What merging makes is judged, not the head** (novox/hq ADR 0323, issue 341): the head merged into the + // trunk's head as this clone found it, in this clone alone. A head that does not merge cleanly fails here. + judged := CheckVerdict{Head: twelve(spec.Ref)} + paths := spec.Paths + if spec.Base != "" { + m, err := mergeIntoTrunk(ctx, run, tree, spec.Base, spec.Ref) + if err != nil { + return CheckVerdict{}, err + } + judged.Head, judged.Trunk = twelve(m.Head), twelve(m.Trunk) + if len(m.Conflicts) > 0 { + said := conflictWords(m) + say("check", "%s", said) + return CheckVerdict{Verdict: "fail", Summary: said, Report: said, Took: time.Since(began), + Gate: &Layer{Verdict: "fail", Summary: said}, Repo: &Layer{Verdict: "fail", Summary: said}, + Head: judged.Head, Trunk: judged.Trunk}, nil + } + judged.Merge = m.Merge + if changed := changedByMerge(ctx, run, tree, m); len(changed) > 0 { + paths = changed + } + say("check", "head %s merged into trunk %s (%s): the merge is judged, and never pushed", judged.Head, + judged.Trunk, spec.Base) + } + named := func(v CheckVerdict) CheckVerdict { + v.Head, v.Trunk, v.Merge, v.Beside, v.Members = judged.Head, judged.Trunk, judged.Merge, judged.Beside, judged.Members + return v + } // **The base branch's check judges the change** (novox/hq issue 310), never the one the change brings. judging, err := TheCheckThatJudges(ctx, tree, spec.Base) if err != nil { @@ -297,7 +341,7 @@ func Check(ctx context.Context, run Runner, spec CheckSpec, workspace, registry v := CheckVerdict{Verdict: "pass", Summary: noModule, Took: time.Since(began), Gate: &Layer{Verdict: "pass", Summary: noModule}, Repo: &Layer{Verdict: "warning", Summary: noScript}} say("check", "%s; %s", noModule, noScript) - return v, nil + return named(v), nil } for dir, b := range spec.Beside { if dir == name || !safeName.MatchString(dir) { @@ -306,6 +350,13 @@ func Check(ctx context.Context, run Runner, spec CheckSpec, workspace, registry if err := clone(b.Repository, b.Ref, dir); err != nil { return CheckVerdict{}, fmt.Errorf("beside it, %w", err) } + // The commit each one read is named in the verdict (ADR 0323 §3): its move is issue 568's question. + if at := headAt(ctx, run, filepath.Join(root, dir)); at != "" { + if judged.Beside == nil { + judged.Beside = map[string]string{} + } + judged.Beside[dir] = at + } say("check", "beside it %s at %s", dir, short(b.Ref)) } @@ -322,6 +373,26 @@ func Check(ctx context.Context, run Runner, spec CheckSpec, workspace, registry return CheckVerdict{}, fmt.Errorf("a head of the group, %w", err) } say("check", "with it, of its group, %s/%s at %s", g.Owner, g.Repo, short(g.Ref)) + // Each member is composed as its merge into its own trunk's head (ADR 0323 §6). + member := JudgedHead{Owner: g.Owner, Repo: g.Repo, Head: twelve(g.Ref)} + if g.Base != "" { + m, err := mergeIntoTrunk(ctx, run, filepath.Join(root, dir), g.Base, g.Ref) + if err != nil { + return CheckVerdict{}, fmt.Errorf("a head of the group, %s/%s: %w", g.Owner, g.Repo, err) + } + member.Head, member.Trunk = twelve(m.Head), twelve(m.Trunk) + if len(m.Conflicts) > 0 { + said := fmt.Sprintf("%s/%s: %s", g.Owner, g.Repo, conflictWords(m)) + say("check", "%s", said) + judged.Members = append(judged.Members, member) + return named(CheckVerdict{Verdict: "fail", Summary: said, Report: said, Took: time.Since(began), + Gate: &Layer{Verdict: "fail", Summary: said}, Repo: &Layer{Verdict: "fail", Summary: said}}), nil + } + if changed := changedByMerge(ctx, run, filepath.Join(root, dir), m); len(changed) > 0 { + g.Paths = changed + } + } + judged.Members = append(judged.Members, member) heads = append(heads, map[string]any{"repository": g.Owner + "/" + g.Repo, "tree": filepath.Join(root, dir), "changed": g.Paths}) } @@ -433,7 +504,7 @@ func Check(ctx context.Context, run Runner, spec CheckSpec, workspace, registry verdictFile := filepath.Join(root, "verdict.json") env := []string{EnvFacts + "=" + factsFile, EnvGate + "=" + gate, EnvGateStore + "=" + storeURL, EnvTestStore + "=" + storeURL, EnvTestBus + "=nats://" + bus, EnvRepository + "=" + spec.Owner + "/" + spec.Repo, - EnvChanged + "=" + strings.Join(spec.Paths, ","), EnvBeside + "=" + root, + EnvChanged + "=" + strings.Join(paths, ","), EnvBeside + "=" + root, EnvModules + "=" + strings.Join(append(append([]string{}, spec.Modules...), spec.New...), ","), EnvGroup + "=" + groupFile, "GOCACHE=" + filepath.Join(workspace, "go-cache"), "GOMODCACHE=" + filepath.Join(workspace, "go-modules")} @@ -501,7 +572,7 @@ func Check(ctx context.Context, run Runner, spec CheckSpec, workspace, registry v.Report, v.Took = redact.redact(withWhatFailed(out.String(), v.Repo)), time.Since(began) say("check", "gate %s — %s; repository %s — %s (%s)", strings.ToUpper(v.Gate.Verdict), v.Gate.Summary, strings.ToUpper(v.Repo.Verdict), v.Repo.Summary, v.Took.Round(time.Second)) - return v, nil + return named(v), nil } // ownCheck runs the repository's own check, part by part, each in its toolchain's own container: the diff --git a/internal/builder/check_merge.go b/internal/builder/check_merge.go new file mode 100644 index 00000000..fda7f359 --- /dev/null +++ b/internal/builder/check_merge.go @@ -0,0 +1,114 @@ +package builder + +import ( + "context" + "fmt" + "strings" +) + +// **A merge check judges the merge, not the head** (novox/hq ADR 0323, issue 341). +// +// On 2026-10-09 a node-engine pull request was checked green at its head and broke the trunk when it merged: +// a merge an hour earlier had added a field under the same JSON key on other lines, git merged the two edits +// without a conflict, and the result did not compile. The check had judged the head alone. So a check now +// takes the trunk the pull request merges into, its head at the moment the check clones it as the check's +// **trunk commit**, and makes in its own clone the merge of the head into that commit, as the forge's +// default merge would. The gate and the repository's own check judge that merge. +// +// **The merge is only checked** (ADR 0323 §2, ADR 0238 §1): it is a commit off the trunk, made in the +// check's own clone, never pushed to the forge and never registered; the clone is removed when the check +// ends. Nothing here writes to a remote. +// +// The forge keeps no merge reference for a pull request, only its head, so the builder makes it itself. + +// TrunkMerge is the merge a check made: the head and the trunk commit it was merged into, whole, and the merge +// commit; or the files in conflict when the head does not merge cleanly. +type TrunkMerge struct { + Head, Trunk, Merge string + Conflicts []string +} + +// JudgedHead is one head a check judged, with the trunk commit it merged that head into, each as twelve +// characters (ADR 0323 §3, §6). +type JudgedHead struct { + Owner, Repo string + Head, Trunk string +} + +// mergeIdentity is who a check's merge is made as: it is never pushed, so it names nobody's account. +var mergeIdentity = []string{"-c", "user.name=mesh merge check", "-c", "user.email=merge-check@mesh.invalid", + "-c", "commit.gpgsign=false"} + +// mergeIntoTrunk makes, in the clone at tree, the merge of head into the head of origin/ as it stands in +// that fresh clone: the trunk commit. A head that does not merge cleanly answers the files in conflict and leaves +// the tree at the trunk commit; an error is that the merge could not be tried at all. +func mergeIntoTrunk(ctx context.Context, run Runner, tree, base, head string) (TrunkMerge, error) { + var m TrunkMerge + revParse := func(ref string) (string, error) { + out, err := run(ctx, tree, "git", "rev-parse", "--verify", "--quiet", ref+"^{commit}") + return strings.TrimSpace(out), err + } + var err error + if m.Head, err = revParse(head); err != nil || m.Head == "" { + return m, fmt.Errorf("the head %s cannot be read in the check's clone: %v", head, err) + } + if m.Trunk, err = revParse("refs/remotes/origin/" + base); err != nil || m.Trunk == "" { + return m, fmt.Errorf("the trunk %s cannot be read, and a check judges the head merged into it "+ + "(novox/hq ADR 0323): %v", base, err) + } + if _, err := run(ctx, tree, "git", "checkout", "--quiet", "--detach", m.Trunk); err != nil { + return m, fmt.Errorf("the trunk commit %s cannot be checked out: %w", twelve(m.Trunk), err) + } + message := fmt.Sprintf("Check the merge of %s into %s at %s (made for a merge check, never pushed)", + twelve(m.Head), base, twelve(m.Trunk)) + args := append(append([]string{}, mergeIdentity...), "merge", "--no-ff", "--no-edit", "--quiet", "-m", message, m.Head) + if _, err := run(ctx, tree, "git", args...); err != nil { + out, _ := run(ctx, tree, "git", "diff", "--name-only", "--diff-filter=U") + m.Conflicts = strings.Fields(out) + _, _ = run(ctx, tree, "git", "merge", "--abort") + if len(m.Conflicts) == 0 { + return m, fmt.Errorf("the head %s could not be merged into trunk %s: %w", twelve(m.Head), twelve(m.Trunk), err) + } + return m, nil + } + if m.Merge, err = revParse("HEAD"); err != nil { + return m, fmt.Errorf("the merge made for the check cannot be read: %w", err) + } + return m, nil +} + +// changedByMerge is every path the merge changes against the trunk commit: what the gate's delivery plan is +// computed from (ADR 0323 §1). Nil when it cannot be read; the caller keeps the pull request's own list. +func changedByMerge(ctx context.Context, run Runner, tree string, m TrunkMerge) []string { + if m.Merge == "" || m.Trunk == "" { + return nil + } + out, err := run(ctx, tree, "git", "diff", "--name-only", "--no-renames", m.Trunk, m.Merge) + if err != nil { + return nil + } + return strings.Fields(out) +} + +// conflictWords is a head that does not merge cleanly, said as its check's failure. +func conflictWords(m TrunkMerge) string { + return fmt.Sprintf("head %s does not merge cleanly into trunk %s: in conflict %s", twelve(m.Head), twelve(m.Trunk), + strings.Join(m.Conflicts, ", ")) +} + +// headAt is the commit a clone is checked out at, as twelve characters; empty when it cannot be read. +func headAt(ctx context.Context, run Runner, tree string) string { + out, err := run(ctx, tree, "git", "rev-parse", "--verify", "--quiet", "HEAD^{commit}") + if err != nil { + return "" + } + return twelve(strings.TrimSpace(out)) +} + +// twelve is a commit as a verdict names it: its first twelve characters. +func twelve(commit string) string { + if len(commit) > 12 { + return commit[:12] + } + return commit +} diff --git a/internal/builder/check_merge_test.go b/internal/builder/check_merge_test.go new file mode 100644 index 00000000..4c6955a1 --- /dev/null +++ b/internal/builder/check_merge_test.go @@ -0,0 +1,188 @@ +package builder + +import ( + "os" + "os/exec" + "path/filepath" + "strings" + "testing" +) + +// **A merge check judges the merge, not the head** (novox/hq ADR 0323, issue 341): the head merged into the trunk +// commit in the check's own clone, never pushed; a head that does not merge cleanly fails, naming the files; the +// verdict names the head, the trunk commit and the commits of the repositories read beside it. + +// aTrunkAndABranch is a repository whose main moved after the branch was cut: main holds trunkFiles over the +// common files, the branch headFiles. It answers the repository, the branch's head and main's head. +func aTrunkAndABranch(t *testing.T, common, trunkFiles, headFiles map[string]string) (string, string, string) { + t.Helper() + dir := t.TempDir() + git := func(args ...string) string { + t.Helper() + cmd := exec.Command("git", args...) + cmd.Dir = dir + cmd.Env = append(os.Environ(), "GIT_AUTHOR_NAME=t", "GIT_AUTHOR_EMAIL=t@example.org", + "GIT_COMMITTER_NAME=t", "GIT_COMMITTER_EMAIL=t@example.org") + out, err := cmd.CombinedOutput() + if err != nil { + t.Fatalf("git %v: %v\n%s", args, err, out) + } + return strings.TrimSpace(string(out)) + } + write := func(files map[string]string) { + for name, body := range files { + if err := os.MkdirAll(filepath.Dir(filepath.Join(dir, name)), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, name), []byte(body), 0o755); err != nil { + t.Fatal(err) + } + } + git("add", "-A") + } + git("init", "--quiet", "-b", "main") + write(common) + git("commit", "--quiet", "-m", "common") + git("checkout", "--quiet", "-b", "change") + write(headFiles) + git("commit", "--quiet", "-m", "the change") + head := git("rev-parse", "HEAD") + git("checkout", "--quiet", "main") + write(trunkFiles) + git("commit", "--quiet", "-m", "the trunk moved") + return dir, head, git("rev-parse", "HEAD") +} + +func TestACheckJudgesTheHeadMergedIntoTheTrunk(t *testing.T) { + // main's script passes only in a tree holding both the trunk's file and the head's: the merge. + repo, head, trunk := aTrunkAndABranch(t, + map[string]string{CheckScript: "test -f trunk-only && test -f head-only || { echo 'FAIL: not the merge'; exit 1; }\n"}, + map[string]string{"trunk-only": "t"}, map[string]string{"head-only": "h"}) + beside, besideHead := aCheckedRepository(t, map[string]string{"README": "the catalogue"}) + registry := aFactsRegistry(t) + aRuntimeOnThisMachine(t) + + v, err := Check(t.Context(), Command, CheckSpec{ID: "check-341", Repository: repo, Ref: head, Owner: "novox", + Repo: "mesh-lab", Number: 1, Base: "main", Toolchain: "golang", + Beside: map[string]Beside{"mesh-catalog": {Repository: beside, Ref: besideHead}}}, t.TempDir(), registry, + GitCredential{}, nil) + if err != nil { + t.Fatal(err) + } + if v.Repo == nil || v.Repo.Verdict != "pass" { + t.Fatalf("the check did not judge the head merged into the trunk: %+v\n%s", v.Repo, v.Report) + } + if v.Head != head[:12] || v.Trunk != trunk[:12] { + t.Fatalf("the verdict names head %q and trunk %q, not %s and %s", v.Head, v.Trunk, head[:12], trunk[:12]) + } + if v.Beside["mesh-catalog"] != besideHead[:12] { + t.Fatalf("the verdict names the catalogue read beside it as %q, not %s: %v", v.Beside["mesh-catalog"], + besideHead[:12], v.Beside) + } + if v.Merge == "" || v.Merge == head || v.Merge == trunk { + t.Fatalf("no merge of the head into the trunk was made for the check: %q", v.Merge) + } +} + +func TestAHeadThatDoesNotMergeCleanlyFailsNamingTheFiles(t *testing.T) { + repo, head, trunk := aTrunkAndABranch(t, + map[string]string{"store.go": "package store\n\nconst Declared = 1\n", "other.go": "package store\n"}, + map[string]string{"store.go": "package store\n\nconst Declared = 2\n"}, + map[string]string{"store.go": "package store\n\nconst Declared = 3\n"}) + + // No artifact store and no container runtime: a head that does not merge is judged before either is needed. + v, err := Check(t.Context(), Command, CheckSpec{ID: "check-341-conflict", Repository: repo, Ref: head, + Owner: "novox", Repo: "mesh-lab", Number: 2, Base: "main", Toolchain: "golang"}, t.TempDir(), "", + GitCredential{}, nil) + if err != nil { + t.Fatalf("a head that does not merge cleanly was not judged: %v", err) + } + want := "does not merge cleanly into trunk " + trunk[:12] + for name, l := range map[string]*Layer{"gate": v.Gate, "repository": v.Repo} { + if l == nil || l.Verdict != "fail" || !strings.Contains(l.Summary, want) || !strings.Contains(l.Summary, "store.go") { + t.Fatalf("its %s layer said %+v, not a failure saying %q and naming store.go", name, l, want) + } + } + if strings.Contains(v.Summary, "other.go") { + t.Fatalf("a file not in conflict is named: %s", v.Summary) + } + if v.Head != head[:12] || v.Trunk != trunk[:12] { + t.Fatalf("the failure names head %q and trunk %q", v.Head, v.Trunk) + } +} + +// **The merge made for a check is only checked** (ADR 0323 §2): the forge never receives it. +func TestTheMergeMadeForACheckIsNeverPushed(t *testing.T) { + repo, head, trunk := aTrunkAndABranch(t, map[string]string{CheckScript: "true\n"}, + map[string]string{"trunk-only": "t"}, map[string]string{"head-only": "h"}) + registry := aFactsRegistry(t) + aRuntimeOnThisMachine(t) + refsBefore := gitOut(t, repo, "for-each-ref") + + v, err := Check(t.Context(), Command, CheckSpec{ID: "check-341-push", Repository: repo, Ref: head, Owner: "novox", + Repo: "mesh-lab", Number: 3, Base: "main", Toolchain: "golang"}, t.TempDir(), registry, GitCredential{}, nil) + if err != nil { + t.Fatal(err) + } + if v.Merge == "" { + t.Fatal("the check made no merge of the head into the trunk") + } + if err := exec.Command("git", "-C", repo, "cat-file", "-e", v.Merge+"^{commit}").Run(); err == nil { + t.Fatalf("the merge %s made for the check reached the forge's repository", v.Merge) + } + if after := gitOut(t, repo, "for-each-ref"); after != refsBefore { + t.Fatalf("the check moved a ref of the forge's repository:\nbefore %s\nafter %s", refsBefore, after) + } + if got := gitOut(t, repo, "rev-parse", "main"); got != trunk { + t.Fatalf("the trunk moved to %s", got) + } +} + +// A delivery group's composed check merges each member's head into its own trunk's head (ADR 0323 §6), and names +// both for each; a member that does not merge cleanly fails the composed check, naming it. +func TestAGroupsComposedCheckMergesEachMemberIntoItsTrunk(t *testing.T) { + repo, head, trunk := aTrunkAndABranch(t, map[string]string{CheckScript: "true\n"}, + map[string]string{"trunk-only": "t"}, map[string]string{"head-only": "h"}) + member, memberHead, memberTrunk := aTrunkAndABranch(t, map[string]string{"a": "1"}, + map[string]string{"b": "trunk"}, map[string]string{"c": "head"}) + registry := aFactsRegistry(t) + aRuntimeOnThisMachine(t) + + v, err := Check(t.Context(), Command, CheckSpec{ID: "check-341-group", Repository: repo, Ref: head, Owner: "novox", + Repo: "mesh-controller", Number: 4, Base: "main", Toolchain: "golang", Group: []GroupHead{{Owner: "novox", + Repo: "mesh-host", Repository: member, Ref: memberHead, Base: "main"}}}, t.TempDir(), registry, + GitCredential{}, nil) + if err != nil { + t.Fatal(err) + } + if v.Head != head[:12] || v.Trunk != trunk[:12] { + t.Fatalf("the composed check names its first head %q into %q", v.Head, v.Trunk) + } + if len(v.Members) != 1 || v.Members[0].Head != memberHead[:12] || v.Members[0].Trunk != memberTrunk[:12] { + t.Fatalf("the composed check names its members %+v, not mesh-host %s into %s", v.Members, memberHead[:12], + memberTrunk[:12]) + } + + conflicting, conflictHead, _ := aTrunkAndABranch(t, map[string]string{"a": "1"}, map[string]string{"a": "2"}, + map[string]string{"a": "3"}) + v, err = Check(t.Context(), Command, CheckSpec{ID: "check-341-group-conflict", Repository: repo, Ref: head, + Owner: "novox", Repo: "mesh-controller", Number: 4, Base: "main", Toolchain: "golang", Group: []GroupHead{{ + Owner: "novox", Repo: "mesh-host", Repository: conflicting, Ref: conflictHead, Base: "main"}}}, t.TempDir(), + registry, GitCredential{}, nil) + if err != nil { + t.Fatal(err) + } + if v.Verdict != "fail" || !strings.Contains(v.Summary, "novox/mesh-host") || + !strings.Contains(v.Summary, "does not merge cleanly") { + t.Fatalf("a member that does not merge cleanly was answered %s: %s", v.Verdict, v.Summary) + } +} + +func gitOut(t *testing.T, dir string, args ...string) string { + t.Helper() + out, err := exec.Command("git", append([]string{"-C", dir}, args...)...).CombinedOutput() + if err != nil { + t.Fatalf("git %v: %v\n%s", args, err, out) + } + return strings.TrimSpace(string(out)) +} diff --git a/internal/builder/replay341_test.go b/internal/builder/replay341_test.go new file mode 100644 index 00000000..031a903d --- /dev/null +++ b/internal/builder/replay341_test.go @@ -0,0 +1,139 @@ +package builder + +import ( + "net" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" +) + +// TestReplay341 is the replay of novox/hq issue 341, written with only what the build agent had before its fix, +// so mesh-lab's prover can lay it over the commit before. On 2026-10-09 node-engine #64/#65 merged at 00:15 and +// added Applied.Declared, under the JSON key `declared`, to the store record. #66, checked at 00:49 at its head, +// had added its own Applied.Declared under the same key on other lines. Its check passed; git merged the two +// edits without a conflict, and the trunk no longer compiled: "Declared redeclared". Before the fix the check +// judged the head alone and passed. After it (novox/hq ADR 0323), the check judges the head merged into the +// trunk commit that held the other field, and fails saying the trunk does not compile. +// +// It runs the check whole, with a container runtime that runs each container's command on this machine +// (aRuntimeOnThisMachine): the repository's own check is `go build ./...`. +func TestReplay341(t *testing.T) { + origin, head := theNinthOfOctober(t) + registry := aFactsRegistry(t) + aRuntimeOnThisMachine(t) + + v, err := Check(t.Context(), Command, CheckSpec{ID: "replay-341", Repository: origin, Ref: head, Owner: "novox", + Repo: "mesh-host", Number: 66, Base: "main", Toolchain: "golang"}, t.TempDir(), registry, GitCredential{}, nil) + if err != nil { + t.Fatalf("the check could not run: %v", err) + } + if v.Repo == nil || v.Repo.Verdict != "fail" { + t.Fatalf("node-engine #66 at its head was answered %+v: the merge it makes into the trunk that holds #65's "+ + "field does not compile, and the check let it merge", v.Repo) + } + if said := v.Repo.Summary + "\n" + v.Repo.Failed + "\n" + v.Report; !strings.Contains(said, "Declared redeclared") { + t.Fatalf("the check failed without saying the trunk does not compile:\n%s", said) + } +} + +// theNinthOfOctober is the node-engine's repository as it stood at 00:49 on 2026-10-09, in miniature: a trunk +// holding #65's Applied.Declared (a string, key `declared`), and #66's head, branched before it, holding its own +// Applied.Declared (a *FoundUnit, the same key) on other lines. Each compiles alone; their merge does not. +func theNinthOfOctober(t *testing.T) (string, string) { + t.Helper() + origin := t.TempDir() + git := func(args ...string) string { + t.Helper() + cmd := exec.Command("git", args...) + cmd.Dir = origin + cmd.Env = append(os.Environ(), "GIT_AUTHOR_NAME=t", "GIT_AUTHOR_EMAIL=t@example.org", + "GIT_COMMITTER_NAME=t", "GIT_COMMITTER_EMAIL=t@example.org") + out, err := cmd.CombinedOutput() + if err != nil { + t.Fatalf("git %v: %v\n%s", args, err, out) + } + return strings.TrimSpace(string(out)) + } + write := func(name, body string) { + t.Helper() + if err := os.MkdirAll(filepath.Dir(filepath.Join(origin, name)), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(origin, name), []byte(body), 0o755); err != nil { + t.Fatal(err) + } + } + store := func(declared65, declared66 string) string { + return "package store\n\n// FoundUnit is a unit found on the machine.\ntype FoundUnit struct{ Name string }\n\n" + + "// Applied is what the node-engine applied.\ntype Applied struct {\n\tGeneration int `json:\"generation\"`\n" + + declared66 + "\tModules []string `json:\"modules\"`\n\tFiles []string `json:\"files\"`\n" + + "\tUnits []string `json:\"units\"`\n\tAt string `json:\"at\"`\n" + declared65 + "}\n" + } + git("init", "--quiet", "-b", "main") + write("go.mod", "module example.org/mesh-host\n\ngo 1.21\n") + write("internal/store/store.go", store("", "")) + write(CheckScript, "# mesh-check-toolchain: go\nset -e\nGOFLAGS= GOWORK=off GOTOOLCHAIN=local GOPROXY=off go build ./...\n") + git("add", "-A") + git("commit", "--quiet", "-m", "#63: the store record") + git("checkout", "--quiet", "-b", "adr-0269") + write("internal/store/store.go", store("", "\tDeclared *FoundUnit `json:\"declared\"`\n")) + git("commit", "--quiet", "-am", "#66: the unit a declaration names") + head := git("rev-parse", "HEAD") + git("checkout", "--quiet", "main") + write("internal/store/store.go", store("\tDeclared string `json:\"declared\"`\n", "")) + git("commit", "--quiet", "-am", "#65: the declaration applied") + return origin, head +} + +// aRuntimeOnThisMachine puts a container runtime on PATH that runs a container's command on this machine, in the +// directory and with the environment the check gives it: a store and a bus that answer, a judge that is never +// built, and the repository's own check run as written. Only for a check whose code is the test's own. +func aRuntimeOnThisMachine(t *testing.T) { + t.Helper() + bus, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { bus.Close() }) + go func() { + for { + c, err := bus.Accept() + if err != nil { + return + } + c.Close() + } + }() + dir := t.TempDir() + runtime := `#!/bin/sh +cmd="$1"; shift +case "$cmd" in +ps|rm|exec) exit 0 ;; +port) echo "` + bus.Addr().String() + `"; exit 0 ;; +run) + workdir=""; detached="" + while [ $# -gt 0 ]; do + case "$1" in + -d) detached=1; shift ;; + --rm) shift ;; + --workdir) workdir="$2"; shift 2 ;; + --env) export "$2"; shift 2 ;; + --network|--volume|--user|--label|--name|-p|-e) shift 2 ;; + *) break ;; + esac + done + if [ -n "$detached" ]; then echo replay-container; exit 0; fi + shift + if [ "$1" = "sh" ] && [ -n "$workdir" ]; then cd "$workdir" && exec "$@"; fi + echo "no such image on this machine" >&2 + exit 1 ;; +esac +exit 1 +` + if err := os.WriteFile(filepath.Join(dir, "docker"), []byte(runtime), 0o755); err != nil { + t.Fatal(err) + } + t.Setenv("PATH", dir+string(os.PathListSeparator)+os.Getenv("PATH")) +} diff --git a/internal/link/build.go b/internal/link/build.go index aa90165e..33f4c372 100644 --- a/internal/link/build.go +++ b/internal/link/build.go @@ -138,6 +138,9 @@ type GroupMember struct { Repository string `json:"repository"` Ref string `json:"ref"` Paths []string `json:"paths,omitempty"` + // Base is the branch it merges into: the build seat merges the head into that trunk's head before the + // group is composed (novox/hq ADR 0323 §6). A build seat from before it composes the head alone. + Base string `json:"base,omitempty"` } // Who judges a merge check's gate. @@ -164,6 +167,14 @@ type CheckOutcome struct { // Gate and RepoCheck are its two layers; Verdict and Summary above are the gate's. Gate *CheckLayer `json:"gate,omitempty"` RepoCheck *CheckLayer `json:"repo-check,omitempty"` + // Head and Trunk are what was judged (novox/hq ADR 0323 §3): the pull request's head merged into the trunk + // commit, each as twelve characters. Trunk is empty from a build seat from before it, which judged the head + // alone. Beside are the commits of the repositories read beside it, by directory, named and not judged + // against (issue 568); Members a delivery group's other heads, each with the trunk commit it merged into. + Head string `json:"head,omitempty"` + Trunk string `json:"trunk,omitempty"` + Beside map[string]string `json:"beside,omitempty"` + Members []CheckedMember `json:"members,omitempty"` } // SourceOnSeat names a repository by the seat whose holder serves it and its path there. diff --git a/internal/link/events.go b/internal/link/events.go index eab227d9..aeddd28d 100644 --- a/internal/link/events.go +++ b/internal/link/events.go @@ -278,14 +278,24 @@ type Checked struct { // status from it — the group's verdict is its owner's, mesh-delivery, to say on each member's head. Group string `json:"group,omitempty"` Members []CheckedMember `json:"members,omitempty"` + // Trunk is the trunk commit the head was merged into and judged as, twelve characters (novox/hq ADR 0323 + // §3): the verdict holds for that merge only, and goes out of date when the trunk moves. Empty from a build + // seat that judged the head alone. Beside are the commits of the other repositories the check read, by + // directory, named and not judged against (issue 568). Took is how long the check ran, for the merge tool + // to say how long the next one may take (ADR 0323 §5). + Trunk string `json:"trunk,omitempty"` + Beside map[string]string `json:"beside,omitempty"` + Took string `json:"took,omitempty"` } -// CheckedMember is one head a group's composed check judged. +// CheckedMember is one head a group's composed check judged, and the trunk commit it was merged into (novox/hq +// ADR 0323 §6), twelve characters; empty from a build seat that composed the head alone. type CheckedMember struct { Owner string `json:"owner"` Repo string `json:"repo"` Number int `json:"number,omitempty"` Commit string `json:"commit"` + Trunk string `json:"trunk,omitempty"` } // ChangePlan is what a change does to the mesh, computed from its diffset — a repository, the branch it @@ -311,6 +321,9 @@ type ChangePlan struct { Steps []string `json:"steps,omitempty"` // Summary is the plan in one line, for a commit status. Summary string `json:"summary"` + // Trunk is the trunk commit the head was merged into when it was judged (novox/hq ADR 0323 §3), twelve + // characters: set on the plan a verdict carries, which mesh-delivery keeps with its other inputs. + Trunk string `json:"trunk,omitempty"` } // MachinePlan is one machine's part of a change plan. From 25deb499bf903c389be462c80302b0c8bea6ffa0 Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 12 Oct 2026 17:31:36 +0200 Subject: [PATCH 19/26] Say why a merge's changed files read as none on an error --- internal/builder/check_merge.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/builder/check_merge.go b/internal/builder/check_merge.go index fda7f359..b9423ab8 100644 --- a/internal/builder/check_merge.go +++ b/internal/builder/check_merge.go @@ -85,7 +85,7 @@ func changedByMerge(ctx context.Context, run Runner, tree string, m TrunkMerge) } out, err := run(ctx, tree, "git", "diff", "--name-only", "--no-renames", m.Trunk, m.Merge) if err != nil { - return nil + return nil // empty-on-error: the caller keeps the pull request's own list of changed files, the safe one } return strings.Fields(out) } From ce9dc8e157e4618ba8aa755870ce350ef9bdb5f5 Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 12 Oct 2026 17:39:59 +0200 Subject: [PATCH 20/26] Run no hook and no signature setting when a check makes its merge A build machine's or a clone's git configuration must not run code or change the merge a check judges (novox/hq ADR 0323). --- internal/builder/check_merge.go | 7 ++++--- internal/builder/check_merge_test.go | 24 ++++++++++++++++++++++++ 2 files changed, 28 insertions(+), 3 deletions(-) diff --git a/internal/builder/check_merge.go b/internal/builder/check_merge.go index b9423ab8..e656d4e9 100644 --- a/internal/builder/check_merge.go +++ b/internal/builder/check_merge.go @@ -35,9 +35,10 @@ type JudgedHead struct { Head, Trunk string } -// mergeIdentity is who a check's merge is made as: it is never pushed, so it names nobody's account. +// mergeIdentity is who a check's merge is made as: it is never pushed, so it names nobody's account. No hook of the +// clone's or the build machine's git configuration runs, and no setting there changes the merge. var mergeIdentity = []string{"-c", "user.name=mesh merge check", "-c", "user.email=merge-check@mesh.invalid", - "-c", "commit.gpgsign=false"} + "-c", "commit.gpgsign=false", "-c", "core.hooksPath=/dev/null", "-c", "merge.verifySignatures=false"} // mergeIntoTrunk makes, in the clone at tree, the merge of head into the head of origin/ as it stands in // that fresh clone: the trunk commit. A head that does not merge cleanly answers the files in conflict and leaves @@ -61,7 +62,7 @@ func mergeIntoTrunk(ctx context.Context, run Runner, tree, base, head string) (T } message := fmt.Sprintf("Check the merge of %s into %s at %s (made for a merge check, never pushed)", twelve(m.Head), base, twelve(m.Trunk)) - args := append(append([]string{}, mergeIdentity...), "merge", "--no-ff", "--no-edit", "--quiet", "-m", message, m.Head) + args := append(append([]string{}, mergeIdentity...), "merge", "--no-ff", "--no-edit", "--no-verify", "--quiet", "-m", message, m.Head) if _, err := run(ctx, tree, "git", args...); err != nil { out, _ := run(ctx, tree, "git", "diff", "--name-only", "--diff-filter=U") m.Conflicts = strings.Fields(out) diff --git a/internal/builder/check_merge_test.go b/internal/builder/check_merge_test.go index 4c6955a1..a35fe888 100644 --- a/internal/builder/check_merge_test.go +++ b/internal/builder/check_merge_test.go @@ -186,3 +186,27 @@ func gitOut(t *testing.T, dir string, args ...string) string { } return strings.TrimSpace(string(out)) } + +// No hook runs when a check makes its merge: not one in the clone, whatever the build machine's git configuration says. +func TestACheckMergeRunsNoHook(t *testing.T) { + repo, head, _ := aTrunkAndABranch(t, map[string]string{"a": "1"}, map[string]string{"b": "trunk"}, + map[string]string{"c": "head"}) + clone := filepath.Join(t.TempDir(), "clone") + if out, err := exec.Command("git", "clone", "--quiet", repo, clone).CombinedOutput(); err != nil { + t.Fatalf("%v\n%s", err, out) + } + ran := filepath.Join(t.TempDir(), "ran") + for _, hook := range []string{"pre-merge-commit", "commit-msg", "prepare-commit-msg", "post-merge"} { + if err := os.WriteFile(filepath.Join(clone, ".git", "hooks", hook), + []byte("#!/bin/sh\necho "+hook+" >> "+ran+"\n"), 0o755); err != nil { + t.Fatal(err) + } + } + m, err := mergeIntoTrunk(t.Context(), Command, clone, "main", head) + if err != nil || m.Merge == "" { + t.Fatalf("the merge was not made: %+v, %v", m, err) + } + if said, err := os.ReadFile(ran); err == nil { + t.Fatalf("a hook ran when the check made its merge: %s", said) + } +} From bc623bc1f944088c80b16f285e1e696ccb1789d0 Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 12 Oct 2026 17:44:18 +0200 Subject: [PATCH 21/26] Name the announced trunk commit on a verdict no build judges A change touching no module, in a repository with no merge-check.sh, was answered by the controller with no trunk commit, so the merge tool refused it, asked again and refused it for ever (novox/mesh-tools-go; novox/hq ADR 0323). --- cmd/mesh-controller/checks.go | 22 +++++++++++++++++++--- cmd/mesh-controller/checks_test.go | 18 ++++++++++++++++++ internal/link/events.go | 4 ++++ 3 files changed, 41 insertions(+), 3 deletions(-) diff --git a/cmd/mesh-controller/checks.go b/cmd/mesh-controller/checks.go index 9faa4915..1ff6a275 100644 --- a/cmd/mesh-controller/checks.go +++ b/cmd/mesh-controller/checks.go @@ -161,9 +161,7 @@ func (f following) PullUpdated(ctx context.Context, p link.PullUpdated) error { return u, err == nil }) fmt.Print(planText(plan)) - direct := link.Checked{Owner: p.Owner, Repo: p.Repo, Number: p.Number, Commit: p.Commit, - ID: link.NewBuildID(time.Now()), Verdict: "pass", Summary: noModuleTouched, - Gate: &link.CheckLayer{Verdict: "pass", Summary: noModuleTouched}, Plan: &plan} + direct := directVerdict(p, plan) switch { case !scope.gated() && !scope.Mesh: fmt.Printf("%s/%s#%d (%.8s): %s, and the repository is not the mesh's: said, nothing run\n", @@ -201,6 +199,24 @@ func (f following) PullUpdated(ctx context.Context, p link.PullUpdated) error { return nil } +// directVerdict is the verdict of a change no build judges — it touches no module, and either the repository is not +// the mesh's or the head holds no merge-check.sh. **It names the trunk commit it holds for** (novox/hq ADR 0323 §3): +// the base branch's head the forge announced with the head, so the merge tool can weigh it against the trunk's head +// and does not refuse it for ever (a verdict naming no trunk is never the trunk's head's). Nothing was built to +// judge: the trunk commit is the one the forge said the change was announced against. +func directVerdict(p link.PullUpdated, plan link.ChangePlan) link.Checked { + trunk := p.BaseSHA + if len(trunk) > 12 { + trunk = trunk[:12] + } + if trunk != "" { + plan.Trunk = trunk + } + return link.Checked{Owner: p.Owner, Repo: p.Repo, Number: p.Number, Commit: p.Commit, + ID: link.NewBuildID(time.Now()), Verdict: "pass", Summary: noModuleTouched, + Gate: &link.CheckLayer{Verdict: "pass", Summary: noModuleTouched}, Plan: &plan, Trunk: trunk} +} + // checkRequestFor is the ask for one pull request's head: the repository as the mesh clones it, the head, // and what is read beside it. func checkRequestFor(ctx context.Context, open *stores, p link.PullUpdated, scope checkScope, diff --git a/cmd/mesh-controller/checks_test.go b/cmd/mesh-controller/checks_test.go index 65a8314f..4d62e4dc 100644 --- a/cmd/mesh-controller/checks_test.go +++ b/cmd/mesh-controller/checks_test.go @@ -260,3 +260,21 @@ func TestABuildOfACheckMergeIsRefusedNamingTheTrunk(t *testing.T) { t.Fatalf("a build of a check's merge was answered %v", err) } } + +// **A verdict no build judges names its trunk commit too** (novox/hq ADR 0323 §3): a change touching no module of a +// repository with no merge-check.sh (novox/mesh-tools-go) is answered by the controller itself. Naming no trunk, the +// merge tool refused it, asked again, got the same verdict, and refused for ever. +func TestADirectVerdictNamesTheTrunkTheForgeAnnounced(t *testing.T) { + p := link.PullUpdated{Owner: "novox", Repo: "mesh-tools-go", Number: 3, Base: "main", Commit: "1a2b3c4d5e6f7a8b9c0d", + BaseSHA: "7a8b9c0d1e2f3a4b5c6d7e8f9a0b1c2d3e4f5a6b"} + c := directVerdict(p, link.ChangePlan{Repository: "novox/mesh-tools-go", Base: "main", Head: p.Commit}) + if c.Trunk != "7a8b9c0d1e2f" || c.Plan == nil || c.Plan.Trunk != "7a8b9c0d1e2f" { + t.Fatalf("the direct verdict names trunk %q, its plan %+v", c.Trunk, c.Plan) + } + if c.Verdict != "pass" || c.Commit != p.Commit { + t.Fatalf("the direct verdict is %+v", c) + } + if c := directVerdict(link.PullUpdated{Owner: "novox", Repo: "x", Commit: "abc"}, link.ChangePlan{}); c.Trunk != "" { + t.Fatalf("a forge that said no base head gave the trunk %q", c.Trunk) + } +} diff --git a/internal/link/events.go b/internal/link/events.go index aeddd28d..cd5c1b1d 100644 --- a/internal/link/events.go +++ b/internal/link/events.go @@ -228,6 +228,10 @@ type PullUpdated struct { Base string `json:"base"` Head string `json:"head"` Commit string `json:"head_sha"` + // BaseSHA is the base branch's head when the forge announced the head (novox/hq ADR 0323): the trunk commit a + // verdict the controller gives without the build seat — a change that runs no build — is judged against. Empty + // from a forge module from before it. + BaseSHA string `json:"base_sha,omitempty"` CloneURL string `json:"clone_url"` HTMLURL string `json:"html_url,omitempty"` // Paths are the files the pull request changes; PathsTruncated says there were more. From 875ebc093e187557fb908a958ed8336081a6bf7c Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 12 Oct 2026 17:52:12 +0200 Subject: [PATCH 22/26] Format PullUpdated, and keep the announced base head on a rechecked member A head checked again through mesh-delivery reached the direct verdict with no base head, so a head no build judges was answered with no trunk commit and the merge tool would refuse it (novox/hq ADR 0323). --- cmd/mesh-controller/checks_test.go | 10 ++++++++++ cmd/mesh-controller/delivery.go | 5 ++++- internal/link/events.go | 16 ++++++++-------- 3 files changed, 22 insertions(+), 9 deletions(-) diff --git a/cmd/mesh-controller/checks_test.go b/cmd/mesh-controller/checks_test.go index 4d62e4dc..8ea4242a 100644 --- a/cmd/mesh-controller/checks_test.go +++ b/cmd/mesh-controller/checks_test.go @@ -278,3 +278,13 @@ func TestADirectVerdictNamesTheTrunkTheForgeAnnounced(t *testing.T) { t.Fatalf("a forge that said no base head gave the trunk %q", c.Trunk) } } + +// A head checked again through mesh-delivery (`delivery check --members`, a recheck) reaches the same direct verdict: +// it names the base head the forge last announced, kept on the member, or the merge tool would refuse it for ever. +func TestARecheckedHeadsDirectVerdictNamesItsTrunk(t *testing.T) { + m := orderMember{ID: "novox/mesh-tools-go@1a2b3c4d5e6f", Repository: "novox/mesh-tools-go", Base: "main", + Head: "1a2b3c4d5e6f7a8b9c0d", Number: 3, BaseSHA: "7a8b9c0d1e2f3a4b5c6d7e8f9a0b1c2d3e4f5a6b"} + if c := directVerdict(m.pull(), link.ChangePlan{}); c.Trunk != "7a8b9c0d1e2f" { + t.Fatalf("a rechecked head's direct verdict names trunk %q", c.Trunk) + } +} diff --git a/cmd/mesh-controller/delivery.go b/cmd/mesh-controller/delivery.go index 235f6a51..615386c0 100644 --- a/cmd/mesh-controller/delivery.go +++ b/cmd/mesh-controller/delivery.go @@ -580,6 +580,9 @@ type orderMember struct { CloneURL string `json:"clone_url,omitempty"` // After are the repositories its pull request says it goes after (`after: `). After []string `json:"after,omitempty"` + // BaseSHA is the base branch's head the forge last announced the head with (novox/hq ADR 0323): the trunk commit + // a verdict no build judges names when the head is checked again through mesh-delivery. + BaseSHA string `json:"base_sha,omitempty"` } // pull is the member as the forge announced it. @@ -589,7 +592,7 @@ func (m orderMember) pull() link.PullUpdated { if base == "" { base = "main" } - return link.PullUpdated{Owner: owner, Repo: repo, Number: m.Number, Base: base, Commit: m.Head, + return link.PullUpdated{Owner: owner, Repo: repo, Number: m.Number, Base: base, Commit: m.Head, BaseSHA: m.BaseSHA, CloneURL: m.CloneURL, Paths: m.Paths, PathsTruncated: m.PathsTruncated, Removed: m.Removed, ModuleDirs: m.ModuleDirs, ModuleDirsSaid: m.ModuleDirsSaid} } diff --git a/internal/link/events.go b/internal/link/events.go index cd5c1b1d..ab2a511e 100644 --- a/internal/link/events.go +++ b/internal/link/events.go @@ -221,17 +221,17 @@ type SourceMoved struct { // PullUpdated is what the forge announces when an open pull request's head moves — opened, or pushed // to (novox/hq to-be 45 §9): what the controller asks the build seat to check before it merges. type PullUpdated struct { - Owner string `json:"owner"` - Repo string `json:"repo"` - Number int `json:"number"` - Title string `json:"title,omitempty"` - Base string `json:"base"` - Head string `json:"head"` - Commit string `json:"head_sha"` + Owner string `json:"owner"` + Repo string `json:"repo"` + Number int `json:"number"` + Title string `json:"title,omitempty"` + Base string `json:"base"` + Head string `json:"head"` + Commit string `json:"head_sha"` // BaseSHA is the base branch's head when the forge announced the head (novox/hq ADR 0323): the trunk commit a // verdict the controller gives without the build seat — a change that runs no build — is judged against. Empty // from a forge module from before it. - BaseSHA string `json:"base_sha,omitempty"` + BaseSHA string `json:"base_sha,omitempty"` CloneURL string `json:"clone_url"` HTMLURL string `json:"html_url,omitempty"` // Paths are the files the pull request changes; PathsTruncated says there were more. From 9038d4b0d317d27be0183354b3d4b73cd3b89b9c Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 12 Oct 2026 18:21:17 +0200 Subject: [PATCH 23/26] Refuse a send that puts the controller or node-engine behind its machine's build (issue 201) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit On 2026-10-02 a plan sent the control node a controller older than the one running there, whose seat row the newer build had widened; the older binary refused the row and the mesh went quiet for ten minutes. ADR 0185 took the outage out of it, and the race stayed open. Every send but a person's push now refuses a move of mesh-controller or mesh-host to a build asked before the one its machine was last sent (the builds' registration order, issue 219), whatever composed it — gated or not. A gate's own put-back is the one exception, for the modules it puts back, and says so in its send's scope. The refusal is the send's error, which the plan keeps as its note, and a warning condition on the machine, cleared by the next send there that carries no such move. --- cmd/mesh-controller/core_downgrade.go | 171 ++++++++++++++++++++++++++ cmd/mesh-controller/gate.go | 4 +- cmd/mesh-controller/issue201_test.go | 169 +++++++++++++++++++++++++ cmd/mesh-controller/plain_words.go | 14 +++ cmd/mesh-controller/release.go | 38 +++++- 5 files changed, 390 insertions(+), 6 deletions(-) create mode 100644 cmd/mesh-controller/core_downgrade.go create mode 100644 cmd/mesh-controller/issue201_test.go diff --git a/cmd/mesh-controller/core_downgrade.go b/cmd/mesh-controller/core_downgrade.go new file mode 100644 index 00000000..e66f11c9 --- /dev/null +++ b/cmd/mesh-controller/core_downgrade.go @@ -0,0 +1,171 @@ +package main + +import ( + "context" + "errors" + "fmt" + "sort" + "time" + + "github.com/novox/mesh-controller/internal/conditions" + "github.com/novox/mesh-controller/internal/inventory" + "github.com/novox/mesh-controller/internal/lease" +) + +// **A send never moves the mesh's own core to a build older than the one its machine runs** (novox/hq issue +// 201). On 2026-10-02 two merges a minute apart made two plans; the controller's plan rolled the new controller, +// which widened its seat's row, and the node-engine's plan then sent the control node the controller build +// before it. The older binary read the row it could not serve and the mesh lost its voice for ten minutes — the +// thing that would have corrected it was the thing that was down. ADR 0185 took the outage out of it (a +// controller behind its row serves what it can); the race is closed here, without needing its trigger: whatever +// composed the older build, the send that would carry it is refused. +// +// Older is by the builds' registration order — when each was asked (novox/hq issue 219) — never by the +// commits' dates. A gate's own put-back is the one send that may carry it, for the modules it puts back: it says +// so (sendScope.puttingBack), and the controller's put-back is held to the store's schema already (issue 352). +// A person's push carries what it carries, as every other refusal of a send leaves it. +// +// The refusal is said in the plan's words — the send's error, which the plan keeps as its note and tries again +// — and as a warning condition on the machine, cleared by the first send there that carries no such move. + +// errCoreDowngrade is a send refused because it would put a core module on a machine back behind the build that +// machine runs. +var errCoreDowngrade = errors.New("a send would put the mesh's own core back behind the build its machine runs") + +// kindCoreDowngrade is that refusal, as a condition. +const kindCoreDowngrade = "core-downgrade-refused" + +// coreDowngradeSource is the condition's source: the sends, which raise and clear it. +const coreDowngradeSource = "sends" + +// downgradeGuarded is the core components a send may not move backwards: the controller, which reads the +// store and its seat's row, and the node-engine, which reads what the controller declares. +var downgradeGuarded = []string{lease.ComponentController, lease.ComponentEngine} + +// buildAsked is when the newest build of a module from a commit that worked was asked — or recorded, when the +// request time is not known — and false when the records hold none. A variable, so a test reads its own order. +var buildAsked = func(ctx context.Context, inv *inventory.Inventory, module, commit string) (time.Time, bool, error) { + builds, err := inv.Builds(ctx, module, askedLookBack) + if err != nil { + return time.Time{}, false, err + } + for _, b := range builds { + if b.Worked() && sameCommit(b.Commit, commit) { + return b.AskedOrAt(), true, nil + } + } + return time.Time{}, false, nil +} + +// askedLookBack is how many of a module's newest builds are read for when a build was asked. +const askedLookBack = 50 + +// coreDowngrade is one move of a core module to an older build than its machine runs. +type coreDowngrade struct { + move inventory.CarriedMove + component string + fromAsked, asked time.Time +} + +func (d coreDowngrade) String() string { + return fmt.Sprintf("%s would be sent %s %s, older than the %s it runs (asked %s, before %s) — no send but a "+ + "gate's own put-back moves the %s back (novox/hq issue 201)", d.move.Node, d.move.Module, short(d.move.To), + short(d.move.From), d.asked.UTC().Format(time.RFC3339), d.fromAsked.UTC().Format(time.RFC3339), d.component) +} + +// coreDowngrades is the moves of a machine's send that put a core module back to a build asked before the one it +// runs, leaving out what a put-back puts back. A move whose either build the records do not hold is not proved +// older, and is left to the gate that judges it. Pure but for asked. +func coreDowngrades(moves []inventory.CarriedMove, scope sendScope, + asked func(module, commit string) (time.Time, bool, error)) ([]coreDowngrade, error) { + var out []coreDowngrade + for _, mv := range moves { + component := coreComponent(mv.Module) + if !guarded(component) || mv.From == "" || mv.To == "" || sameCommit(mv.From, mv.To) { + continue + } + if scope.puttingBack && scope.modules[mv.Module] { + continue + } + from, known, err := asked(mv.Module, mv.From) + if err != nil || !known { + if err != nil { + return nil, fmt.Errorf("when %s's build %s was asked cannot be read: %w", mv.Module, short(mv.From), err) + } + continue + } + to, known, err := asked(mv.Module, mv.To) + if err != nil { + return nil, fmt.Errorf("when %s's build %s was asked cannot be read: %w", mv.Module, short(mv.To), err) + } + if known && to.Before(from) { + out = append(out, coreDowngrade{move: mv, component: component, fromAsked: from, asked: to}) + } + } + sort.Slice(out, func(i, j int) bool { return out[i].move.Module < out[j].move.Module }) + return out, nil +} + +func guarded(component string) bool { + for _, c := range downgradeGuarded { + if c == component { + return true + } + } + return false +} + +// refuseCoreDowngrades says, for one machine's moves, each that would put a core module back, raising its +// condition; and clears the condition of each guarded component that machine's send does not move back. +func refuseCoreDowngrades(ctx context.Context, inv *inventory.Inventory, scope sendScope, node string, + moves []inventory.CarriedMove) ([]string, error) { + found, err := coreDowngrades(moves, scope, func(module, commit string) (time.Time, bool, error) { + return buildAsked(ctx, inv, module, commit) + }) + if err != nil { + return nil, err + } + var said []string + refused := map[string]bool{} + for _, d := range found { + said = append(said, d.String()) + refused[d.component] = true + sayCoreDowngrade(ctx, d) + } + for _, c := range downgradeGuarded { + if !refused[c] { + clearCoreDowngrade(ctx, c, node) + } + } + return said, nil +} + +// coreDowngradeObservation is a refused downgrade as a condition: `core...`. +func coreDowngradeObservation(component, node, summary string) conditions.Observation { + return conditions.Observation{Scope: conditions.ScopeCore, ID: component + "." + node, Kind: kindCoreDowngrade, + Token: kindCoreDowngrade, Machine: node, Severity: conditions.Warning, Source: coreDowngradeSource, + Summary: summary} +} + +func sayCoreDowngrade(ctx context.Context, d coreDowngrade) { + fmt.Printf("NOT SENT: %s\n", d) + keeper := conditionsFrom + if keeper == nil { + return + } + if _, err := keeper.Observe(ctx, coreDowngradeObservation(d.component, d.move.Node, d.String())); err != nil { + fmt.Printf("the condition for %s's refused downgrade on %s could not be raised: %v\n", d.move.Module, + d.move.Node, err) + } +} + +func clearCoreDowngrade(ctx context.Context, component, node string) { + keeper := conditionsFrom + if keeper == nil { + return + } + key := coreDowngradeObservation(component, node, "").Key() + if _, err := keeper.Clear(ctx, key, "a send to "+node+" carries no older "+component); err != nil { + fmt.Printf("the condition %s could not be cleared: %v\n", key, err) + } +} diff --git a/cmd/mesh-controller/gate.go b/cmd/mesh-controller/gate.go index b4dd98e2..f5aa1689 100644 --- a/cmd/mesh-controller/gate.go +++ b/cmd/mesh-controller/gate.go @@ -1039,7 +1039,7 @@ func gateFailed(ctx context.Context, open *stores, p *inventory.Plan, module str } return } - sent, err := sendRollout(withScope(ctx, sendScope{modules: map[string]bool{module: true}}), open, g.Machines) + sent, err := sendRollout(withScope(ctx, sendScope{modules: map[string]bool{module: true}, puttingBack: true}), open, g.Machines) if err != nil { notBack(fmt.Sprintf("its registered build is back at %s, and sending it to %s was refused: %v — `push %s` "+ "sends it", short(previous.Commit), strings.Join(g.Machines, ", "), err, g.Machines[0])) @@ -1132,7 +1132,7 @@ func sendRollbacks(ctx context.Context, open *stores, p *inventory.Plan, b *roll } inv := open.inventory sort.Strings(b.machines) - sent, err := sendRollout(withScope(ctx, sendScope{modules: b.modules}), open, b.machines) + sent, err := sendRollout(withScope(ctx, sendScope{modules: b.modules, puttingBack: true}), open, b.machines) var back []string for _, r := range b.pending { if err != nil { diff --git a/cmd/mesh-controller/issue201_test.go b/cmd/mesh-controller/issue201_test.go new file mode 100644 index 00000000..9ec8f3bd --- /dev/null +++ b/cmd/mesh-controller/issue201_test.go @@ -0,0 +1,169 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "strings" + "testing" + "time" + + "github.com/novox/mesh-controller/internal/catalogue" + "github.com/novox/mesh-controller/internal/inventory" + "github.com/novox/mesh-controller/internal/lease" +) + +// novox/hq issue 201: on 2026-10-02 a plan sent the control node a controller older than the one running there, +// whose seat row the newer one had already widened; the older binary refused the row and the mesh went quiet for +// ten minutes. No send but a gate's own put-back moves the controller or the node-engine back behind the build its +// machine runs, by the builds' registration order. + +// askedIn answers when a build was asked from a fixed table: module@commit → when. +func askedIn(table map[string]time.Time) func(module, commit string) (time.Time, bool, error) { + return func(module, commit string) (time.Time, bool, error) { + at, known := table[module+"@"+commit] + return at, known, nil + } +} + +func TestACoreModuleIsNeverMovedBehindTheBuildItsMachineRuns(t *testing.T) { + old, now := t0, t0.Add(time.Minute) + asked := askedIn(map[string]time.Time{ + "mesh-controller@c1": old, "mesh-controller@c2": now, + "mesh-host@h1": old, "mesh-host@h2": now, + "app@a1": old, "app@a2": now, + }) + back := []inventory.CarriedMove{ + {Module: "mesh-controller", Node: "anchor", From: "c2", To: "c1"}, + {Module: "mesh-host", Node: "anchor", From: "h2", To: "h1"}, + {Module: "app", Node: "anchor", From: "a2", To: "a1"}, + } + found, err := coreDowngrades(back, sendScope{judged: map[string]bool{"anchor": true}}, asked) + if err != nil { + t.Fatal(err) + } + if len(found) != 2 || found[0].move.Module != "mesh-controller" || found[0].component != lease.ComponentController || + found[1].move.Module != "mesh-host" || found[1].component != lease.ComponentEngine { + t.Fatalf("the controller and the node-engine sent back, and nothing else, are refused: %+v", found) + } + if said := found[0].String(); !strings.Contains(said, "anchor would be sent mesh-controller c1") || + !strings.Contains(said, "issue 201") { + t.Fatalf("the refusal says the machine, the module and the build: %s", said) + } + + // Forward, the same build, or a build the records do not hold: not refused. + for _, mv := range []inventory.CarriedMove{ + {Module: "mesh-controller", Node: "anchor", From: "c1", To: "c2"}, + {Module: "mesh-controller", Node: "anchor", From: "c2", To: "c2"}, + {Module: "mesh-controller", Node: "anchor", From: "c2", To: "c9"}, + {Module: "mesh-controller", Node: "anchor", From: "c9", To: "c1"}, + } { + if found, err := coreDowngrades([]inventory.CarriedMove{mv}, sendScope{}, asked); err != nil || len(found) != 0 { + t.Fatalf("%+v is no downgrade proved: %+v %v", mv, found, err) + } + } + + // A gate's own put-back carries its own modules back, and nothing else. + putBack := sendScope{modules: map[string]bool{"mesh-controller": true}, puttingBack: true} + found, err = coreDowngrades(back, putBack, asked) + if err != nil { + t.Fatal(err) + } + if len(found) != 1 || found[0].move.Module != "mesh-host" { + t.Fatalf("a put-back of the controller carries it back, and not the node-engine: %+v", found) + } + // Naming a module is not a put-back. + if found, _ := coreDowngrades(back[:1], sendScope{modules: map[string]bool{"mesh-controller": true}}, asked); len(found) != 1 { + t.Fatalf("a send for the controller that is no put-back carries it back: %+v", found) + } +} + +// The refusal is a warning condition on the machine, which the next send there carrying no such move clears. +func TestARefusedCoreDowngradeIsSaidAndClearedByTheNextSend(t *testing.T) { + k, _ := withConditionsInMemory(t) + ctx := t.Context() + was := buildAsked + buildAsked = func(_ context.Context, _ *inventory.Inventory, module, commit string) (time.Time, bool, error) { + return askedIn(map[string]time.Time{"mesh-controller@c1": t0, "mesh-controller@c2": t0.Add(time.Minute)})(module, commit) + } + t.Cleanup(func() { buildAsked = was }) + + said, err := refuseCoreDowngrades(ctx, nil, sendScope{}, "anchor", + []inventory.CarriedMove{{Module: "mesh-controller", Node: "anchor", From: "c2", To: "c1"}}) + if err != nil || len(said) != 1 { + t.Fatalf("the downgrade is refused: %v %v", said, err) + } + key := coreDowngradeObservation(lease.ComponentController, "anchor", "").Key() + c, open, err := k.Get(ctx, key) + if err != nil || !open || !strings.Contains(c.Summary, "mesh-controller c1") { + t.Fatalf("the condition %s is raised, saying the build: %+v %v %v", key, c, open, err) + } + + said, err = refuseCoreDowngrades(ctx, nil, sendScope{}, "anchor", + []inventory.CarriedMove{{Module: "mesh-controller", Node: "anchor", From: "c1", To: "c2"}}) + if err != nil || len(said) != 0 { + t.Fatalf("a move forward is sent: %v %v", said, err) + } + if _, open, err := k.Get(ctx, key); err != nil || open { + t.Fatalf("the condition is cleared by a send carrying no older controller: %v %v", open, err) + } +} + +// TestReplay201 replays the race on a store: the control node runs the newer controller, the mesh's record of +// the controller is the build before it, and a plan's gated send there — the node-engine's plan of 2026-10-02 — +// is refused, said as the plan's words; the gate's own put-back of the controller is not. +func TestReplay201(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + inv := open.inventory + old, now := time.Now().Add(-2*time.Hour), time.Now().Add(-time.Minute) + record := func(commit string, asked time.Time) inventory.Build { + manifest, _ := json.Marshal(catalogue.Manifest{Module: "mesh-controller", Version: "1"}) + b := inventory.Build{ID: "build-mesh-controller-" + commit, Module: "mesh-controller", Commit: commit, + Repository: "novox/mesh-controller", Manifest: manifest, Asked: asked, At: asked, + Made: []inventory.Artifact{{Name: "x", Kind: "bundle", Reference: "sha256:" + commit + "0000000000000000"}}} + if err := inv.RecordBuild(ctx, b); err != nil { + t.Fatal(err) + } + return b + } + before := record("c1aaaaaaaaaa", old) + record("c2bbbbbbbbbb", now) + if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "mesh-controller", Version: "1"}, inventory.Source{ + Repository: "novox/mesh-controller", BuiltFrom: "c2bbbbbbbbbb", Head: "c2bbbbbbbbbb", Asked: now}); err != nil { + t.Fatal(err) + } + if _, err := inv.Assign(ctx, "anchor", "mesh-controller"); err != nil { + t.Fatal(err) + } + if err := inv.RecordSent(ctx, nodeID(t, open, "anchor"), "d-anchor", map[string]string{"mesh-controller": "c2bbbbbbbbbb"}); err != nil { + t.Fatal(err) + } + // The mesh's record goes back to the build before, as the record went on 2026-10-02. + if err := inv.RestoreModule(ctx, before); err != nil { + t.Fatal(err) + } + wasMoves := machineMoves + machineMoves = func(ctx context.Context, open *stores, f moveFacts, node string, all bool) ([]inventory.CarriedMove, error) { + modules, err := open.inventory.Assigned(ctx, node) + if err != nil { + return nil, err + } + sent, known, err := open.inventory.SentBuilds(ctx, node) + if err != nil { + return nil, err + } + return f.moves(node, modules, sent, known, all), nil + } + t.Cleanup(func() { machineMoves = wasMoves }) + + judged := withScope(ctx, sendScope{judged: map[string]bool{"anchor": true}}) + if _, err := ungatedIn(judged, open, []string{"anchor"}, ""); !errors.Is(err, errCoreDowngrade) || + !strings.Contains(err.Error(), "mesh-controller c1aaaaaa") { + t.Fatalf("a plan's send of the older controller to the machine running the newer is refused: %v", err) + } + putBack := withScope(ctx, sendScope{modules: map[string]bool{"mesh-controller": true}, puttingBack: true}) + if kept, err := ungatedIn(putBack, open, []string{"anchor"}, ""); err != nil || len(kept) != 1 { + t.Fatalf("the gate's own put-back of the controller is sent: %v %v", kept, err) + } +} diff --git a/cmd/mesh-controller/plain_words.go b/cmd/mesh-controller/plain_words.go index 7cd4f3b6..e2caf2d4 100644 --- a/cmd/mesh-controller/plain_words.go +++ b/cmd/mesh-controller/plain_words.go @@ -182,6 +182,20 @@ var plainWordings = map[string]func(conditions.Observation) words{ "register it, so it was not made.", module, m), Resolved: "Resolved: " + module + " on " + m + " is settled"} }), + // A send that would have put the controller or the node-engine back behind what a machine runs (novox/hq + // issue 201). + kindCoreDowngrade: worded(func(o conditions.Observation) words { + m := machineOr(o, "a machine") + component := idPart(o, 0) + if o.Scope != conditions.ScopeCore || !guarded(component) { + component = "core component" + } + return words{Headline: "An older " + component + " was not sent to " + m, + Needs: "decide which build of the " + component + " the mesh should hold; the details say which.", + Explanation: fmt.Sprintf("A send would have put the mesh's own %s on %s back to an older build than "+ + "the one it runs, so it was not sent and %s keeps its build.", component, m, m), + Resolved: "Resolved: sends to " + m + " carry no older " + component} + }), kindBindingKept: worded(func(o conditions.Observation) words { m := machineOr(o, "a machine") return words{Headline: "A module's data source is held on " + m, diff --git a/cmd/mesh-controller/release.go b/cmd/mesh-controller/release.go index bc16fd84..5f23d856 100644 --- a/cmd/mesh-controller/release.go +++ b/cmd/mesh-controller/release.go @@ -51,6 +51,9 @@ type sendScope struct { modules map[string]bool // person is a person's act: it carries what it carries. person bool + // puttingBack is a gate's own put-back of its modules: the one send that may carry a core module to a build + // older than the one its machine runs (novox/hq issue 201). + puttingBack bool } type sendScopeKey struct{} @@ -248,9 +251,11 @@ func ungatedIn(ctx context.Context, open *stores, names []string, addedHolder st if err != nil { return nil, err } - var kept, refused []string + var kept, refused, older []string for _, n := range names { - moves, err := machineMoves(ctx, open, f, n, false) + // Every move, gated or not: a move to a build that passed a gate elsewhere is no gate's concern here, and + // may still be a core module's downgrade (novox/hq issue 201). + every, err := machineMoves(ctx, open, f, n, true) if err != nil { return nil, err } @@ -262,11 +267,33 @@ func ungatedIn(ctx context.Context, open *stores, names []string, addedHolder st return nil, err } } - var waiting []string - for _, mv := range moves { + var made []inventory.CarriedMove + for _, mv := range every { if was, kept := keeps[mv.Module]; kept && sameCommit(was, mv.From) { continue } + made = append(made, mv) + } + // **No send but a gate's own put-back moves the controller or the node-engine to an older build than + // the one its machine runs** (novox/hq issue 201). + downgrades, err := refuseCoreDowngrades(ctx, open.inventory, scope, n, made) + if err != nil { + return nil, err + } + if len(downgrades) > 0 { + if n == addedHolder { + fmt.Printf("%s holds the bus and its user list changed, and it is not sent now: %s\n", n, + strings.Join(downgrades, "; ")) + continue + } + older = append(older, downgrades...) + continue + } + var waiting []string + for _, mv := range made { + if f.gated(mv.Module, mv.To) { + continue + } if !scope.judged[n] && !scope.modules[mv.Module] { waiting = append(waiting, fmt.Sprintf("%s %s → %s", mv.Module, short(mv.From), short(mv.To))) } @@ -281,6 +308,9 @@ func ungatedIn(ctx context.Context, open *stores, names []string, addedHolder st refused = append(refused, fmt.Sprintf("%s (%s)", n, strings.Join(waiting, ", "))) } } + if len(older) > 0 { + return nil, fmt.Errorf("%w: %s", errCoreDowngrade, strings.Join(older, "; ")) + } if len(refused) > 0 { return nil, fmt.Errorf("%w: %s — a walk sends them, one machine at a time, each judged (novox/hq ADR "+ "0236); `upgrade backlog` lists them", errUngated, strings.Join(refused, "; ")) From f3c15e00af5004d053bc424fe01c7f9a24656d59 Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 12 Oct 2026 18:36:22 +0200 Subject: [PATCH 24/26] Add issue 401's replay, written with what the controller had before its fix, so the lab can prove it The fix's own test reads GivenOwnSecret, which the commit before does not have, so it cannot be laid over it. TestReplay401 asserts the same outcome with register, assign, planFor, declarationFor and the inventory's secret reads; it fails on 1e28342^1 and passes on main. Named in testdata/store-tests so a skip without the store fails the check. Replay for hq issue 401. --- cmd/mesh-controller/replay401_test.go | 83 +++++++++++++++++++++++++++ testdata/store-tests | 1 + 2 files changed, 84 insertions(+) create mode 100644 cmd/mesh-controller/replay401_test.go diff --git a/cmd/mesh-controller/replay401_test.go b/cmd/mesh-controller/replay401_test.go new file mode 100644 index 00000000..0c4fd94b --- /dev/null +++ b/cmd/mesh-controller/replay401_test.go @@ -0,0 +1,83 @@ +package main + +import ( + "testing" + + "github.com/novox/mesh-controller/internal/catalogue" +) + +// TestReplay401 replays novox/hq issue 401 (2026-10-10): the mounts module's smb-credentials, an own secret +// only an outside party issues, was given by nobody, and the send made a random value for it and placed it on +// the machine; the module read it as a credential given and wrong. The outcome asserted (ADR 0228): the send +// makes no value for an own secret issued outside the mesh that nobody gave and places none, and a value the +// mesh made for it before is not placed either; once a person gives one, that value is placed. Written with +// only what the controller had before its fix (mesh-controller #262) — register, assign, planFor, +// declarationFor and the inventory's secret reads — so it is laid over the commit before. Needs the store +// (MESH_TEST_POSTGRES), and is named in testdata/store-tests. +func TestReplay401(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + const made, outside = "/var/lib/mesh/mounts/password", "/var/lib/mesh/mounts/smb-credentials" + register(t, open, catalogue.Manifest{Module: "mounts", Version: "1", + OwnSecrets: catalogue.OwnSecrets{ + "password": {Path: made, Taken: catalogue.TakenAtStart}, + "smb-credentials": {Path: outside, Taken: catalogue.TakenAtStart, IssuedBy: catalogue.IssuedOutside}, + }, + Resources: []map[string]any{ + {"id": "state", "type": "directory", "path": "/var/lib/mesh/mounts", "mode": "0700"}, + }}) + if _, err := assign(ctx, open, "laptop", "mounts"); err != nil { + t.Fatal(err) + } + sent := func() []map[string]any { + t.Helper() + plan, settings, err := planFor(ctx, open, "laptop") + if err != nil { + t.Fatal(err) + } + declared, err := declarationFor(ctx, open, "laptop", plan, settings) + if err != nil { + t.Fatalf("the send to laptop did not compose: %v", err) + } + return declared.Resources + } + placed := func(resources []map[string]any, path string) map[string]any { + for _, r := range resources { + if r["path"] == path && r["sealed"] != nil { + return r + } + } + return nil + } + + first := sent() + if placed(first, made) == nil { + t.Fatal("the secret the mesh makes was not placed") + } + if got := placed(first, outside); got != nil { + t.Errorf("the send placed a value for smb-credentials, which nobody gave: %v", got) + } + if _, held, err := open.inventory.ModuleSecretIfIssued(ctx, "laptop", "mounts", "smb-credentials"); err != nil || held { + t.Fatalf("the send made a value for a secret issued outside the mesh (held %v, %v)", held, err) + } + + // A value the mesh made for it before the rule is not one a person gave: it is not placed. + if _, err := open.inventory.SecretForModule(ctx, "laptop", "mounts", "smb-credentials"); err != nil { + t.Fatal(err) + } + if got := placed(sent(), outside); got != nil { + t.Fatalf("a value the mesh made for a secret issued outside was placed: %v", got) + } + + // Given by a person, it is placed, as given. + if err := open.inventory.AcceptSecretForModule(ctx, "laptop", "mounts", "smb-credentials", "the share's credentials"); err != nil { + t.Fatal(err) + } + given, held, err := open.inventory.ModuleSecretIfIssued(ctx, "laptop", "mounts", "smb-credentials") + if err != nil || !held { + t.Fatalf("the value given is not held (%v)", err) + } + if got := placed(sent(), outside); got == nil || got["sealed"] != given { + t.Fatalf("the value given is placed as %v", got) + } +} diff --git a/testdata/store-tests b/testdata/store-tests index c510f6d9..716c4f8e 100644 --- a/testdata/store-tests +++ b/testdata/store-tests @@ -29,3 +29,4 @@ ./cmd/mesh-controller TestADirectoryOwnersKeyIsRefusedThroughAVerb ./cmd/mesh-controller TestTheSendMakesNoValueForAnOutsideSecretNobodyGave ./cmd/mesh-controller TestAQuestionComposesWithoutAnOutsideSecretNobodyGave +./cmd/mesh-controller TestReplay401 From d17049125655568a90396626c682df23f25e58bd Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 12 Oct 2026 18:51:17 +0200 Subject: [PATCH 25/26] Keep the core over an older record instead of refusing the send (issue 201, review 90) Refusing the whole send wedged a machine whose gate put back the controller and whose put-back send was refused: every later automatic send there was refused, retried each tick, and held up every other walk. A send now composes the controller and the node-engine at the build the machine was last sent when the mesh's record is older, and carries the rest; nothing is refused or retried, and a plan whose own build is the older one ends with its gate superseded. A sent build that failed its gate is never kept, so the next send completes a put-back. The condition names the two ways out and clears on any send that carries the record, a person's push included. --- cmd/mesh-controller/core_downgrade.go | 201 ++++++++++++++++---------- cmd/mesh-controller/issue201_test.go | 188 ++++++++++++++---------- cmd/mesh-controller/plain_words.go | 12 +- cmd/mesh-controller/push.go | 3 + cmd/mesh-controller/recorded_kept.go | 17 +++ cmd/mesh-controller/release.go | 46 ++---- 6 files changed, 267 insertions(+), 200 deletions(-) diff --git a/cmd/mesh-controller/core_downgrade.go b/cmd/mesh-controller/core_downgrade.go index e66f11c9..e4b9885d 100644 --- a/cmd/mesh-controller/core_downgrade.go +++ b/cmd/mesh-controller/core_downgrade.go @@ -2,7 +2,6 @@ package main import ( "context" - "errors" "fmt" "sort" "time" @@ -12,28 +11,31 @@ import ( "github.com/novox/mesh-controller/internal/lease" ) -// **A send never moves the mesh's own core to a build older than the one its machine runs** (novox/hq issue -// 201). On 2026-10-02 two merges a minute apart made two plans; the controller's plan rolled the new controller, -// which widened its seat's row, and the node-engine's plan then sent the control node the controller build -// before it. The older binary read the row it could not serve and the mesh lost its voice for ten minutes — the -// thing that would have corrected it was the thing that was down. ADR 0185 took the outage out of it (a -// controller behind its row serves what it can); the race is closed here, without needing its trigger: whatever -// composed the older build, the send that would carry it is refused. +// **No send but a person's push and a gate's own put-back moves the mesh's own core to a build older than the +// one its machine was last sent** (novox/hq issue 201). On 2026-10-02 two merges a minute apart made two plans; +// the controller's plan rolled the new controller, which widened its seat's row, and the node-engine's plan then +// sent the control node the controller build before it. The older binary read the row it could not serve and the +// mesh lost its voice for ten minutes — the thing that would have corrected it was the thing that was down. ADR +// 0185 took the outage out of it (a controller behind its row serves what it can); the race is closed here, +// without needing its trigger: whatever made the mesh's record older, the send keeps the build. // -// Older is by the builds' registration order — when each was asked (novox/hq issue 219) — never by the -// commits' dates. A gate's own put-back is the one send that may carry it, for the modules it puts back: it says -// so (sendScope.puttingBack), and the controller's put-back is held to the store's schema already (issue 352). -// A person's push carries what it carries, as every other refusal of a send leaves it. +// **Kept, not refused.** The send composes the controller or the node-engine at the build the machine was last +// sent, as a recorded module is kept (ADR 0245), and carries everything else: no other module, secret, healer or +// walk waits on it, and nothing is retried. A plan whose own build is the older one finds its gate superseded by +// the build kept there (issue 352) and ends. The mesh's record stays older, and a warning condition says so until +// a send carries the record — a person's `push ` accepts the older build — or the record moves forward: a +// newer merge, or `rebuild mesh-controller`. // -// The refusal is said in the plan's words — the send's error, which the plan keeps as its note and tries again -// — and as a warning condition on the machine, cleared by the first send there that carries no such move. +// Older is by the builds' registration order — when each was asked (novox/hq issue 219) — never by the commits' +// dates. So a retry or a rebuild of an old commit, asked anew, counts as newer: it is registered as the module's +// current build by the same order, and the guard does not stop that move. A build that failed its gate is never +// ahead: the machine was sent it and the record was put back before it (gateFailed), and if the put-back's own +// send was refused, the next send completes it rather than keeping the failed build there. A gate's own put-back +// carries its modules back whatever the order (sendScope.puttingBack); the controller's is held to the store's +// schema already (issue 352). -// errCoreDowngrade is a send refused because it would put a core module on a machine back behind the build that -// machine runs. -var errCoreDowngrade = errors.New("a send would put the mesh's own core back behind the build its machine runs") - -// kindCoreDowngrade is that refusal, as a condition. -const kindCoreDowngrade = "core-downgrade-refused" +// kindCoreDowngrade is a core module kept at the build its machine was last sent, over an older record. +const kindCoreDowngrade = "core-downgrade-kept" // coreDowngradeSource is the condition's source: the sends, which raise and clear it. const coreDowngradeSource = "sends" @@ -42,67 +44,84 @@ const coreDowngradeSource = "sends" // store and its seat's row, and the node-engine, which reads what the controller declares. var downgradeGuarded = []string{lease.ComponentController, lease.ComponentEngine} -// buildAsked is when the newest build of a module from a commit that worked was asked — or recorded, when the -// request time is not known — and false when the records hold none. A variable, so a test reads its own order. -var buildAsked = func(ctx context.Context, inv *inventory.Inventory, module, commit string) (time.Time, bool, error) { +// askedBuild is what the order needs of one build of a module: when it was asked, and whether it failed its gate. +type askedBuild struct { + asked time.Time + failedGate bool +} + +// buildAsked is the newest build of a module from a commit that worked — when it was asked (or recorded, when +// the request time is not known), and whether it failed its gate — and false when the records hold none. A +// variable, so a test reads its own order. +var buildAsked = func(ctx context.Context, inv *inventory.Inventory, module, commit string) (askedBuild, bool, error) { builds, err := inv.Builds(ctx, module, askedLookBack) if err != nil { - return time.Time{}, false, err + return askedBuild{}, false, err } for _, b := range builds { if b.Worked() && sameCommit(b.Commit, commit) { - return b.AskedOrAt(), true, nil + failed, err := inv.GateFailed(ctx, b.ID) + if err != nil { + return askedBuild{}, false, err + } + return askedBuild{asked: b.AskedOrAt(), failedGate: failed}, true, nil } } - return time.Time{}, false, nil + return askedBuild{}, false, nil } // askedLookBack is how many of a module's newest builds are read for when a build was asked. const askedLookBack = 50 -// coreDowngrade is one move of a core module to an older build than its machine runs. +// coreDowngrade is one core module a machine was last sent a build of that is newer than the mesh's record. type coreDowngrade struct { - move inventory.CarriedMove - component string - fromAsked, asked time.Time + module, node, component string + // sent is the build the machine was last sent, record the one the mesh holds. + sent, record string + sentAsked, recordAsked time.Time } func (d coreDowngrade) String() string { - return fmt.Sprintf("%s would be sent %s %s, older than the %s it runs (asked %s, before %s) — no send but a "+ - "gate's own put-back moves the %s back (novox/hq issue 201)", d.move.Node, d.move.Module, short(d.move.To), - short(d.move.From), d.asked.UTC().Format(time.RFC3339), d.fromAsked.UTC().Format(time.RFC3339), d.component) + return fmt.Sprintf("%s was last sent %s %s, and the mesh holds the older %s (asked %s, before %s): a send there "+ + "keeps %s; `push %s` sends the older build, and a newer merge or `rebuild %s` moves the record forward "+ + "(novox/hq issue 201)", d.node, d.module, short(d.sent), short(d.record), + d.recordAsked.UTC().Format(time.RFC3339), d.sentAsked.UTC().Format(time.RFC3339), short(d.sent), d.node, + d.module) } -// coreDowngrades is the moves of a machine's send that put a core module back to a build asked before the one it -// runs, leaving out what a put-back puts back. A move whose either build the records do not hold is not proved -// older, and is left to the gate that judges it. Pure but for asked. -func coreDowngrades(moves []inventory.CarriedMove, scope sendScope, - asked func(module, commit string) (time.Time, bool, error)) ([]coreDowngrade, error) { +// coreDowngrades is, for one machine, each guarded core module whose record is older than the build the machine +// was last sent, leaving out what a gate's own put-back puts back and a sent build that failed its gate. A move +// whose either build the records do not hold is not proved older and is left to the gate that judges it. Pure +// but for asked. +func coreDowngrades(node string, sent map[string]string, record map[string]string, scope sendScope, + asked func(module, commit string) (askedBuild, bool, error)) ([]coreDowngrade, error) { var out []coreDowngrade - for _, mv := range moves { - component := coreComponent(mv.Module) - if !guarded(component) || mv.From == "" || mv.To == "" || sameCommit(mv.From, mv.To) { + for module, was := range sent { + component := coreComponent(module) + now := record[module] + if !guarded(component) || was == "" || now == "" || sameCommit(was, now) { continue } - if scope.puttingBack && scope.modules[mv.Module] { + if scope.puttingBack && scope.modules[module] { continue } - from, known, err := asked(mv.Module, mv.From) - if err != nil || !known { - if err != nil { - return nil, fmt.Errorf("when %s's build %s was asked cannot be read: %w", mv.Module, short(mv.From), err) - } - continue - } - to, known, err := asked(mv.Module, mv.To) + from, known, err := asked(module, was) if err != nil { - return nil, fmt.Errorf("when %s's build %s was asked cannot be read: %w", mv.Module, short(mv.To), err) + return nil, fmt.Errorf("when %s's build %s was asked cannot be read: %w", module, short(was), err) } - if known && to.Before(from) { - out = append(out, coreDowngrade{move: mv, component: component, fromAsked: from, asked: to}) + if !known || from.failedGate { + continue + } + to, known, err := asked(module, now) + if err != nil { + return nil, fmt.Errorf("when %s's build %s was asked cannot be read: %w", module, short(now), err) + } + if known && to.asked.Before(from.asked) { + out = append(out, coreDowngrade{module: module, node: node, component: component, sent: was, record: now, + sentAsked: from.asked, recordAsked: to.asked}) } } - sort.Slice(out, func(i, j int) bool { return out[i].move.Module < out[j].move.Module }) + sort.Slice(out, func(i, j int) bool { return out[i].module < out[j].module }) return out, nil } @@ -115,32 +134,29 @@ func guarded(component string) bool { return false } -// refuseCoreDowngrades says, for one machine's moves, each that would put a core module back, raising its -// condition; and clears the condition of each guarded component that machine's send does not move back. -func refuseCoreDowngrades(ctx context.Context, inv *inventory.Inventory, scope sendScope, node string, - moves []inventory.CarriedMove) ([]string, error) { - found, err := coreDowngrades(moves, scope, func(module, commit string) (time.Time, bool, error) { +// coreKept is, for a send that keeps (sendKeeps), each guarded core module of a machine to keep at the build it +// was last sent, module → commit, each said and raised as a condition. +func coreKept(ctx context.Context, inv *inventory.Inventory, node string, sent map[string]string, + current map[string]inventory.CurrentBuild) (map[string]string, error) { + record := map[string]string{} + for m, c := range current { + record[m] = c.Commit + } + found, err := coreDowngrades(node, sent, record, scopeOf(ctx), func(module, commit string) (askedBuild, bool, error) { return buildAsked(ctx, inv, module, commit) }) - if err != nil { + if err != nil || len(found) == 0 { return nil, err } - var said []string - refused := map[string]bool{} + out := map[string]string{} for _, d := range found { - said = append(said, d.String()) - refused[d.component] = true + out[d.module] = d.sent sayCoreDowngrade(ctx, d) } - for _, c := range downgradeGuarded { - if !refused[c] { - clearCoreDowngrade(ctx, c, node) - } - } - return said, nil + return out, nil } -// coreDowngradeObservation is a refused downgrade as a condition: `core...`. +// coreDowngradeObservation is a kept core build as a condition: `core...`. func coreDowngradeObservation(component, node, summary string) conditions.Observation { return conditions.Observation{Scope: conditions.ScopeCore, ID: component + "." + node, Kind: kindCoreDowngrade, Token: kindCoreDowngrade, Machine: node, Severity: conditions.Warning, Source: coreDowngradeSource, @@ -148,24 +164,49 @@ func coreDowngradeObservation(component, node, summary string) conditions.Observ } func sayCoreDowngrade(ctx context.Context, d coreDowngrade) { - fmt.Printf("NOT SENT: %s\n", d) + fmt.Printf("KEPT: %s\n", d) keeper := conditionsFrom if keeper == nil { return } - if _, err := keeper.Observe(ctx, coreDowngradeObservation(d.component, d.move.Node, d.String())); err != nil { - fmt.Printf("the condition for %s's refused downgrade on %s could not be raised: %v\n", d.move.Module, - d.move.Node, err) + if _, err := keeper.Observe(ctx, coreDowngradeObservation(d.component, d.node, d.String())); err != nil { + fmt.Printf("the condition for %s kept on %s could not be raised: %v\n", d.module, d.node, err) } } -func clearCoreDowngrade(ctx context.Context, component, node string) { +// settleCoreDowngrades clears, after a send to a machine is away, the condition of each guarded core module the +// send carried at the mesh's record — a person's push that accepted the older build, or a send after the record +// moved forward — or did not carry at all. A send that kept the newer build leaves it open. +func settleCoreDowngrades(ctx context.Context, inv *inventory.Inventory, node string, carried map[string]string) { keeper := conditionsFrom - if keeper == nil { + if keeper == nil || carried == nil { return } - key := coreDowngradeObservation(component, node, "").Key() - if _, err := keeper.Clear(ctx, key, "a send to "+node+" carries no older "+component); err != nil { - fmt.Printf("the condition %s could not be cleared: %v\n", key, err) + var open []string + for _, c := range downgradeGuarded { + if _, found, err := keeper.Get(ctx, coreDowngradeObservation(c, node, "").Key()); err == nil && found { + open = append(open, c) + } + } + if len(open) == 0 { + return + } + current, err := inv.CurrentBuilds(ctx) + if err != nil { + fmt.Printf("whether %s still keeps a core build over the mesh's record cannot be read: %v\n", node, err) + return + } + for module, now := range current { + component := coreComponent(module) + if !guarded(component) { + continue + } + if was, has := carried[module]; has && !sameCommit(was, now.Commit) { + continue + } + key := coreDowngradeObservation(component, node, "").Key() + if _, err := keeper.Clear(ctx, key, node+" was sent the "+component+" the mesh holds"); err != nil { + fmt.Printf("the condition %s could not be cleared: %v\n", key, err) + } } } diff --git a/cmd/mesh-controller/issue201_test.go b/cmd/mesh-controller/issue201_test.go index 9ec8f3bd..b601ecda 100644 --- a/cmd/mesh-controller/issue201_test.go +++ b/cmd/mesh-controller/issue201_test.go @@ -3,7 +3,6 @@ package main import ( "context" "encoding/json" - "errors" "strings" "testing" "time" @@ -15,104 +14,76 @@ import ( // novox/hq issue 201: on 2026-10-02 a plan sent the control node a controller older than the one running there, // whose seat row the newer one had already widened; the older binary refused the row and the mesh went quiet for -// ten minutes. No send but a gate's own put-back moves the controller or the node-engine back behind the build its -// machine runs, by the builds' registration order. +// ten minutes. No send but a person's push and a gate's own put-back moves the controller or the node-engine back +// behind the build its machine was last sent: the send keeps it there, and carries the rest. -// askedIn answers when a build was asked from a fixed table: module@commit → when. -func askedIn(table map[string]time.Time) func(module, commit string) (time.Time, bool, error) { - return func(module, commit string) (time.Time, bool, error) { +// askedIn answers a build from a fixed table: module@commit → when it was asked; failed names those that failed +// their gate. +func askedIn(table map[string]time.Time, failed ...string) func(module, commit string) (askedBuild, bool, error) { + return func(module, commit string) (askedBuild, bool, error) { at, known := table[module+"@"+commit] - return at, known, nil + b := askedBuild{asked: at} + for _, f := range failed { + if f == module+"@"+commit { + b.failedGate = true + } + } + return b, known, nil } } -func TestACoreModuleIsNeverMovedBehindTheBuildItsMachineRuns(t *testing.T) { +func TestACoreModuleIsKeptOverAnOlderRecord(t *testing.T) { old, now := t0, t0.Add(time.Minute) - asked := askedIn(map[string]time.Time{ + table := map[string]time.Time{ "mesh-controller@c1": old, "mesh-controller@c2": now, "mesh-host@h1": old, "mesh-host@h2": now, "app@a1": old, "app@a2": now, - }) - back := []inventory.CarriedMove{ - {Module: "mesh-controller", Node: "anchor", From: "c2", To: "c1"}, - {Module: "mesh-host", Node: "anchor", From: "h2", To: "h1"}, - {Module: "app", Node: "anchor", From: "a2", To: "a1"}, } - found, err := coreDowngrades(back, sendScope{judged: map[string]bool{"anchor": true}}, asked) + sent := map[string]string{"mesh-controller": "c2", "mesh-host": "h2", "app": "a2"} + record := map[string]string{"mesh-controller": "c1", "mesh-host": "h1", "app": "a1"} + found, err := coreDowngrades("anchor", sent, record, sendScope{judged: map[string]bool{"anchor": true}}, askedIn(table)) if err != nil { t.Fatal(err) } - if len(found) != 2 || found[0].move.Module != "mesh-controller" || found[0].component != lease.ComponentController || - found[1].move.Module != "mesh-host" || found[1].component != lease.ComponentEngine { - t.Fatalf("the controller and the node-engine sent back, and nothing else, are refused: %+v", found) + if len(found) != 2 || found[0].module != "mesh-controller" || found[0].component != lease.ComponentController || + found[0].sent != "c2" || found[1].module != "mesh-host" || found[1].component != lease.ComponentEngine { + t.Fatalf("the controller and the node-engine over an older record are kept, and nothing else: %+v", found) } - if said := found[0].String(); !strings.Contains(said, "anchor would be sent mesh-controller c1") || - !strings.Contains(said, "issue 201") { - t.Fatalf("the refusal says the machine, the module and the build: %s", said) + if said := found[0].String(); !strings.Contains(said, "anchor was last sent mesh-controller c2") || + !strings.Contains(said, "`push anchor`") || !strings.Contains(said, "`rebuild mesh-controller`") { + t.Fatalf("it says the machine, the build kept and the two ways out: %s", said) } - // Forward, the same build, or a build the records do not hold: not refused. - for _, mv := range []inventory.CarriedMove{ - {Module: "mesh-controller", Node: "anchor", From: "c1", To: "c2"}, - {Module: "mesh-controller", Node: "anchor", From: "c2", To: "c2"}, - {Module: "mesh-controller", Node: "anchor", From: "c2", To: "c9"}, - {Module: "mesh-controller", Node: "anchor", From: "c9", To: "c1"}, - } { - if found, err := coreDowngrades([]inventory.CarriedMove{mv}, sendScope{}, asked); err != nil || len(found) != 0 { - t.Fatalf("%+v is no downgrade proved: %+v %v", mv, found, err) + // The record forward, the same build, or a build the records do not hold: nothing kept. + for _, c := range [][2]string{{"c1", "c2"}, {"c2", "c2"}, {"c2", "c9"}, {"c9", "c1"}} { + found, err := coreDowngrades("anchor", map[string]string{"mesh-controller": c[0]}, + map[string]string{"mesh-controller": c[1]}, sendScope{}, askedIn(table)) + if err != nil || len(found) != 0 { + t.Fatalf("sent %s, record %s: no downgrade proved, nothing kept: %+v %v", c[0], c[1], found, err) } } + // A sent build that failed its gate is never ahead: the record put back before it is sent. + if found, _ := coreDowngrades("anchor", sent, record, sendScope{}, askedIn(table, "mesh-controller@c2")); len(found) != 1 || + found[0].module != "mesh-host" { + t.Fatalf("a controller that failed its gate was kept over the build put back: %+v", found) + } + // A gate's own put-back carries its own modules back, and nothing else. putBack := sendScope{modules: map[string]bool{"mesh-controller": true}, puttingBack: true} - found, err = coreDowngrades(back, putBack, asked) - if err != nil { - t.Fatal(err) - } - if len(found) != 1 || found[0].move.Module != "mesh-host" { + if found, _ := coreDowngrades("anchor", sent, record, putBack, askedIn(table)); len(found) != 1 || found[0].module != "mesh-host" { t.Fatalf("a put-back of the controller carries it back, and not the node-engine: %+v", found) } - // Naming a module is not a put-back. - if found, _ := coreDowngrades(back[:1], sendScope{modules: map[string]bool{"mesh-controller": true}}, asked); len(found) != 1 { + if found, _ := coreDowngrades("anchor", sent, record, sendScope{modules: map[string]bool{"mesh-controller": true}}, + askedIn(table)); len(found) != 2 { t.Fatalf("a send for the controller that is no put-back carries it back: %+v", found) } } -// The refusal is a warning condition on the machine, which the next send there carrying no such move clears. -func TestARefusedCoreDowngradeIsSaidAndClearedByTheNextSend(t *testing.T) { - k, _ := withConditionsInMemory(t) - ctx := t.Context() - was := buildAsked - buildAsked = func(_ context.Context, _ *inventory.Inventory, module, commit string) (time.Time, bool, error) { - return askedIn(map[string]time.Time{"mesh-controller@c1": t0, "mesh-controller@c2": t0.Add(time.Minute)})(module, commit) - } - t.Cleanup(func() { buildAsked = was }) - - said, err := refuseCoreDowngrades(ctx, nil, sendScope{}, "anchor", - []inventory.CarriedMove{{Module: "mesh-controller", Node: "anchor", From: "c2", To: "c1"}}) - if err != nil || len(said) != 1 { - t.Fatalf("the downgrade is refused: %v %v", said, err) - } - key := coreDowngradeObservation(lease.ComponentController, "anchor", "").Key() - c, open, err := k.Get(ctx, key) - if err != nil || !open || !strings.Contains(c.Summary, "mesh-controller c1") { - t.Fatalf("the condition %s is raised, saying the build: %+v %v %v", key, c, open, err) - } - - said, err = refuseCoreDowngrades(ctx, nil, sendScope{}, "anchor", - []inventory.CarriedMove{{Module: "mesh-controller", Node: "anchor", From: "c1", To: "c2"}}) - if err != nil || len(said) != 0 { - t.Fatalf("a move forward is sent: %v %v", said, err) - } - if _, open, err := k.Get(ctx, key); err != nil || open { - t.Fatalf("the condition is cleared by a send carrying no older controller: %v %v", open, err) - } -} - -// TestReplay201 replays the race on a store: the control node runs the newer controller, the mesh's record of -// the controller is the build before it, and a plan's gated send there — the node-engine's plan of 2026-10-02 — -// is refused, said as the plan's words; the gate's own put-back of the controller is not. -func TestReplay201(t *testing.T) { +// core201 is a mesh whose control node, anchor, was sent the newer controller c2 while the mesh's record of the +// controller is the build before it, c1, which passed its gate when it rolled out. +func core201(t *testing.T) *stores { + t.Helper() open := aMesh(t) ctx := t.Context() inv := open.inventory @@ -128,6 +99,10 @@ func TestReplay201(t *testing.T) { return b } before := record("c1aaaaaaaaaa", old) + if err := inv.RecordGate(ctx, inventory.GateVerdict{Build: before.ID, Module: "mesh-controller", Commit: before.Commit, + Machines: []string{"anchor"}, Verdict: inventory.GatePassed}); err != nil { + t.Fatal(err) + } record("c2bbbbbbbbbb", now) if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "mesh-controller", Version: "1"}, inventory.Source{ Repository: "novox/mesh-controller", BuiltFrom: "c2bbbbbbbbbb", Head: "c2bbbbbbbbbb", Asked: now}); err != nil { @@ -139,7 +114,6 @@ func TestReplay201(t *testing.T) { if err := inv.RecordSent(ctx, nodeID(t, open, "anchor"), "d-anchor", map[string]string{"mesh-controller": "c2bbbbbbbbbb"}); err != nil { t.Fatal(err) } - // The mesh's record goes back to the build before, as the record went on 2026-10-02. if err := inv.RestoreModule(ctx, before); err != nil { t.Fatal(err) } @@ -156,14 +130,70 @@ func TestReplay201(t *testing.T) { return f.moves(node, modules, sent, known, all), nil } t.Cleanup(func() { machineMoves = wasMoves }) + return open +} + +// TestReplay201 replays the race on a store: the control node runs the newer controller and the mesh's record is +// the build before it. A plan's send there — the node-engine's plan of 2026-10-02 — goes, keeps the newer +// controller, and says so as a condition; the gate's own put-back carries it back; a send that carries the record, +// as a person's push does, clears the condition. +func TestReplay201(t *testing.T) { + open := core201(t) + ctx := t.Context() + k, _ := withConditionsInMemory(t) judged := withScope(ctx, sendScope{judged: map[string]bool{"anchor": true}}) - if _, err := ungatedIn(judged, open, []string{"anchor"}, ""); !errors.Is(err, errCoreDowngrade) || - !strings.Contains(err.Error(), "mesh-controller c1aaaaaa") { - t.Fatalf("a plan's send of the older controller to the machine running the newer is refused: %v", err) + if kept, err := ungatedIn(judged, open, []string{"anchor"}, ""); err != nil || len(kept) != 1 { + t.Fatalf("a plan's send to the machine is made, the rest of it with it: %v %v", kept, err) } - putBack := withScope(ctx, sendScope{modules: map[string]bool{"mesh-controller": true}, puttingBack: true}) - if kept, err := ungatedIn(putBack, open, []string{"anchor"}, ""); err != nil || len(kept) != 1 { - t.Fatalf("the gate's own put-back of the controller is sent: %v %v", kept, err) + keeps, err := recordedKept(keepingRecorded(judged), open, "anchor") + if err != nil || keeps["mesh-controller"] != "c2bbbbbbbbbb" { + t.Fatalf("the send keeps the newer controller there: %v %v", keeps, err) + } + shelf := map[string]catalogue.Manifest{"mesh-controller": {Module: "mesh-controller", Version: "old"}} + if _, err := keepRecorded(keepingRecorded(judged), open, "anchor", shelf); err != nil || shelf["mesh-controller"].Version != "1" { + t.Fatalf("the send composes the kept build's manifest: %+v %v", shelf, err) + } + key := coreDowngradeObservation(lease.ComponentController, "anchor", "").Key() + if c, found, err := k.Get(ctx, key); err != nil || !found || !strings.Contains(c.Summary, "c2bbbbbb") { + t.Fatalf("the condition %s is raised, saying the build kept: %+v %v %v", key, c, found, err) + } + + putBack := keepingRecorded(withScope(ctx, sendScope{modules: map[string]bool{"mesh-controller": true}, puttingBack: true})) + if keeps, err := recordedKept(putBack, open, "anchor"); err != nil || len(keeps) != 0 { + t.Fatalf("the gate's own put-back of the controller carries it back: %v %v", keeps, err) + } + + settleCoreDowngrades(ctx, open.inventory, "anchor", map[string]string{"mesh-controller": "c2bbbbbbbbbb"}) + if _, found, _ := k.Get(ctx, key); !found { + t.Fatal("a send that kept the newer controller cleared the condition") + } + settleCoreDowngrades(ctx, open.inventory, "anchor", map[string]string{"mesh-controller": "c1aaaaaaaaaa"}) + if _, found, _ := k.Get(ctx, key); found { + t.Fatal("a send carrying the mesh's record — a person's push — leaves the condition open") + } +} + +// The review of #273: a gate failed the newer controller, put the record back, and the put-back's own send was +// refused. The next automatic send — a healer's, which judges nothing — completes the put-back: a build that +// failed its gate is never kept. +func TestARollbackWhoseSendWasRefusedIsCompletedByTheNextSend(t *testing.T) { + open := core201(t) + ctx := t.Context() + k, _ := withConditionsInMemory(t) + if err := open.inventory.RecordGate(ctx, inventory.GateVerdict{Build: "build-mesh-controller-c2bbbbbbbbbb", + Module: "mesh-controller", Commit: "c2bbbbbbbbbb", Previous: "c1aaaaaaaaaa", Machines: []string{"anchor"}, + Verdict: inventory.GateFailed, Rollback: inventory.NotRolledBack}); err != nil { + t.Fatal(err) + } + healer := withScope(ctx, sendScope{}) + if kept, err := ungatedIn(healer, open, []string{"anchor"}, ""); err != nil || len(kept) != 1 { + t.Fatalf("the healer's send is made: %v %v", kept, err) + } + if keeps, err := recordedKept(keepingRecorded(healer), open, "anchor"); err != nil || len(keeps) != 0 { + t.Fatalf("the failed controller was kept over the build put back: %v %v", keeps, err) + } + if open, _ := k.Open(ctx); len(open) != 0 { + t.Fatalf("a completed put-back raised a condition: %+v", open) } } diff --git a/cmd/mesh-controller/plain_words.go b/cmd/mesh-controller/plain_words.go index e2caf2d4..a5f705b1 100644 --- a/cmd/mesh-controller/plain_words.go +++ b/cmd/mesh-controller/plain_words.go @@ -182,7 +182,7 @@ var plainWordings = map[string]func(conditions.Observation) words{ "register it, so it was not made.", module, m), Resolved: "Resolved: " + module + " on " + m + " is settled"} }), - // A send that would have put the controller or the node-engine back behind what a machine runs (novox/hq + // The controller or the node-engine kept at the build a machine was last sent, over an older record (novox/hq // issue 201). kindCoreDowngrade: worded(func(o conditions.Observation) words { m := machineOr(o, "a machine") @@ -190,11 +190,11 @@ var plainWordings = map[string]func(conditions.Observation) words{ if o.Scope != conditions.ScopeCore || !guarded(component) { component = "core component" } - return words{Headline: "An older " + component + " was not sent to " + m, - Needs: "decide which build of the " + component + " the mesh should hold; the details say which.", - Explanation: fmt.Sprintf("A send would have put the mesh's own %s on %s back to an older build than "+ - "the one it runs, so it was not sent and %s keeps its build.", component, m, m), - Resolved: "Resolved: sends to " + m + " carry no older " + component} + return words{Headline: m + " keeps a newer " + component + " than the mesh holds", + Needs: fmt.Sprintf("push %s to take the older build, or a newer merge or rebuild moves the record forward.", m), + Explanation: fmt.Sprintf("The mesh's record of its own %s is older than the build %s was last sent, so "+ + "the mesh's sends keep the newer one there rather than put it back.", component, m), + Resolved: "Resolved: " + m + " runs the " + component + " the mesh holds"} }), kindBindingKept: worded(func(o conditions.Observation) words { m := machineOr(o, "a machine") diff --git a/cmd/mesh-controller/push.go b/cmd/mesh-controller/push.go index 8aef75c2..7ff94186 100644 --- a/cmd/mesh-controller/push.go +++ b/cmd/mesh-controller/push.go @@ -1027,6 +1027,9 @@ func (b overTheBus) declare(ctx context.Context, s readyNode, body []byte) (stri if err != nil { return "", err } + // A send that carried the mesh's record of the controller and the node-engine — a person's push included — + // ends the condition of one kept over it (novox/hq issue 201). + settleCoreDowngrades(ctx, b.open.inventory, s.node, s.declared.Builds) if len(s.declared.Bindings) > 0 { // And where it bound each consumer of a provision that keeps its data (novox/hq ADR 0232): // what the next resolution keeps it at. On the same outliving context as the send's record. diff --git a/cmd/mesh-controller/recorded_kept.go b/cmd/mesh-controller/recorded_kept.go index a8e8cf5b..ecf9e25c 100644 --- a/cmd/mesh-controller/recorded_kept.go +++ b/cmd/mesh-controller/recorded_kept.go @@ -135,6 +135,17 @@ func recordedKept(ctx context.Context, open *stores, node string) (map[string]st } out[m] = was } + // **And the controller and the node-engine at the build the machine was last sent, over an older record** + // (novox/hq issue 201, core_downgrade.go). Under keepingEveryBuild they are kept already. + if !every { + core, err := coreKept(ctx, inv, node, sent, current) + if err != nil { + return nil, err + } + for m, was := range core { + out[m] = was + } + } return out, nil } @@ -159,6 +170,12 @@ func keepRecorded(ctx context.Context, open *stores, node string, shelf map[stri return nil, fmt.Errorf("%s runs %s's build %s, which the build records no longer hold: the bus's user "+ "list cannot be sent there alone with it kept (novox/hq issue 490)", node, m, short(kept[m])) } + if !found && guarded(coreComponent(m)) { + return nil, fmt.Errorf("%s was last sent %s's build %s, newer than the one the mesh holds, and the build records "+ + "no longer hold it: this send cannot keep it and does not move it back — `push %s` sends the older build on "+ + "a person's word, and a newer merge or `rebuild %s` moves the record forward (novox/hq issue 201)", + node, m, short(kept[m]), node, m) + } if !found { return nil, fmt.Errorf("%s records rather than rolls out, and %s runs its build %s, which the build "+ "records no longer hold: this send cannot keep it and does not move it — `push %s` sends the new "+ diff --git a/cmd/mesh-controller/release.go b/cmd/mesh-controller/release.go index 5f23d856..ab21b05a 100644 --- a/cmd/mesh-controller/release.go +++ b/cmd/mesh-controller/release.go @@ -251,47 +251,26 @@ func ungatedIn(ctx context.Context, open *stores, names []string, addedHolder st if err != nil { return nil, err } - var kept, refused, older []string + var kept, refused []string for _, n := range names { - // Every move, gated or not: a move to a build that passed a gate elsewhere is no gate's concern here, and - // may still be a core module's downgrade (novox/hq issue 201). - every, err := machineMoves(ctx, open, f, n, true) + moves, err := machineMoves(ctx, open, f, n, false) if err != nil { return nil, err } - // The grants step composes every module at the build its machine was last sent (novox/hq issue 490): - // a move it keeps is not made. Read, not assumed, so a move it could not keep is still refused here. - var keeps map[string]string - if everyBuildKept(ctx, n) { - if keeps, err = recordedKept(ctx, open, n); err != nil { - return nil, err - } + // The grants step composes every module at the build its machine was last sent (novox/hq issue 490), + // and every send keeps the controller and the node-engine over an older record (issue 201): a move it + // keeps is not made. Read, not assumed, so a move it could not keep is still refused here. + keeping := ctx + if _, on := keptExcept(ctx); !on { + keeping = keepingRecorded(ctx) // as the send composes it (sendKeeps), which is set after this check } - var made []inventory.CarriedMove - for _, mv := range every { - if was, kept := keeps[mv.Module]; kept && sameCommit(was, mv.From) { - continue - } - made = append(made, mv) - } - // **No send but a gate's own put-back moves the controller or the node-engine to an older build than - // the one its machine runs** (novox/hq issue 201). - downgrades, err := refuseCoreDowngrades(ctx, open.inventory, scope, n, made) + keeps, err := recordedKept(keeping, open, n) if err != nil { return nil, err } - if len(downgrades) > 0 { - if n == addedHolder { - fmt.Printf("%s holds the bus and its user list changed, and it is not sent now: %s\n", n, - strings.Join(downgrades, "; ")) - continue - } - older = append(older, downgrades...) - continue - } var waiting []string - for _, mv := range made { - if f.gated(mv.Module, mv.To) { + for _, mv := range moves { + if was, kept := keeps[mv.Module]; kept && sameCommit(was, mv.From) { continue } if !scope.judged[n] && !scope.modules[mv.Module] { @@ -308,9 +287,6 @@ func ungatedIn(ctx context.Context, open *stores, names []string, addedHolder st refused = append(refused, fmt.Sprintf("%s (%s)", n, strings.Join(waiting, ", "))) } } - if len(older) > 0 { - return nil, fmt.Errorf("%w: %s", errCoreDowngrade, strings.Join(older, "; ")) - } if len(refused) > 0 { return nil, fmt.Errorf("%w: %s — a walk sends them, one machine at a time, each judged (novox/hq ADR "+ "0236); `upgrade backlog` lists them", errUngated, strings.Join(refused, "; ")) From f0fd7fe8868e01035286bb9a282c2aa9f30969e8 Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 12 Oct 2026 19:16:26 +0200 Subject: [PATCH 26/26] Leave a kept core move out of what a gate judges (issue 201, review 91) gatedSend took its judged moves from the record, so on a machine kept at a newer controller the gate judged the controller moving back to the record, read its own send's kept controller as a later build there, and was superseded with nothing judged. The kept moves are now left out there as in ungatedIn (withoutKept). The doc says the bus step keeps too, and the three store tests are listed. --- cmd/mesh-controller/core_downgrade.go | 9 ++-- cmd/mesh-controller/issue201_test.go | 60 +++++++++++++++++++++++++++ cmd/mesh-controller/release.go | 44 ++++++++++++++------ testdata/store-tests | 3 ++ 4 files changed, 101 insertions(+), 15 deletions(-) diff --git a/cmd/mesh-controller/core_downgrade.go b/cmd/mesh-controller/core_downgrade.go index e4b9885d..cae82843 100644 --- a/cmd/mesh-controller/core_downgrade.go +++ b/cmd/mesh-controller/core_downgrade.go @@ -11,8 +11,10 @@ import ( "github.com/novox/mesh-controller/internal/lease" ) -// **No send but a person's push and a gate's own put-back moves the mesh's own core to a build older than the -// one its machine was last sent** (novox/hq issue 201). On 2026-10-02 two merges a minute apart made two plans; +// **No send but a person's `push ` and a gate's own put-back moves the mesh's own core to a build older +// than the one its machine was last sent** (novox/hq issue 201). Every send made through sendToEach keeps it — a +// plan's, a release's, a healer's, a rotation's, and the bus step too, which is a person's act but composes under +// sendKeeps like the rest. On 2026-10-02 two merges a minute apart made two plans; // the controller's plan rolled the new controller, which widened its seat's row, and the node-engine's plan then // sent the control node the controller build before it. The older binary read the row it could not serve and the // mesh lost its voice for ten minutes — the thing that would have corrected it was the thing that was down. ADR @@ -21,7 +23,8 @@ import ( // // **Kept, not refused.** The send composes the controller or the node-engine at the build the machine was last // sent, as a recorded module is kept (ADR 0245), and carries everything else: no other module, secret, healer or -// walk waits on it, and nothing is retried. A plan whose own build is the older one finds its gate superseded by +// walk waits on it, and nothing is retried. The kept move is no move for a gate either: ungatedIn and gatedSend +// leave it out (withoutKept), so a gate judges the other moves of its send. A plan whose own build is the older one finds its gate superseded by // the build kept there (issue 352) and ends. The mesh's record stays older, and a warning condition says so until // a send carries the record — a person's `push ` accepts the older build — or the record moves forward: a // newer merge, or `rebuild mesh-controller`. diff --git a/cmd/mesh-controller/issue201_test.go b/cmd/mesh-controller/issue201_test.go index b601ecda..2e9c5c91 100644 --- a/cmd/mesh-controller/issue201_test.go +++ b/cmd/mesh-controller/issue201_test.go @@ -197,3 +197,63 @@ func TestARollbackWhoseSendWasRefusedIsCompletedByTheNextSend(t *testing.T) { t.Fatalf("a completed put-back raised a condition: %+v", open) } } + +// Review 91 of #273: a gated send to a machine whose controller is kept over an older record leaves the kept +// controller out of what its gate judges. Judged as moving back to the record, the gate would read the send's own +// kept controller as another build sent there after it (issue 352) and be superseded, judging nothing. +func TestAGateJudgesTheOtherMovesBesideAKeptCore(t *testing.T) { + open := core201(t) + ctx := t.Context() + withConditionsInMemory(t) + inv := open.inventory + old, now := time.Now().Add(-2*time.Hour), time.Now().Add(-time.Minute) + for _, b := range []struct { + commit string + at time.Time + }{{"a1aaaaaaaaaa", old}, {"a2bbbbbbbbbb", now}} { + manifest, _ := json.Marshal(catalogue.Manifest{Module: "app", Version: "1"}) + if err := inv.RecordBuild(ctx, inventory.Build{ID: "build-app-" + b.commit, Module: "app", Commit: b.commit, + Repository: "novox/mesh-catalog", Path: "modules/app", Manifest: manifest, Asked: b.at, At: b.at, + Made: []inventory.Artifact{{Name: "x", Kind: "bundle", Reference: "sha256:app-" + b.commit}}}); err != nil { + t.Fatal(err) + } + if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "app", Version: "1"}, inventory.Source{ + Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/app", BuiltFrom: b.commit, Head: b.commit, + Asked: b.at}); err != nil { + t.Fatal(err) + } + } + if _, err := inv.Assign(ctx, "anchor", "app"); err != nil { + t.Fatal(err) + } + if err := inv.RecordSent(ctx, nodeID(t, open, "anchor"), "d-anchor-2", + map[string]string{"mesh-controller": "c2bbbbbbbbbb", "app": "a1aaaaaaaaaa"}); err != nil { + t.Fatal(err) + } + // The send as it is made: the controller kept at c2, the app moved to the record. + wasSend := sendRollout + sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) { + return names, open.inventory.RecordSent(ctx, nodeID(t, open, "anchor"), "d-anchor-3", + map[string]string{"mesh-controller": "c2bbbbbbbbbb", "app": "a2bbbbbbbbbb"}) + } + t.Cleanup(func() { sendRollout = wasSend }) + + moves, sent, _, err := gatedSend(ctx, open, "anchor", nil) + if err != nil || len(sent) != 1 { + t.Fatalf("the gated send is made: %v %v", sent, err) + } + if len(moves) != 1 || moves[0].Module != "app" || moves[0].To != "a2bbbbbbbbbb" { + t.Fatalf("the gate judges the app's move and not the kept controller: %+v", moves) + } + builds, _, err := inv.SentBuilds(ctx, "anchor") + if err != nil { + t.Fatal(err) + } + g := &inventory.PlanGate{Carried: moves} + f := gateFacts{commits: judgedCommits(g, nil), sentBuilds: map[string]map[string]string{"anchor": builds}} + for _, m := range []string{"app", "mesh-controller"} { + if why, superseded := f.supersededOn(m, "anchor"); superseded { + t.Fatalf("the gate is superseded by its own send: %s", why) + } + } +} diff --git a/cmd/mesh-controller/release.go b/cmd/mesh-controller/release.go index ab21b05a..a97cf4a7 100644 --- a/cmd/mesh-controller/release.go +++ b/cmd/mesh-controller/release.go @@ -240,6 +240,32 @@ var machineMoves = func(ctx context.Context, open *stores, f moveFacts, node str return f.moves(node, modules, sent, known, all), nil } +// withoutKept is a machine's moves less those its send keeps at the build the machine was last sent, and so does +// not make: the grants step keeps every module (novox/hq issue 490), and every send keeps the controller and the +// node-engine over an older record (issue 201). Read as the send composes it (sendKeeps, set after the checks that +// call this), so a move it could not keep is still counted. +func withoutKept(ctx context.Context, open *stores, node string, moves []inventory.CarriedMove) ([]inventory.CarriedMove, error) { + if len(moves) == 0 { + return moves, nil + } + keeping := ctx + if _, on := keptExcept(ctx); !on { + keeping = keepingRecorded(ctx) + } + keeps, err := recordedKept(keeping, open, node) + if err != nil || len(keeps) == 0 { + return moves, err + } + var out []inventory.CarriedMove + for _, mv := range moves { + if was, kept := keeps[mv.Module]; kept && sameCommit(was, mv.From) { + continue + } + out = append(out, mv) + } + return out, nil +} + // ungatedIn refuses a send whose machines would move a build no gate has seen, outside its scope: the // machines named, and why; the machine holding the bus, added only for its user list, is left instead. func ungatedIn(ctx context.Context, open *stores, names []string, addedHolder string) ([]string, error) { @@ -257,22 +283,11 @@ func ungatedIn(ctx context.Context, open *stores, names []string, addedHolder st if err != nil { return nil, err } - // The grants step composes every module at the build its machine was last sent (novox/hq issue 490), - // and every send keeps the controller and the node-engine over an older record (issue 201): a move it - // keeps is not made. Read, not assumed, so a move it could not keep is still refused here. - keeping := ctx - if _, on := keptExcept(ctx); !on { - keeping = keepingRecorded(ctx) // as the send composes it (sendKeeps), which is set after this check - } - keeps, err := recordedKept(keeping, open, n) - if err != nil { + if moves, err = withoutKept(ctx, open, n, moves); err != nil { return nil, err } var waiting []string for _, mv := range moves { - if was, kept := keeps[mv.Module]; kept && sameCommit(was, mv.From) { - continue - } if !scope.judged[n] && !scope.modules[mv.Module] { waiting = append(waiting, fmt.Sprintf("%s %s → %s", mv.Module, short(mv.From), short(mv.To))) } @@ -314,6 +329,11 @@ func gatedSend(ctx context.Context, open *stores, node string, owns []inventory. if err != nil { return nil, nil, nil, err } + // A move the send keeps is not made, and is no move for the gate to judge (novox/hq issue 201): a core module + // kept over an older record judged as moving back to it would read the gate as superseded by its own send. + if moves, err = withoutKept(ctx, open, node, moves); err != nil { + return nil, nil, nil, err + } own := func(module string) bool { return slices.ContainsFunc(owns, func(o inventory.CarriedMove) bool { return o.Module == module }) } diff --git a/testdata/store-tests b/testdata/store-tests index 0d195b8c..c0d0515a 100644 --- a/testdata/store-tests +++ b/testdata/store-tests @@ -31,3 +31,6 @@ ./cmd/mesh-controller TestADirectoryOwnersKeyIsRefusedThroughAVerb ./cmd/mesh-controller TestTheSendMakesNoValueForAnOutsideSecretNobodyGave ./cmd/mesh-controller TestAQuestionComposesWithoutAnOutsideSecretNobodyGave +./cmd/mesh-controller TestReplay201 +./cmd/mesh-controller TestARollbackWhoseSendWasRefusedIsCompletedByTheNextSend +./cmd/mesh-controller TestAGateJudgesTheOtherMovesBesideAKeptCore