Give every test a bus of its own, at the release the mesh runs (hq ADR 0237)
The live tests reached one shared bus and assert, read and remove the mesh's own objects by their fixed names, so packages run in parallel deleted what each other read and the suite passed only one package at a time; a red suite read as noise. internal/testbus starts a server per test, linked in at the nats-server release go.mod pins, and a test holds that pin to the catalogue's bus image and to the facts snapshot's bus when there is one, so the tests never run a bus the mesh does not. The waiter test read a timing (the most connections held at one look) and now reads the state it means (the fewest held across the wait). make check runs the packages in parallel under the race detector, with a timeout.
This commit is contained in:
jochen
@@ -102,18 +102,18 @@ proxy-image:
|
||||
# The whole gate. Raises a database, runs everything against it, and takes it down again --
|
||||
# including when the tests fail, which is why the teardown is not conditional.
|
||||
#
|
||||
# **One package at a time (-p 1), and it is not about speed.** The live tests reach one bus, and on
|
||||
# it they assert, read and remove the mesh's own objects -- streams and consumers with fixed names,
|
||||
# because those names are the mesh's and a test cannot choose others. Two packages doing that at once
|
||||
# is one deleting a consumer the other is reading through, and the failure lands in whichever test
|
||||
# was reading, as "no response from stream". That reads as a bug in the code under test.
|
||||
# **Packages in parallel, under the race detector, each test on a bus of its own** (internal/testbus).
|
||||
# It was one package at a time against one shared bus, because the live tests assert, read and remove
|
||||
# the mesh's own objects by their fixed names, and two packages at once deleted what the other read; the
|
||||
# suite was red run as Go runs it and read as noise. A bus per test, of the release the mesh runs, made
|
||||
# it the same in any order. The timeout bounds a hang to a failure with a stack, never a stalled gate.
|
||||
check: fmt vet postgres
|
||||
@go test -p 1 ./... ; status=$$? ; $(MAKE) postgres-stop ; exit $$status
|
||||
@go test -race -timeout 15m ./... ; status=$$? ; $(MAKE) postgres-stop ; exit $$status
|
||||
|
||||
# Without a database the live tests skip rather than fail, so this is the honest subset and not
|
||||
# the gate. Serialised for the same reason check is: a bus may be configured even when a store is not.
|
||||
# Without a database the store's tests skip rather than fail, so this is the honest subset and not
|
||||
# the gate.
|
||||
test:
|
||||
go test -p 1 ./...
|
||||
go test -timeout 15m ./...
|
||||
|
||||
vet:
|
||||
go vet ./...
|
||||
|
||||
@@ -2,7 +2,6 @@ package main
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
@@ -12,6 +11,8 @@ import (
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/testbus"
|
||||
)
|
||||
|
||||
// novox/hq issue 208: the bus's objects a declaration implies are asserted whenever one is sent, not
|
||||
@@ -158,10 +159,7 @@ func TestASendThatCannotAssertTheBusSaysSoAndRaisesACondition(t *testing.T) {
|
||||
// Against a real bus, through the send's own grant: a module assigned after start has its consumer
|
||||
// once a declaration is sent, and a holder assigned after start its seat's worker.
|
||||
func TestNatsAModuleAssignedAfterStartGetsItsConsumerAtItsFirstSend(t *testing.T) {
|
||||
url := os.Getenv("MESH_TEST_NATS")
|
||||
if url == "" {
|
||||
t.Skip("MESH_TEST_NATS unset")
|
||||
}
|
||||
url := testbus.URL(t)
|
||||
open := aMesh(t)
|
||||
js, err := broker.Dial(url)
|
||||
if err != nil {
|
||||
|
||||
@@ -3,7 +3,6 @@ package main
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"os"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
@@ -17,6 +16,8 @@ import (
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/testbus"
|
||||
)
|
||||
|
||||
func bytesOf(n int64) *int64 { return &n }
|
||||
@@ -305,7 +306,7 @@ func TestNatsUnassigningIrreplaceableDataRetiresItAndAnEmptyReplacementIsUrgent(
|
||||
}
|
||||
|
||||
conn := onATestBus(t)
|
||||
js, err := broker.Dial(os.Getenv("MESH_TEST_NATS"))
|
||||
js, err := broker.Dial(testbus.URL(t))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
@@ -5,7 +5,6 @@ import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"net"
|
||||
"os"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
@@ -20,6 +19,8 @@ import (
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/testbus"
|
||||
)
|
||||
|
||||
// The self-check (novox/hq to-be 45 §4): a probe that fails raises its condition, one that cannot run
|
||||
@@ -250,10 +251,7 @@ func TestAResolverAnsweringNoSuchNameForIPv6IsWrong(t *testing.T) {
|
||||
// **D6, D7: what the controller defines is what it finds**, and a consumer deleted or a stream
|
||||
// redefined is said — against a real bus, raised by the same derivation the controller starts with.
|
||||
func TestNatsTheBusIsWhatTheControllerDefines(t *testing.T) {
|
||||
url := os.Getenv("MESH_TEST_NATS")
|
||||
if url == "" {
|
||||
t.Skip("MESH_TEST_NATS unset")
|
||||
}
|
||||
url := testbus.URL(t)
|
||||
open := aMesh(t)
|
||||
js, err := broker.Dial(url)
|
||||
if err != nil {
|
||||
@@ -300,10 +298,7 @@ func TestNatsTheBusIsWhatTheControllerDefines(t *testing.T) {
|
||||
|
||||
// **S9 hears the bus**: a consumer that gives up on a message, and one deleted, as the server says.
|
||||
func TestNatsTheBusSaysAConsumerGaveUpAndOneWasDeleted(t *testing.T) {
|
||||
url := os.Getenv("MESH_TEST_NATS")
|
||||
if url == "" {
|
||||
t.Skip("MESH_TEST_NATS unset")
|
||||
}
|
||||
url := testbus.URL(t)
|
||||
conn, err := nats.Connect(url)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
|
||||
@@ -18,6 +18,8 @@ import (
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/testbus"
|
||||
)
|
||||
|
||||
// The test generated from the healer registry (novox/hq to-be 45 §7, ADR 0227 rule 7 "how it is
|
||||
@@ -434,10 +436,7 @@ func TestH4ResetsOnlyWhatTheTableMarksResettable(t *testing.T) {
|
||||
// asserts the bus's objects the way a send does, and D6's next run clears it — the healer never does.
|
||||
// Deleted again within the hour, the budget is spent: the operator's, urgent, and H3 stops.
|
||||
func TestNatsH3AssertsAMissingConsumerAgainAndBrakesAfterItsBudget(t *testing.T) {
|
||||
url := os.Getenv("MESH_TEST_NATS")
|
||||
if url == "" {
|
||||
t.Skip("MESH_TEST_NATS unset")
|
||||
}
|
||||
url := testbus.URL(t)
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
js, err := broker.Dial(url)
|
||||
@@ -530,10 +529,7 @@ func TestNatsH3AssertsAMissingConsumerAgainAndBrakesAfterItsBudget(t *testing.T)
|
||||
// once replayed — is re-made from now by the mesh itself, and the controller's bound subscription still
|
||||
// receives what comes next: a reset that left the controller deaf would be the incident.
|
||||
func TestNatsH4ResetsTheControllersEventsConsumerAndItStillDelivers(t *testing.T) {
|
||||
url := os.Getenv("MESH_TEST_NATS")
|
||||
if url == "" {
|
||||
t.Skip("MESH_TEST_NATS unset")
|
||||
}
|
||||
url := testbus.URL(t)
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
js, err := broker.Dial(url)
|
||||
@@ -619,10 +615,7 @@ func TestNatsH4ResetsTheControllersEventsConsumerAndItStillDelivers(t *testing.T
|
||||
|
||||
// **H1's question reaches the machine**, on the subject its grant lets it hear and nothing else.
|
||||
func TestNatsAskToReportReachesTheMachine(t *testing.T) {
|
||||
url := os.Getenv("MESH_TEST_NATS")
|
||||
if url == "" {
|
||||
t.Skip("MESH_TEST_NATS unset")
|
||||
}
|
||||
url := testbus.URL(t)
|
||||
conn, err := nats.Connect(url)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
|
||||
@@ -3,7 +3,6 @@ package main
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"os"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
@@ -14,6 +13,8 @@ import (
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/lease"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/testbus"
|
||||
)
|
||||
|
||||
// Two controllers at once (novox/hq to-be 45 §6, issue 204; the half of replay R1 that lives here): two
|
||||
@@ -22,15 +23,12 @@ import (
|
||||
// one that lost it acts no more — no declaration composed, no plan and no condition written — the
|
||||
// moment it lost it.
|
||||
//
|
||||
// MESH_TEST_POSTGRES=… MESH_TEST_NATS=nats://127.0.0.1:14222 go test ./cmd/mesh-controller/ -run Controllers
|
||||
// MESH_TEST_POSTGRES=… go test ./cmd/mesh-controller/ -run Controllers (each test on a bus of its own: internal/testbus)
|
||||
|
||||
// aBusForTheLease is the test bus with the controller's lease bucket new.
|
||||
func aBusForTheLease(t *testing.T) (string, jetstream.KeyValue) {
|
||||
t.Helper()
|
||||
url := os.Getenv("MESH_TEST_NATS")
|
||||
if url == "" {
|
||||
t.Skip("MESH_TEST_NATS unset")
|
||||
}
|
||||
url := testbus.URL(t)
|
||||
conn, err := nats.Connect(url)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
|
||||
@@ -4,7 +4,6 @@ import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
@@ -16,15 +15,15 @@ import (
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/testbus"
|
||||
)
|
||||
|
||||
// The build queue, controlled by hand (novox/hq ADR 0219), and the plans that follow it.
|
||||
//
|
||||
// docker run -d --rm --name bq-nats -p 14294:4222 nats:2.10-alpine -js
|
||||
// make postgres PG_PORT=55566 PG_CONTAINER=bq-pg
|
||||
// MESH_TEST_NATS=nats://127.0.0.1:14294 \
|
||||
// MESH_TEST_POSTGRES='postgres://postgres:check@127.0.0.1:55566/postgres?sslmode=disable' \
|
||||
// go test ./cmd/mesh-controller/ -run 'Queue|Cancel|Clear|Retry|Rebuild|Replay|Paused'
|
||||
// go test ./cmd/mesh-controller/ -run 'Queue|Cancel|Clear|Retry|Rebuild|Replay|Paused' (each test on a bus of its own)
|
||||
|
||||
// asksRecorded makes every ask a plan makes return the next id in a row, and says which were asked.
|
||||
func asksRecorded(t *testing.T) *[]string {
|
||||
@@ -357,10 +356,7 @@ func TestTheQueueVerbsRunTheirCommands(t *testing.T) {
|
||||
// pointed at it, and a function that asks the seat one build.
|
||||
func aBuildQueue(t *testing.T) (*broker.JetStream, func(id, repository string) link.BuildRequest) {
|
||||
t.Helper()
|
||||
url := os.Getenv("MESH_TEST_NATS")
|
||||
if url == "" {
|
||||
t.Skip("MESH_TEST_NATS unset")
|
||||
}
|
||||
url := testbus.URL(t)
|
||||
t.Setenv(broker.NATSVar, url)
|
||||
js, err := broker.Dial(url)
|
||||
if err != nil {
|
||||
|
||||
@@ -4,7 +4,6 @@ import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"flag"
|
||||
"os"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
@@ -18,6 +17,8 @@ import (
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/testbus"
|
||||
)
|
||||
|
||||
// A consumer the mesh stops asking for is retired, not withdrawn, and deleted only by a person
|
||||
@@ -261,10 +262,7 @@ func (f *fakeProvider) serve(t *testing.T, conn *nats.Conn, module, node string)
|
||||
|
||||
func onATestBus(t *testing.T) *nats.Conn {
|
||||
t.Helper()
|
||||
url := os.Getenv("MESH_TEST_NATS")
|
||||
if url == "" {
|
||||
t.Skip("MESH_TEST_NATS unset")
|
||||
}
|
||||
url := testbus.URL(t)
|
||||
js, err := broker.Dial(url)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
|
||||
@@ -11,15 +11,22 @@ require (
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op // indirect
|
||||
github.com/google/go-tpm v0.9.8 // indirect
|
||||
github.com/jackc/pgpassfile v1.0.0 // indirect
|
||||
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
|
||||
github.com/jackc/puddle/v2 v2.2.2 // indirect
|
||||
github.com/klauspost/compress v1.20.0 // indirect
|
||||
github.com/minio/highwayhash v1.0.4 // indirect
|
||||
github.com/nats-io/jwt/v2 v2.8.1 // indirect
|
||||
github.com/nats-io/nats-server/v2 v2.11.17 // indirect
|
||||
github.com/nats-io/nkeys v0.4.16 // indirect
|
||||
github.com/nats-io/nuid v1.0.1 // indirect
|
||||
go.uber.org/automaxprocs v1.6.0 // indirect
|
||||
golang.org/x/sync v0.23.0 // indirect
|
||||
golang.org/x/sys v0.48.0 // indirect
|
||||
golang.org/x/text v0.42.0 // indirect
|
||||
golang.org/x/time v0.15.0 // indirect
|
||||
)
|
||||
|
||||
// The node-engine's own validator (mesh-host/validate, novox/hq to-be 45 D1): one validator, the host's.
|
||||
|
||||
@@ -1,8 +1,12 @@
|
||||
git.novox.be/novox/mesh-host v0.0.0-20261006095519-3e80b7ae325e h1:g9h4QRaAMg5yaJLwqtb0FoOs23DVGUYpW6qvnQ3oY5A=
|
||||
git.novox.be/novox/mesh-host v0.0.0-20261006095519-3e80b7ae325e/go.mod h1:VlilMCRZ5yyNXg7SNigNBLr0Gt32jrGw5KSNq5JAVYs=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
|
||||
github.com/google/go-tpm v0.9.8/go.mod h1:h9jEsEECg7gtLis0upRBQU+GhYVH6jMjrFxI8u6bVUY=
|
||||
github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM=
|
||||
github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg=
|
||||
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo=
|
||||
@@ -13,6 +17,12 @@ github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo
|
||||
github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4=
|
||||
github.com/klauspost/compress v1.20.0 h1:a3C1ke2ohxFymNlb2HWAHjDeKCI90scRskErZkR0ezA=
|
||||
github.com/klauspost/compress v1.20.0/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI=
|
||||
github.com/minio/highwayhash v1.0.4 h1:asJizugGgchQod2ja9NJlGOWq4s7KsAWr5XUc9Clgl4=
|
||||
github.com/minio/highwayhash v1.0.4/go.mod h1:GGYsuwP/fPD6Y9hMiXuapVvlIUEhFhMTh0rxU3ik1LQ=
|
||||
github.com/nats-io/jwt/v2 v2.8.1 h1:V0xpGuD/N8Mi+fQNDynXohVvp7ZztevW5io8CUWlPmU=
|
||||
github.com/nats-io/jwt/v2 v2.8.1/go.mod h1:nWnOEEiVMiKHQpnAy4eXlizVEtSfzacZ1Q43LIRavZg=
|
||||
github.com/nats-io/nats-server/v2 v2.11.17 h1:GKEghcFK6A+aFx11Yf1LjgLC3txAwvyhnYzhBIQZA8I=
|
||||
github.com/nats-io/nats-server/v2 v2.11.17/go.mod h1:B1sFVz4StNosQ903ak4N1G01Fl/9f8e06mXpFIE2K24=
|
||||
github.com/nats-io/nats.go v1.54.0 h1:vsXoOxjHp/GmPUN+EcI7uOf/uB+iAP+kEsAFNQN0yzA=
|
||||
github.com/nats-io/nats.go v1.54.0/go.mod h1:y+DZoD1oBOYfZTU681eTUiUjI0vbqYGixNVFHcjHJ0k=
|
||||
github.com/nats-io/nkeys v0.4.16 h1:rd5oAuLOb8mnAycB0xleuEBNS1pVVnN0fv/FF34Eypg=
|
||||
@@ -26,16 +36,21 @@ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UV
|
||||
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
|
||||
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
|
||||
go.uber.org/automaxprocs v1.6.0 h1:O3y2/QNTOdbF+e/dpXNNW7Rx2hZ4sTIPyybbxyNqTUs=
|
||||
go.uber.org/automaxprocs v1.6.0/go.mod h1:ifeIMSnPZuznNm6jmdzmU3/bfk01Fe2fotchwEFJ8r8=
|
||||
golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M=
|
||||
golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA=
|
||||
golang.org/x/net v0.58.0 h1:ynWG7rqYi4ccpTEuPZ2QGWHktVEM9DMCj9yzDE0Q7To=
|
||||
golang.org/x/net v0.58.0/go.mod h1:YwCddHnFlT7eLQqVprV19OnhLGtc5xOKgE0RyqgfWAU=
|
||||
golang.org/x/sync v0.23.0 h1:KameEIfc1IkluZyXWLn39Wd4tURc6GbCiISGiZm2bQk=
|
||||
golang.org/x/sync v0.23.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0=
|
||||
golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo=
|
||||
golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
|
||||
golang.org/x/text v0.42.0 h1:JbOZXgfeCPU9gacVtYliJqOhD+zhrEqK4LfdpmlUZqI=
|
||||
golang.org/x/text v0.42.0/go.mod h1:ojzP1Z+2QtioaF8DTtO8K5q7JWVVYwZKenzujK0Zd0E=
|
||||
golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U=
|
||||
golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||
|
||||
@@ -2,24 +2,21 @@ package broker
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/testbus"
|
||||
)
|
||||
|
||||
// A consumer that reacts to announcements, made on a stream that keeps a week of them, starts from now:
|
||||
// made from the start it replays every merge and build of that week (novox/hq issue 248). And a reset
|
||||
// re-makes a stuck one from now, its configuration otherwise kept, and refuses a work queue.
|
||||
//
|
||||
// docker run -d --rm --name t -p 14231:4222 nats:2.10-alpine -js
|
||||
// MESH_TEST_NATS=nats://127.0.0.1:14231 go test ./internal/broker/ -run TestAConsumerMadeFromNow
|
||||
// go test ./internal/broker/ -run TestAConsumerMadeFromNow (each test on a bus of its own: internal/testbus)
|
||||
func TestAConsumerMadeFromNowNeverReplaysTheStreamsHistory(t *testing.T) {
|
||||
url := os.Getenv("MESH_TEST_NATS")
|
||||
if url == "" {
|
||||
t.Skip("MESH_TEST_NATS unset")
|
||||
}
|
||||
url := testbus.URL(t)
|
||||
js, err := Dial(url)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
|
||||
@@ -2,12 +2,13 @@ package broker
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/testbus"
|
||||
)
|
||||
|
||||
const twoSeconds = 2 * time.Second
|
||||
@@ -17,13 +18,9 @@ const twoSeconds = 2 * time.Second
|
||||
// so bringing one to match must replace it — and must not replay what it already acknowledged
|
||||
// (novox/hq 04-ISSUES/156).
|
||||
//
|
||||
// docker run -d --rm --name t -p 14231:4222 nats:2.10-alpine -js
|
||||
// MESH_TEST_NATS=nats://127.0.0.1:14231 go test ./internal/broker/ -run TestUpgrading
|
||||
// go test ./internal/broker/ -run TestUpgrading (each test on a bus of its own: internal/testbus)
|
||||
func TestUpgradingAConsumerWhoseDeliverySubjectMoved(t *testing.T) {
|
||||
url := os.Getenv("MESH_TEST_NATS")
|
||||
if url == "" {
|
||||
t.Skip("MESH_TEST_NATS unset")
|
||||
}
|
||||
url := testbus.URL(t)
|
||||
js, err := Dial(url)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -133,10 +130,7 @@ func TestUpgradingAConsumerWhoseDeliverySubjectMoved(t *testing.T) {
|
||||
// not undo. The controller's own two consumers are in exactly this position: it asserts them before
|
||||
// it subscribes.
|
||||
func TestAConsumerNothingIsBoundToDoesMove(t *testing.T) {
|
||||
url := os.Getenv("MESH_TEST_NATS")
|
||||
if url == "" {
|
||||
t.Skip("MESH_TEST_NATS unset")
|
||||
}
|
||||
url := testbus.URL(t)
|
||||
js, err := Dial(url)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
|
||||
@@ -1,8 +1,9 @@
|
||||
package broker
|
||||
|
||||
import (
|
||||
"os"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/testbus"
|
||||
)
|
||||
|
||||
// Against a real server, because the questions here are all "does the server accept this" —
|
||||
@@ -10,13 +11,9 @@ import (
|
||||
//
|
||||
// Skipped unless MESH_TEST_NATS names one, so the ordinary suite stays fast and offline:
|
||||
//
|
||||
// docker run -d --rm --name t -p 14222:4222 nats:2.10-alpine -js
|
||||
// MESH_TEST_NATS=nats://127.0.0.1:14222 go test ./internal/broker/ -run TestAgainstARealServer
|
||||
// go test ./internal/broker/ -run TestAgainstARealServer (each test on a bus of its own: internal/testbus)
|
||||
func TestAgainstARealServer(t *testing.T) {
|
||||
url := os.Getenv("MESH_TEST_NATS")
|
||||
if url == "" {
|
||||
t.Skip("MESH_TEST_NATS unset")
|
||||
}
|
||||
url := testbus.URL(t)
|
||||
js, err := Dial(url)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
|
||||
@@ -1,10 +1,11 @@
|
||||
package broker
|
||||
|
||||
import (
|
||||
"os"
|
||||
"testing"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/testbus"
|
||||
)
|
||||
|
||||
// Raising the bus's objects against a real server.
|
||||
@@ -14,15 +15,11 @@ import (
|
||||
// could answer: that asserting twice changes nothing, and that a consumer really is bound to the one
|
||||
// subject its node is allowed to read.
|
||||
//
|
||||
// docker run -d --rm --name t -p 14227:4222 nats:2.10-alpine -js
|
||||
// MESH_TEST_NATS=nats://127.0.0.1:14227 go test ./internal/broker/ -run TestRaising
|
||||
// go test ./internal/broker/ -run TestRaising (each test on a bus of its own: internal/testbus)
|
||||
|
||||
func aLiveBus(t *testing.T) *JetStream {
|
||||
t.Helper()
|
||||
url := os.Getenv("MESH_TEST_NATS")
|
||||
if url == "" {
|
||||
t.Skip("MESH_TEST_NATS unset")
|
||||
}
|
||||
url := testbus.URL(t)
|
||||
js, err := Dial(url)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
|
||||
@@ -1,11 +1,12 @@
|
||||
package broker
|
||||
|
||||
import (
|
||||
"os"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/testbus"
|
||||
)
|
||||
|
||||
// A seat's worker that changed from push to pull delivery strands a holder built for the new shape
|
||||
@@ -14,13 +15,9 @@ import (
|
||||
// shape, type included: on a work queue it re-makes one of the wrong type, losing nothing, and a
|
||||
// pull subscription then binds and takes what was pending.
|
||||
//
|
||||
// docker run -d --rm --name t -p 14231:4222 nats:2.10-alpine -js
|
||||
// MESH_TEST_NATS=nats://127.0.0.1:14231 go test ./internal/broker/ -run TestAWorker
|
||||
// go test ./internal/broker/ -run TestAWorker (each test on a bus of its own: internal/testbus)
|
||||
func TestAWorkerOfTheWrongTypeIsRemadeOnAWorkQueueAndAPullThenBinds(t *testing.T) {
|
||||
url := os.Getenv("MESH_TEST_NATS")
|
||||
if url == "" {
|
||||
t.Skip("MESH_TEST_NATS unset")
|
||||
}
|
||||
url := testbus.URL(t)
|
||||
js, err := Dial(url)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -112,10 +109,7 @@ func TestAWorkerOfTheWrongTypeIsRemadeOnAWorkQueueAndAPullThenBinds(t *testing.T
|
||||
// re-making it from the start would replay what it acknowledged (novox/hq issue 156), and leaving it
|
||||
// for a hand re-made it exactly that way on 2026-10-03 (issue 207).
|
||||
func TestAWorkerOfTheWrongTypeOnAHistoryStreamIsRemadeFromNowOn(t *testing.T) {
|
||||
url := os.Getenv("MESH_TEST_NATS")
|
||||
if url == "" {
|
||||
t.Skip("MESH_TEST_NATS unset")
|
||||
}
|
||||
url := testbus.URL(t)
|
||||
js, err := Dial(url)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
|
||||
@@ -2,13 +2,14 @@ package conditions
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/testbus"
|
||||
)
|
||||
|
||||
// The condition store against a real server: compare-and-set, an unreadable store refused, and the
|
||||
@@ -16,10 +17,7 @@ import (
|
||||
|
||||
func busStoreForTest(t *testing.T) (*broker.JetStream, Backend, History) {
|
||||
t.Helper()
|
||||
url := os.Getenv("MESH_TEST_NATS")
|
||||
if url == "" {
|
||||
t.Skip("MESH_TEST_NATS unset")
|
||||
}
|
||||
url := testbus.URL(t)
|
||||
js, err := broker.Dial(url)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
|
||||
@@ -69,16 +69,19 @@ func TestAWaiterPinsNoConnectionAndGivesUp(t *testing.T) {
|
||||
done <- err
|
||||
}()
|
||||
}
|
||||
// While they wait, the pool lends nothing to them for longer than a look.
|
||||
pinned := 0
|
||||
for range 10 {
|
||||
time.Sleep(60 * time.Millisecond)
|
||||
if n := int(pool.Stat().AcquiredConns() - base); n > pinned {
|
||||
pinned = n
|
||||
// While they wait, the pool lends nothing to them for longer than a look: **pinned is held throughout**,
|
||||
// so at some moment of the wait fewer than all of them hold one. Asked of the least seen over many
|
||||
// looks, not of the most: a look that lands while every waiter is mid-query is a busy machine, not a
|
||||
// pinned connection, and on a loaded one (the suite's packages at once, the race detector) it did land.
|
||||
least := waiters
|
||||
for range 100 {
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
if n := int(pool.Stat().AcquiredConns() - base); n < least {
|
||||
least = n
|
||||
}
|
||||
}
|
||||
if pinned >= waiters {
|
||||
t.Fatalf("%d connections were held by %d waiters", pinned, waiters)
|
||||
if least >= waiters {
|
||||
t.Fatalf("%d waiters held %d connections at every look across their wait", waiters, least)
|
||||
}
|
||||
for range waiters {
|
||||
if err := <-done; !errors.Is(err, ErrNodeBusy) || !strings.Contains(err.Error(), "anchor") {
|
||||
|
||||
@@ -0,0 +1,49 @@
|
||||
package inventory
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// **The store the tests run is the store the mesh runs** (novox/hq ADR 0227 rule 9, ADR 0237). In a merge
|
||||
// check the build seat raises the store of the major release the facts snapshot says the mesh's store
|
||||
// runs; this holds the two equal there, so a test passing here is a test passing on that release. Without
|
||||
// a snapshot — on a workstation — it says what it ran against and compares nothing.
|
||||
func TestTheStoreTheTestsRunIsTheStoreTheMeshRuns(t *testing.T) {
|
||||
inv := ForTest(t)
|
||||
ran, err := inv.ServerVersionForTest(t)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
path := os.Getenv("MESH_FACTS")
|
||||
if path == "" {
|
||||
t.Logf("the tests ran the store at %s; with no MESH_FACTS, nothing to compare it with", ran)
|
||||
return
|
||||
}
|
||||
body, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var f struct {
|
||||
Versions struct {
|
||||
Store string `json:"store"`
|
||||
} `json:"versions"`
|
||||
}
|
||||
if err := json.Unmarshal(body, &f); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
major := func(v string) string { m, _, _ := strings.Cut(strings.TrimSpace(v), "."); return m }
|
||||
if f.Versions.Store != "" && major(ran) != major(f.Versions.Store) {
|
||||
t.Errorf("the tests ran the store at %s and the mesh's store runs %s", ran, f.Versions.Store)
|
||||
}
|
||||
}
|
||||
|
||||
// ServerVersionForTest is the store's own word for its release.
|
||||
func (i *Inventory) ServerVersionForTest(t *testing.T) (string, error) {
|
||||
t.Helper()
|
||||
var v string
|
||||
err := i.store.Pool().QueryRow(t.Context(), `show server_version`).Scan(&v)
|
||||
return v, err
|
||||
}
|
||||
@@ -4,13 +4,14 @@ import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/testbus"
|
||||
)
|
||||
|
||||
// Two controllers at once, on a real bus (novox/hq to-be 45 §6, replay R1's half that lives here):
|
||||
@@ -18,7 +19,7 @@ import (
|
||||
// stops renewing is taken over once its key expires, and has stopped acting before then; a renewal
|
||||
// refused is a loss at once; and a bucket raised again from nothing never issues an epoch twice.
|
||||
//
|
||||
// MESH_TEST_NATS=nats://127.0.0.1:14222 go test ./internal/lease/
|
||||
// go test ./internal/lease/ (each test on a bus of its own: internal/testbus)
|
||||
|
||||
// The bounds in these tests are the design's divided by five, so the cases run in seconds.
|
||||
const (
|
||||
@@ -30,10 +31,7 @@ const (
|
||||
// aBucket is a fresh lease bucket on the test bus, with the test's age.
|
||||
func aBucket(t *testing.T) (jetstream.JetStream, string) {
|
||||
t.Helper()
|
||||
url := os.Getenv("MESH_TEST_NATS")
|
||||
if url == "" {
|
||||
t.Skip("MESH_TEST_NATS unset")
|
||||
}
|
||||
url := testbus.URL(t)
|
||||
conn, err := nats.Connect(url)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -55,7 +53,7 @@ func aBucket(t *testing.T) (jetstream.JetStream, string) {
|
||||
// aController is one controller instance's lease over the bucket, on a connection of its own.
|
||||
func aController(t *testing.T, bucket, name string, floor func(context.Context) (uint64, error)) *Lease {
|
||||
t.Helper()
|
||||
conn, err := nats.Connect(os.Getenv("MESH_TEST_NATS"))
|
||||
conn, err := nats.Connect(testbus.URL(t))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -275,10 +273,7 @@ func TestAnEpochIsNeverIssuedTwiceWhenTheBucketStartsOver(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestALeaseBucketWithoutAnAgeIsRefused(t *testing.T) {
|
||||
url := os.Getenv("MESH_TEST_NATS")
|
||||
if url == "" {
|
||||
t.Skip("MESH_TEST_NATS unset")
|
||||
}
|
||||
url := testbus.URL(t)
|
||||
conn, err := nats.Connect(url)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
|
||||
@@ -8,6 +8,8 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/testbus"
|
||||
)
|
||||
|
||||
// **A reader's own consumer gone is not a consumer lost**: every bucket watch and stream read-back
|
||||
@@ -64,10 +66,7 @@ func TestNatsARefusedQuestionIsSaidAtOnce(t *testing.T) {
|
||||
|
||||
// The refusal reaches whoever waits on that subject, and only them.
|
||||
func TestNatsARefusalReachesItsWaiter(t *testing.T) {
|
||||
url := os.Getenv("MESH_TEST_NATS")
|
||||
if url == "" {
|
||||
t.Skip("MESH_TEST_NATS unset")
|
||||
}
|
||||
url := testbus.URL(t)
|
||||
conn, err := nats.Connect(url)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
|
||||
@@ -5,7 +5,6 @@ import (
|
||||
"encoding/json"
|
||||
"io"
|
||||
"log"
|
||||
"os"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
@@ -13,6 +12,8 @@ import (
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/testbus"
|
||||
)
|
||||
|
||||
// A build, end to end, against a real server.
|
||||
@@ -22,15 +23,11 @@ import (
|
||||
// catalogue places it. On the old bus that fan-out came from a shared exchange, and it would be easy
|
||||
// to write a version where only the asker hears it and nobody notices for weeks.
|
||||
//
|
||||
// docker run -d --rm --name t -p 14230:4222 nats:2.10-alpine -js
|
||||
// MESH_TEST_NATS=nats://127.0.0.1:14230 go test ./internal/link/ -run TestNatsABuild
|
||||
// go test ./internal/link/ -run TestNatsABuild (each test on a bus of its own: internal/testbus)
|
||||
|
||||
func aBusWithTheBuildRole(t *testing.T) *broker.JetStream {
|
||||
t.Helper()
|
||||
url := os.Getenv("MESH_TEST_NATS")
|
||||
if url == "" {
|
||||
t.Skip("MESH_TEST_NATS unset")
|
||||
}
|
||||
url := testbus.URL(t)
|
||||
js, err := broker.Dial(url)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -349,7 +346,7 @@ func TestNatsAMachineOnTheRetiredBuildRoleTakesThatRolesAsks(t *testing.T) {
|
||||
})
|
||||
}()
|
||||
|
||||
asker, err := BuildsOverNATSOn(os.Getenv("MESH_TEST_NATS"), TheBuildMachineBefore)
|
||||
asker, err := BuildsOverNATSOn(testbus.URL(t), TheBuildMachineBefore)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
@@ -3,12 +3,13 @@ package link
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/testbus"
|
||||
)
|
||||
|
||||
// **Both implementations, one fixture.** The point of the seam is not that the transport can be
|
||||
@@ -17,13 +18,9 @@ import (
|
||||
//
|
||||
// Against a real server, because what the fixture pins is what reaches the wire:
|
||||
//
|
||||
// docker run -d --rm --name t -p 14222:4222 nats:2.10-alpine -js
|
||||
// MESH_TEST_NATS=nats://127.0.0.1:14222 go test ./internal/link/ -run TestTheNatsBus
|
||||
// go test ./internal/link/ -run TestTheNatsBus (each test on a bus of its own: internal/testbus)
|
||||
func TestTheNatsBusEmitsTheEnvelopeTheFixturePins(t *testing.T) {
|
||||
url := os.Getenv("MESH_TEST_NATS")
|
||||
if url == "" {
|
||||
t.Skip("MESH_TEST_NATS unset")
|
||||
}
|
||||
url := testbus.URL(t)
|
||||
f := loadFixture(t, "events/module-event.json")
|
||||
|
||||
conn, err := nats.Connect(url)
|
||||
@@ -99,10 +96,7 @@ func TestAnEventsSubjectIsDerivedFromItsSource(t *testing.T) {
|
||||
// tool is slow" is the first thing a person asking wants, and waiting out the timeout to say it
|
||||
// is how a fast answer becomes a slow non-answer.
|
||||
func TestAskingAToolNobodyServesFailsAtOnce(t *testing.T) {
|
||||
url := os.Getenv("MESH_TEST_NATS")
|
||||
if url == "" {
|
||||
t.Skip("MESH_TEST_NATS unset")
|
||||
}
|
||||
url := testbus.URL(t)
|
||||
conn, err := nats.Connect(url)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -125,10 +119,7 @@ func TestAskingAToolNobodyServesFailsAtOnce(t *testing.T) {
|
||||
|
||||
// And a served tool answers, with no reply queue to declare and no correlation to check.
|
||||
func TestAskingAServedToolAnswers(t *testing.T) {
|
||||
url := os.Getenv("MESH_TEST_NATS")
|
||||
if url == "" {
|
||||
t.Skip("MESH_TEST_NATS unset")
|
||||
}
|
||||
url := testbus.URL(t)
|
||||
conn, err := nats.Connect(url)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
|
||||
@@ -1,11 +1,12 @@
|
||||
package link
|
||||
|
||||
import (
|
||||
"os"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/testbus"
|
||||
)
|
||||
|
||||
// **Verified 2026-09-27**: the caller asked for a reply to `_INBOX.LCr3M83q…` and the consumer
|
||||
@@ -18,13 +19,9 @@ import (
|
||||
// enrolment design rests on it — the reply subject travels in the payload instead — so it is
|
||||
// checked rather than believed.
|
||||
//
|
||||
// docker run -d --rm --name t -p 14222:4222 nats:2.10-alpine -js
|
||||
// MESH_TEST_NATS=nats://127.0.0.1:14222 go test ./internal/link/ -run TestAReplyAddress
|
||||
// go test ./internal/link/ -run TestAReplyAddress (each test on a bus of its own: internal/testbus)
|
||||
func TestAReplyAddressDoesNotSurviveAStream(t *testing.T) {
|
||||
url := os.Getenv("MESH_TEST_NATS")
|
||||
if url == "" {
|
||||
t.Skip("MESH_TEST_NATS unset")
|
||||
}
|
||||
url := testbus.URL(t)
|
||||
conn, err := nats.Connect(url)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
|
||||
@@ -6,7 +6,6 @@ import (
|
||||
"errors"
|
||||
"io"
|
||||
"log"
|
||||
"os"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
@@ -14,6 +13,8 @@ import (
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/testbus"
|
||||
)
|
||||
|
||||
// The consume side against a real server, because what is being checked is what the server does.
|
||||
@@ -23,15 +24,11 @@ import (
|
||||
// to an address carried in the payload reaches a caller waiting on its own inbox. Each is a claim
|
||||
// about a server, so each is asked of one:
|
||||
//
|
||||
// docker run -d --rm --name t -p 14222:4222 nats:2.10-alpine -js
|
||||
// MESH_TEST_NATS=nats://127.0.0.1:14222 go test ./internal/link/ -run TestNats
|
||||
// go test ./internal/link/ -run TestNats (each test on a bus of its own: internal/testbus)
|
||||
|
||||
func aBus(t *testing.T) *broker.JetStream {
|
||||
t.Helper()
|
||||
url := os.Getenv("MESH_TEST_NATS")
|
||||
if url == "" {
|
||||
t.Skip("MESH_TEST_NATS unset")
|
||||
}
|
||||
url := testbus.URL(t)
|
||||
js, err := broker.Dial(url)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
|
||||
@@ -3,11 +3,12 @@ package link
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"os"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/testbus"
|
||||
)
|
||||
|
||||
// novox/hq issue 213: while a machine hands its controller over, the new controller (the process)
|
||||
@@ -29,7 +30,7 @@ func TestNatsASecondControllerStandsByAndTakesOverWhenTheFirstLetsGo(t *testing.
|
||||
})
|
||||
|
||||
// The new one, on a connection of its own as the process would have.
|
||||
second, err := broker.Dial(os.Getenv("MESH_TEST_NATS"))
|
||||
second, err := broker.Dial(testbus.URL(t))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,77 @@
|
||||
// Package testbus is the bus a test runs against: a server of its own, of the release the mesh runs,
|
||||
// started for the one test and gone after it.
|
||||
//
|
||||
// **Why one per test.** The live tests reached one shared bus, and on it they assert, read and remove the
|
||||
// mesh's own objects — streams and consumers with fixed names, because those names are the mesh's. Two
|
||||
// packages at once was one deleting a consumer the other was reading through, and a test left a stream
|
||||
// behind for the next to trip on: the suite passed one package at a time and failed or hung run as Go
|
||||
// runs it, which taught everybody to read a red suite as noise. A server per test makes every test start
|
||||
// from nothing and leave nothing, so the suite is the same run in any order, at any parallelism, under
|
||||
// the race detector, on a workstation or on the build seat.
|
||||
//
|
||||
// **Why this release.** The server is linked in at the version go.mod pins, and a test beside this one
|
||||
// holds that pin to the release the mesh runs — the catalogue's bus image, and the facts snapshot's when
|
||||
// a merge check has one (novox/hq ADR 0227 rule 9: a version the mesh runs is the one its tests run).
|
||||
// A bus the 2.10 line skipped messages on (issue 266) would have been caught by the tests that ran on it,
|
||||
// had they run on it.
|
||||
//
|
||||
// A person may still point a run at a bus of their own — another release, a lab's — with
|
||||
// MESH_TEST_NATS_EXTERNAL=1 and MESH_TEST_NATS; then the run is theirs to serialise.
|
||||
package testbus
|
||||
|
||||
import (
|
||||
"os"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats-server/v2/server"
|
||||
)
|
||||
|
||||
// Version is the release of the server the tests run.
|
||||
const Version = server.VERSION
|
||||
|
||||
// URL is the bus of this test alone: started at its first ask, the same one at every ask after — a
|
||||
// test that dials twice, or hands the address to the code it tests, reaches one bus — and shut down
|
||||
// when the test ends.
|
||||
func URL(t testing.TB) string {
|
||||
t.Helper()
|
||||
if os.Getenv("MESH_TEST_NATS_EXTERNAL") == "1" {
|
||||
if url := os.Getenv("MESH_TEST_NATS"); url != "" {
|
||||
return url
|
||||
}
|
||||
t.Fatal("MESH_TEST_NATS_EXTERNAL=1 and MESH_TEST_NATS names no bus")
|
||||
}
|
||||
if url, ok := buses.Load(t); ok {
|
||||
return url.(string)
|
||||
}
|
||||
url := Start(t).ClientURL()
|
||||
buses.Store(t, url)
|
||||
t.Cleanup(func() { buses.Delete(t) })
|
||||
return url
|
||||
}
|
||||
|
||||
// buses are the running tests' buses, by test.
|
||||
var buses sync.Map
|
||||
|
||||
// Start is a server with JetStream, for this test alone: on loopback, on a port the system chooses, its
|
||||
// store in the test's own directory.
|
||||
func Start(t testing.TB) *server.Server {
|
||||
t.Helper()
|
||||
opts := &server.Options{Host: "127.0.0.1", Port: server.RANDOM_PORT, JetStream: true, StoreDir: t.TempDir(),
|
||||
NoLog: true, NoSigs: true}
|
||||
s, err := server.NewServer(opts)
|
||||
if err != nil {
|
||||
t.Fatalf("a bus for this test could not be made: %v", err)
|
||||
}
|
||||
go s.Start()
|
||||
if !s.ReadyForConnections(30 * time.Second) {
|
||||
s.Shutdown()
|
||||
t.Fatal("a bus for this test did not come up within 30s")
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
s.Shutdown()
|
||||
s.WaitForShutdown()
|
||||
})
|
||||
return s
|
||||
}
|
||||
@@ -0,0 +1,96 @@
|
||||
package testbus
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"encoding/json"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
)
|
||||
|
||||
// **The bus the tests run is the bus the mesh runs** (novox/hq ADR 0227 rule 9, to-be 45 §9). The server
|
||||
// linked into the tests is held to the release the catalogue's bus image is — read from beside this
|
||||
// checkout — and, in a merge check that has the facts snapshot, to the release the mesh's bus server says
|
||||
// it runs. A bus upgrade moves the catalogue's pin; this then fails until the tests' pin moves with it,
|
||||
// so the controller is never tested against a bus the mesh no longer runs, or not yet.
|
||||
func TestTheBusTheTestsRunIsTheBusTheMeshRuns(t *testing.T) {
|
||||
dockerfile := filepath.Join("..", "..", "..", "mesh-catalog", "modules", "nats", "Dockerfile")
|
||||
raw, err := os.ReadFile(dockerfile)
|
||||
switch {
|
||||
case err == nil:
|
||||
pinned := regexp.MustCompile(`upstream: nats ([0-9.]+)-alpine`).FindSubmatch(raw)
|
||||
if pinned == nil {
|
||||
t.Fatalf("%s says no release its digest is", dockerfile)
|
||||
}
|
||||
if string(pinned[1]) != Version {
|
||||
t.Errorf("the tests run bus %s and the catalogue's bus image is %s: move go.mod's nats-server pin "+
|
||||
"(and `go mod vendor`) with the image", Version, pinned[1])
|
||||
}
|
||||
case os.IsNotExist(err):
|
||||
t.Logf("the catalogue is not beside this checkout, so its bus image is not compared")
|
||||
default:
|
||||
t.Fatal(err)
|
||||
}
|
||||
path := os.Getenv("MESH_FACTS")
|
||||
if path == "" {
|
||||
t.Logf("no MESH_FACTS: the bus the mesh runs is compared in a merge check, which has it")
|
||||
return
|
||||
}
|
||||
body, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Read as little of the snapshot as this needs (internal/facts holds the whole shape).
|
||||
var f struct {
|
||||
Taken string `json:"taken"`
|
||||
Versions struct {
|
||||
Bus string `json:"bus"`
|
||||
} `json:"versions"`
|
||||
}
|
||||
if err := json.Unmarshal(body, &f); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if f.Versions.Bus != "" && strings.TrimPrefix(f.Versions.Bus, "v") != Version {
|
||||
t.Errorf("the tests run bus %s and the mesh's bus runs %s (facts of %s)", Version, f.Versions.Bus, f.Taken)
|
||||
}
|
||||
}
|
||||
|
||||
// A test's bus is its own: what one test makes, the next never sees.
|
||||
func TestEveryTestHasABusOfItsOwn(t *testing.T) {
|
||||
seen := map[string]bool{}
|
||||
for i := 0; i < 2; i++ {
|
||||
t.Run("one", func(t *testing.T) {
|
||||
url := URL(t)
|
||||
if seen[url] {
|
||||
t.Fatalf("two tests were handed %s", url)
|
||||
}
|
||||
seen[url] = true
|
||||
conn, err := nats.Connect(url)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer conn.Close()
|
||||
js, err := conn.JetStream()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := js.StreamInfo("EVENTS"); err == nil {
|
||||
t.Fatal("a stream an earlier test made is here")
|
||||
}
|
||||
if _, err := js.AddStream(&nats.StreamConfig{Name: "EVENTS", Subjects: []string{"mesh.>"}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
})
|
||||
}
|
||||
// Asked twice, one test reaches one bus.
|
||||
if a, b := URL(t), URL(t); a != b {
|
||||
t.Errorf("one test was handed two buses: %s and %s", a, b)
|
||||
}
|
||||
if v := strings.Count(Version, "."); v != 2 {
|
||||
t.Errorf("the server's release reads %q", Version)
|
||||
}
|
||||
}
|
||||
+21
@@ -0,0 +1,21 @@
|
||||
MIT License
|
||||
|
||||
Copyright (c) 2024 Antithesis
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all
|
||||
copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
SOFTWARE.
|
||||
+214
@@ -0,0 +1,214 @@
|
||||
//go:build enable_antithesis_sdk
|
||||
|
||||
// Package assert enables defining [test properties] about your program or [workload]. It is part of the [Antithesis Go SDK], which enables Go applications to integrate with the [Antithesis platform].
|
||||
//
|
||||
// Code that uses this package should be instrumented with the [antithesis-go-generator] utility. This step is required for the Always, Sometime, and Reachable methods. It is not required for the Unreachable and AlwaysOrUnreachable methods, but it will improve the experience of using them.
|
||||
//
|
||||
// These functions are no-ops with minimal performance overhead when called outside of the Antithesis environment. However, if the environment variable ANTITHESIS_SDK_LOCAL_OUTPUT is set, these functions will log to the file pointed to by that variable using a structured JSON format defined [here]. This allows you to make use of the Antithesis assertions package in your regular testing, or even in production. In particular, very few assertions frameworks offer a convenient way to define [Sometimes assertions], but they can be quite useful even outside Antithesis.
|
||||
//
|
||||
// Each function in this package takes a parameter called message, which is a human readable identifier used to aggregate assertions. Antithesis generates one test property per unique message and this test property will be named "<message>" in the [triage report].
|
||||
//
|
||||
// This test property either passes or fails, which depends upon the evaluation of every assertion that shares its message. Different assertions in different parts of the code should have different message, but the same assertion should always have the same message even if it is moved to a different file.
|
||||
//
|
||||
// Each function also takes a parameter called details, which is a key-value map of optional additional information provided by the user to add context for assertion failures. The information that is logged will appear in the [triage report], under the details section of the corresponding property. Normally the values passed to details are evaluated at runtime.
|
||||
//
|
||||
// [Antithesis Go SDK]: https://antithesis.com/docs/using_antithesis/sdk/go/
|
||||
// [Antithesis platform]: https://antithesis.com
|
||||
// [test properties]: https://antithesis.com/docs/properties_assertions/properties/
|
||||
// [workload]: https://antithesis.com/docs/test_templates/first_test/
|
||||
// [antithesis-go-generator]: https://antithesis.com/docs/using_antithesis/sdk/go/instrumentor/
|
||||
// [triage report]: https://antithesis.com/docs/reports/
|
||||
// [here]: https://antithesis.com/docs/using_antithesis/sdk/fallback/
|
||||
// [Sometimes assertions]: https://antithesis.com/docs/best_practices/sometimes_assertions/
|
||||
//
|
||||
// [details]: https://antithesis.com/docs/reports/
|
||||
package assert
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
)
|
||||
|
||||
type assertInfo struct {
|
||||
Location *locationInfo `json:"location"`
|
||||
Details map[string]any `json:"details"`
|
||||
AssertType string `json:"assert_type"`
|
||||
DisplayType string `json:"display_type"`
|
||||
Message string `json:"message"`
|
||||
Id string `json:"id"`
|
||||
Hit bool `json:"hit"`
|
||||
MustHit bool `json:"must_hit"`
|
||||
Condition bool `json:"condition"`
|
||||
}
|
||||
|
||||
// Create a custom json marshaler for assertInfo so that we can force Errors to be marshaled with their error details.
|
||||
// Without this, custom errors are marshaled as an empty object because the default json marshaler doesn't include the error
|
||||
// (because it's a method - not an exported struct field).
|
||||
func (f assertInfo) MarshalJSON() ([]byte, error) {
|
||||
type alias assertInfo // prevent infinite recursion
|
||||
a := alias(f)
|
||||
if a.Details != nil {
|
||||
a.Details = normalizeMap(a.Details)
|
||||
}
|
||||
return json.Marshal(a)
|
||||
}
|
||||
|
||||
type jsonError struct {
|
||||
innerError error
|
||||
}
|
||||
|
||||
func (e jsonError) MarshalJSON() ([]byte, error) {
|
||||
// Marshal this as the debug output string instead of e.Error(). These should be equivalent, but Sprintf correctly
|
||||
// handles nil values for us (which otherwise are annoying to defend against due to this - https://go.dev/doc/faq#nil_error)
|
||||
return json.Marshal(fmt.Sprintf("%+v", e.innerError))
|
||||
}
|
||||
|
||||
// Recursively replace any `error` with jsonError while doing a deep copy.
|
||||
// Most of the logic is in the normalize method below. This method exists to localize the type assertions
|
||||
// and provide a function that takes in/out a map instead of any.
|
||||
func normalizeMap(v map[string]any) map[string]any {
|
||||
return normalize(v).(map[string]any)
|
||||
}
|
||||
|
||||
func normalize(input any) any {
|
||||
// This switch will miss some cases (pointers, structs, non-any types), but should catch a very large proportion of real error interfaces
|
||||
// in real details objects. We can augment this if we find other cases common enough to support.
|
||||
switch inputTyped := input.(type) {
|
||||
case error:
|
||||
// Check if the underlying error implements json.Marshaler, so that if the error
|
||||
// already knows who to marshal itself, we don't override that.
|
||||
if _, ok := inputTyped.(json.Marshaler); ok {
|
||||
return inputTyped
|
||||
} else {
|
||||
return jsonError{inputTyped}
|
||||
}
|
||||
case map[string]any:
|
||||
out := make(map[string]any, len(inputTyped))
|
||||
for k, v := range inputTyped {
|
||||
out[k] = normalize(v)
|
||||
}
|
||||
return out
|
||||
case []any:
|
||||
out := make([]any, len(inputTyped))
|
||||
for i := range inputTyped {
|
||||
out[i] = normalize(inputTyped[i])
|
||||
}
|
||||
return out
|
||||
default:
|
||||
return input
|
||||
}
|
||||
}
|
||||
|
||||
type wrappedAssertInfo struct {
|
||||
A *assertInfo `json:"antithesis_assert"`
|
||||
}
|
||||
|
||||
// --------------------------------------------------------------------------------
|
||||
// Assertions
|
||||
// --------------------------------------------------------------------------------
|
||||
const (
|
||||
wasHit = true
|
||||
mustBeHit = true
|
||||
optionallyHit = false
|
||||
expectingTrue = true
|
||||
)
|
||||
|
||||
const (
|
||||
universalTest = "always"
|
||||
existentialTest = "sometimes"
|
||||
reachabilityTest = "reachability"
|
||||
)
|
||||
|
||||
const (
|
||||
alwaysDisplay = "Always"
|
||||
alwaysOrUnreachableDisplay = "AlwaysOrUnreachable"
|
||||
sometimesDisplay = "Sometimes"
|
||||
reachableDisplay = "Reachable"
|
||||
unreachableDisplay = "Unreachable"
|
||||
)
|
||||
|
||||
// Always asserts that condition is true every time this function is called, and that it is called at least once. The corresponding test property will be viewable in the Antithesis SDK: Always group of your triage report.
|
||||
func Always(condition bool, message string, details map[string]any) {
|
||||
locationInfo := newLocationInfo(offsetAPICaller)
|
||||
id := makeKey(message, locationInfo)
|
||||
assertImpl(condition, message, details, locationInfo, wasHit, mustBeHit, universalTest, alwaysDisplay, id)
|
||||
}
|
||||
|
||||
// AlwaysOrUnreachable asserts that condition is true every time this function is called. The corresponding test property will pass if the assertion is never encountered (unlike Always assertion types). This test property will be viewable in the “Antithesis SDK: Always” group of your triage report.
|
||||
func AlwaysOrUnreachable(condition bool, message string, details map[string]any) {
|
||||
locationInfo := newLocationInfo(offsetAPICaller)
|
||||
id := makeKey(message, locationInfo)
|
||||
assertImpl(condition, message, details, locationInfo, wasHit, optionallyHit, universalTest, alwaysOrUnreachableDisplay, id)
|
||||
}
|
||||
|
||||
// Sometimes asserts that condition is true at least one time that this function was called. (If the assertion is never encountered, the test property will therefore fail.) This test property will be viewable in the “Antithesis SDK: Sometimes” group.
|
||||
func Sometimes(condition bool, message string, details map[string]any) {
|
||||
locationInfo := newLocationInfo(offsetAPICaller)
|
||||
id := makeKey(message, locationInfo)
|
||||
assertImpl(condition, message, details, locationInfo, wasHit, mustBeHit, existentialTest, sometimesDisplay, id)
|
||||
}
|
||||
|
||||
// Unreachable asserts that a line of code is never reached. The corresponding test property will fail if this function is ever called. (If it is never called the test property will therefore pass.) This test property will be viewable in the “Antithesis SDK: Reachablity assertions” group.
|
||||
func Unreachable(message string, details map[string]any) {
|
||||
locationInfo := newLocationInfo(offsetAPICaller)
|
||||
id := makeKey(message, locationInfo)
|
||||
assertImpl(false, message, details, locationInfo, wasHit, optionallyHit, reachabilityTest, unreachableDisplay, id)
|
||||
}
|
||||
|
||||
// Reachable asserts that a line of code is reached at least once. The corresponding test property will pass if this function is ever called. (If it is never called the test property will therefore fail.) This test property will be viewable in the “Antithesis SDK: Reachablity assertions” group.
|
||||
func Reachable(message string, details map[string]any) {
|
||||
locationInfo := newLocationInfo(offsetAPICaller)
|
||||
id := makeKey(message, locationInfo)
|
||||
assertImpl(true, message, details, locationInfo, wasHit, mustBeHit, reachabilityTest, reachableDisplay, id)
|
||||
}
|
||||
|
||||
// AssertRaw is a low-level method designed to be used by third-party frameworks. Regular users of the assert package should not call it.
|
||||
func AssertRaw(cond bool, message string, details map[string]any,
|
||||
classname, funcname, filename string, line int,
|
||||
hit bool, mustHit bool,
|
||||
assertType string, displayType string,
|
||||
id string,
|
||||
) {
|
||||
assertImpl(cond, message, details,
|
||||
&locationInfo{classname, funcname, filename, line, columnUnknown},
|
||||
hit, mustHit,
|
||||
assertType, displayType,
|
||||
id)
|
||||
}
|
||||
|
||||
func assertImpl(cond bool, message string, details map[string]any,
|
||||
loc *locationInfo,
|
||||
hit bool, mustHit bool,
|
||||
assertType string, displayType string,
|
||||
id string,
|
||||
) {
|
||||
trackerEntry := assertTracker.getTrackerEntry(id, loc.Filename, loc.Classname)
|
||||
|
||||
// Always grab the Filename and Classname captured when the trackerEntry was established
|
||||
// This provides the consistency needed between instrumentation-time and runtime
|
||||
if loc.Filename != trackerEntry.Filename {
|
||||
loc.Filename = trackerEntry.Filename
|
||||
}
|
||||
|
||||
if loc.Classname != trackerEntry.Classname {
|
||||
loc.Classname = trackerEntry.Classname
|
||||
}
|
||||
|
||||
aI := &assertInfo{
|
||||
Hit: hit,
|
||||
MustHit: mustHit,
|
||||
AssertType: assertType,
|
||||
DisplayType: displayType,
|
||||
Message: message,
|
||||
Condition: cond,
|
||||
Id: id,
|
||||
Location: loc,
|
||||
Details: details,
|
||||
}
|
||||
|
||||
trackerEntry.emit(aI)
|
||||
}
|
||||
|
||||
func makeKey(message string, _ *locationInfo) string {
|
||||
return message
|
||||
}
|
||||
+16
@@ -0,0 +1,16 @@
|
||||
//go:build !enable_antithesis_sdk
|
||||
|
||||
package assert
|
||||
|
||||
func Always(condition bool, message string, details map[string]any) {}
|
||||
func AlwaysOrUnreachable(condition bool, message string, details map[string]any) {}
|
||||
func Sometimes(condition bool, message string, details map[string]any) {}
|
||||
func Unreachable(message string, details map[string]any) {}
|
||||
func Reachable(message string, details map[string]any) {}
|
||||
func AssertRaw(cond bool, message string, details map[string]any,
|
||||
classname, funcname, filename string, line int,
|
||||
hit bool, mustHit bool,
|
||||
assertType string, displayType string,
|
||||
id string,
|
||||
) {
|
||||
}
|
||||
+30
@@ -0,0 +1,30 @@
|
||||
package assert
|
||||
|
||||
// Allowable numeric types of comparison parameters
|
||||
type Number interface {
|
||||
~int | ~int8 | ~int16 | ~int32 | ~int64 | ~uint8 | ~uint16 | ~uint32 | ~float32 | ~float64 | ~uint64 | ~uint | ~uintptr
|
||||
}
|
||||
|
||||
// Internally, numeric guidanceFn Operands only use these
|
||||
type operandConstraint interface {
|
||||
int32 | int64 | uint64 | float64
|
||||
}
|
||||
|
||||
type numConstraint interface {
|
||||
uint64 | float64
|
||||
}
|
||||
|
||||
// Used for boolean assertions
|
||||
type NamedBool struct {
|
||||
First string `json:"first"`
|
||||
Second bool `json:"second"`
|
||||
}
|
||||
|
||||
// Convenience function to construct a NamedBool used for boolean assertions
|
||||
func NewNamedBool(first string, second bool) *NamedBool {
|
||||
p := NamedBool{
|
||||
First: first,
|
||||
Second: second,
|
||||
}
|
||||
return &p
|
||||
}
|
||||
+66
@@ -0,0 +1,66 @@
|
||||
//go:build enable_antithesis_sdk
|
||||
|
||||
package assert
|
||||
|
||||
import (
|
||||
"sync"
|
||||
|
||||
"github.com/antithesishq/antithesis-sdk-go/internal"
|
||||
)
|
||||
|
||||
// TODO: Tracker is intended to prevent sending the same guidance
|
||||
// more than once. In this case, we always send, so the tracker
|
||||
// is not presently used.
|
||||
type booleanGuidance struct {
|
||||
n int
|
||||
}
|
||||
|
||||
type booleanGuidanceTracker map[string]*booleanGuidance
|
||||
|
||||
var (
|
||||
boolean_guidance_tracker booleanGuidanceTracker = make(booleanGuidanceTracker)
|
||||
boolean_guidance_tracker_mutex sync.Mutex
|
||||
boolean_guidance_info_mutex sync.Mutex
|
||||
)
|
||||
|
||||
func (tracker booleanGuidanceTracker) getTrackerEntry(messageKey string) *booleanGuidance {
|
||||
var trackerEntry *booleanGuidance
|
||||
var ok bool
|
||||
|
||||
if tracker == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
boolean_guidance_tracker_mutex.Lock()
|
||||
defer boolean_guidance_tracker_mutex.Unlock()
|
||||
if trackerEntry, ok = boolean_guidance_tracker[messageKey]; !ok {
|
||||
trackerEntry = newBooleanGuidance()
|
||||
tracker[messageKey] = trackerEntry
|
||||
}
|
||||
|
||||
return trackerEntry
|
||||
}
|
||||
|
||||
// Create a boolean guidance tracker
|
||||
func newBooleanGuidance() *booleanGuidance {
|
||||
trackerInfo := booleanGuidance{}
|
||||
return &trackerInfo
|
||||
}
|
||||
|
||||
func (tI *booleanGuidance) send_value(bgI *booleanGuidanceInfo) {
|
||||
if tI == nil {
|
||||
return
|
||||
}
|
||||
|
||||
boolean_guidance_info_mutex.Lock()
|
||||
defer boolean_guidance_info_mutex.Unlock()
|
||||
|
||||
// The tracker entry should be consulted to determine
|
||||
// if this Guidance info has already been sent, or not.
|
||||
|
||||
emitBooleanGuidance(bgI)
|
||||
}
|
||||
|
||||
func emitBooleanGuidance(bgI *booleanGuidanceInfo) error {
|
||||
return internal.Json_data(map[string]any{"antithesis_guidance": bgI})
|
||||
}
|
||||
+58
@@ -0,0 +1,58 @@
|
||||
//go:build enable_antithesis_sdk
|
||||
|
||||
package assert
|
||||
|
||||
import (
|
||||
"path"
|
||||
"runtime"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// stackFrameOffset indicates how many frames to go up in the
|
||||
// call stack to find the filename/location/line info. As
|
||||
// this work is always done in NewLocationInfo(), the offset is
|
||||
// specified from the perspective of NewLocationInfo
|
||||
type stackFrameOffset int
|
||||
|
||||
// Order is important here since iota is being used
|
||||
const (
|
||||
offsetNewLocationInfo stackFrameOffset = iota
|
||||
offsetHere
|
||||
offsetAPICaller
|
||||
offsetAPICallersCaller
|
||||
)
|
||||
|
||||
// locationInfo represents the attributes known at instrumentation time
|
||||
// for each Antithesis assertion discovered
|
||||
type locationInfo struct {
|
||||
Classname string `json:"class"`
|
||||
Funcname string `json:"function"`
|
||||
Filename string `json:"file"`
|
||||
Line int `json:"begin_line"`
|
||||
Column int `json:"begin_column"`
|
||||
}
|
||||
|
||||
// columnUnknown is used when the column associated with
|
||||
// a locationInfo is not available
|
||||
const columnUnknown = 0
|
||||
|
||||
// NewLocationInfo creates a locationInfo directly from
|
||||
// the current execution context
|
||||
func newLocationInfo(nframes stackFrameOffset) *locationInfo {
|
||||
// Get location info and add to details
|
||||
funcname := "*function*"
|
||||
classname := "*class*"
|
||||
pc, filename, line, ok := runtime.Caller(int(nframes))
|
||||
if !ok {
|
||||
filename = "*file*"
|
||||
line = 0
|
||||
} else {
|
||||
if this_func := runtime.FuncForPC(pc); this_func != nil {
|
||||
fullname := this_func.Name()
|
||||
funcname = path.Ext(fullname)
|
||||
classname, _ = strings.CutSuffix(fullname, funcname)
|
||||
funcname = funcname[1:]
|
||||
}
|
||||
}
|
||||
return &locationInfo{classname, funcname, filename, line, columnUnknown}
|
||||
}
|
||||
+323
@@ -0,0 +1,323 @@
|
||||
//go:build enable_antithesis_sdk
|
||||
|
||||
package assert
|
||||
|
||||
import (
|
||||
"math"
|
||||
"sync"
|
||||
|
||||
"github.com/antithesishq/antithesis-sdk-go/internal"
|
||||
)
|
||||
|
||||
// --------------------------------------------------------------------------------
|
||||
// IntegerGap is used for:
|
||||
// - int, int8, int16, int32, int64:
|
||||
// - uint, uint8, uint16, uint32, uint64, uintptr:
|
||||
//
|
||||
// FloatGap is used for:
|
||||
// - float32, float64
|
||||
// --------------------------------------------------------------------------------
|
||||
type numericGapType int
|
||||
|
||||
const (
|
||||
integerGapType numericGapType = iota
|
||||
floatGapType
|
||||
)
|
||||
|
||||
func gapTypeForOperand[T Number](num T) numericGapType {
|
||||
gapType := integerGapType
|
||||
|
||||
switch any(num).(type) {
|
||||
case float32, float64:
|
||||
gapType = floatGapType
|
||||
}
|
||||
return gapType
|
||||
}
|
||||
|
||||
// --------------------------------------------------------------------------------
|
||||
// numericGuidanceTracker - Tracking Info for Numeric Guidance
|
||||
//
|
||||
// For GuidanceFnMaximize:
|
||||
// - gap is the largest value sent so far
|
||||
//
|
||||
// For GuidanceFnMinimize:
|
||||
// - gap is the most negative value sent so far
|
||||
//
|
||||
// --------------------------------------------------------------------------------
|
||||
type numericGuidanceInfo struct {
|
||||
gap any
|
||||
descriminator numericGapType
|
||||
maximize bool
|
||||
}
|
||||
|
||||
type numericGuidanceTracker map[string]*numericGuidanceInfo
|
||||
|
||||
var (
|
||||
numeric_guidance_tracker numericGuidanceTracker = make(numericGuidanceTracker)
|
||||
numeric_guidance_tracker_mutex sync.Mutex
|
||||
numeric_guidance_info_mutex sync.Mutex
|
||||
)
|
||||
|
||||
func (tracker numericGuidanceTracker) getTrackerEntry(messageKey string, trackerType numericGapType, maximize bool) *numericGuidanceInfo {
|
||||
var trackerEntry *numericGuidanceInfo
|
||||
var ok bool
|
||||
|
||||
if tracker == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
numeric_guidance_tracker_mutex.Lock()
|
||||
defer numeric_guidance_tracker_mutex.Unlock()
|
||||
if trackerEntry, ok = numeric_guidance_tracker[messageKey]; !ok {
|
||||
trackerEntry = newNumericGuidanceInfo(trackerType, maximize)
|
||||
tracker[messageKey] = trackerEntry
|
||||
}
|
||||
|
||||
return trackerEntry
|
||||
}
|
||||
|
||||
// Create an numeric guidance entry
|
||||
func newNumericGuidanceInfo(trackerType numericGapType, maximize bool) *numericGuidanceInfo {
|
||||
|
||||
var gap any
|
||||
if trackerType == integerGapType {
|
||||
gap = newGapValue(uint64(math.MaxUint64), maximize)
|
||||
} else {
|
||||
gap = newGapValue(float64(math.MaxFloat64), maximize)
|
||||
}
|
||||
trackerInfo := numericGuidanceInfo{
|
||||
maximize: maximize,
|
||||
descriminator: trackerType,
|
||||
gap: gap,
|
||||
}
|
||||
return &trackerInfo
|
||||
}
|
||||
|
||||
func (tI *numericGuidanceInfo) should_maximize() bool {
|
||||
return tI.maximize
|
||||
}
|
||||
|
||||
func (tI *numericGuidanceInfo) is_integer_gap() bool {
|
||||
return tI.descriminator == integerGapType
|
||||
}
|
||||
|
||||
// --------------------------------------------------------------------------------
|
||||
// Represents integral and floating point extremes
|
||||
// --------------------------------------------------------------------------------
|
||||
type gapValue[T numConstraint] struct {
|
||||
gap_size T
|
||||
gap_is_negative bool
|
||||
}
|
||||
|
||||
func newGapValue[T numConstraint](sz T, is_neg bool) any {
|
||||
switch any(sz).(type) {
|
||||
case uint64:
|
||||
return gapValue[uint64]{gap_size: uint64(sz), gap_is_negative: is_neg}
|
||||
|
||||
case float64:
|
||||
return gapValue[float64]{gap_size: float64(sz), gap_is_negative: is_neg}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func is_same_sign(left_val int64, right_val int64) bool {
|
||||
same_sign := false
|
||||
if left_val < 0 {
|
||||
// left is negative
|
||||
if right_val < 0 {
|
||||
same_sign = true
|
||||
}
|
||||
} else {
|
||||
// left is non-negative
|
||||
if right_val >= 0 {
|
||||
same_sign = true
|
||||
}
|
||||
}
|
||||
return same_sign
|
||||
}
|
||||
|
||||
func abs_int64(val int64) uint64 {
|
||||
if val >= 0 {
|
||||
return uint64(val)
|
||||
}
|
||||
return uint64(0 - val)
|
||||
}
|
||||
|
||||
func is_greater_than[T numConstraint](left gapValue[T], right gapValue[T]) bool {
|
||||
if !left.gap_is_negative && !right.gap_is_negative {
|
||||
return left.gap_size > right.gap_size
|
||||
}
|
||||
if !left.gap_is_negative && right.gap_is_negative {
|
||||
return true // any positive is greater than a negative
|
||||
}
|
||||
if left.gap_is_negative && right.gap_is_negative {
|
||||
return right.gap_size > left.gap_size
|
||||
}
|
||||
if left.gap_is_negative && !right.gap_is_negative {
|
||||
return false // any negative is less than a positive
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func is_less_than[T numConstraint](left gapValue[T], right gapValue[T]) bool {
|
||||
if !left.gap_is_negative && !right.gap_is_negative {
|
||||
return left.gap_size < right.gap_size
|
||||
}
|
||||
if !left.gap_is_negative && right.gap_is_negative {
|
||||
return false // any positive is greater than a negative
|
||||
}
|
||||
if left.gap_is_negative && right.gap_is_negative {
|
||||
return right.gap_size < left.gap_size
|
||||
}
|
||||
if left.gap_is_negative && !right.gap_is_negative {
|
||||
return true // any negative is less than a positive
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func send_value_if_needed(tI *numericGuidanceInfo, gI *guidanceInfo) {
|
||||
if tI == nil {
|
||||
return
|
||||
}
|
||||
|
||||
numeric_guidance_info_mutex.Lock()
|
||||
defer numeric_guidance_info_mutex.Unlock()
|
||||
|
||||
// if this is a catalog entry (gI.hit is false)
|
||||
// do not update the reference gap in the tracker (tI *numericGuidanceInfo)
|
||||
if !gI.Hit {
|
||||
emitGuidance(gI)
|
||||
return
|
||||
}
|
||||
|
||||
should_send := false
|
||||
maximize := tI.should_maximize()
|
||||
|
||||
var gap gapValue[uint64]
|
||||
var float_gap gapValue[float64]
|
||||
|
||||
// Needs to have individual case statements to assist
|
||||
// the compiler to infer the actual type of the var named 'operands'
|
||||
switch operands := (gI.Data).(type) {
|
||||
case numericOperands[int32]:
|
||||
gap = makeGap(operands)
|
||||
case numericOperands[int64]:
|
||||
gap = makeGap(operands)
|
||||
case numericOperands[uint64]:
|
||||
gap = makeGap(operands)
|
||||
case numericOperands[float64]:
|
||||
float_gap = makeFloatGap(operands)
|
||||
}
|
||||
|
||||
var prev_gap gapValue[uint64]
|
||||
var prev_float_gap gapValue[float64]
|
||||
has_prev_gap := false
|
||||
has_prev_float_gap := false
|
||||
|
||||
prev_gap, has_prev_gap = tI.gap.(gapValue[uint64])
|
||||
if !has_prev_gap {
|
||||
prev_float_gap, has_prev_float_gap = tI.gap.(gapValue[float64])
|
||||
}
|
||||
|
||||
if has_prev_gap {
|
||||
if maximize {
|
||||
should_send = is_greater_than(gap, prev_gap)
|
||||
} else {
|
||||
should_send = is_less_than(gap, prev_gap)
|
||||
}
|
||||
}
|
||||
|
||||
if has_prev_float_gap {
|
||||
if maximize {
|
||||
should_send = is_greater_than(float_gap, prev_float_gap)
|
||||
} else {
|
||||
should_send = is_less_than(float_gap, prev_float_gap)
|
||||
}
|
||||
}
|
||||
|
||||
if should_send {
|
||||
if tI.is_integer_gap() {
|
||||
tI.gap = gap
|
||||
} else {
|
||||
tI.gap = float_gap
|
||||
}
|
||||
emitGuidance(gI)
|
||||
}
|
||||
}
|
||||
|
||||
func emitGuidance(gI *guidanceInfo) error {
|
||||
return internal.Json_data(map[string]any{"antithesis_guidance": gI})
|
||||
}
|
||||
|
||||
// When left and right are the same sign (both negative, or both non-negative)
|
||||
// Calculate: <result> = (left - right). The gap_size is abs(<result>) and
|
||||
// gap_is_negative is (right > left)
|
||||
func makeGap[Op operandConstraint](operand numericOperands[Op]) gapValue[uint64] {
|
||||
|
||||
var gap_size uint64
|
||||
var gap_is_negative bool
|
||||
|
||||
switch this_op := any(operand).(type) {
|
||||
case numericOperands[int32]:
|
||||
result := int64(this_op.Left) - int64(this_op.Right)
|
||||
gap_size = abs_int64(result)
|
||||
gap_is_negative = result < 0
|
||||
|
||||
case numericOperands[int64]:
|
||||
if is_same_sign(this_op.Left, this_op.Right) {
|
||||
result := int64(this_op.Left) - int64(this_op.Right)
|
||||
gap_size = abs_int64(result)
|
||||
gap_is_negative = result < 0
|
||||
break
|
||||
}
|
||||
|
||||
// Otherwise left and right are opposite signs
|
||||
// gap = abs(left) + abs(right)
|
||||
// gap_is_negative = left < right
|
||||
left_gap_size := abs_int64(this_op.Left)
|
||||
right_gap_size := abs_int64(this_op.Right)
|
||||
gap_size = left_gap_size + right_gap_size
|
||||
gap_is_negative = this_op.Left < this_op.Right
|
||||
|
||||
case numericOperands[uint64]:
|
||||
left_val := this_op.Left
|
||||
right_val := this_op.Right
|
||||
gap_is_negative = false
|
||||
if left_val < right_val {
|
||||
gap_is_negative = true
|
||||
gap_size = right_val - left_val
|
||||
} else {
|
||||
gap_size = left_val - right_val
|
||||
}
|
||||
|
||||
default:
|
||||
zero_gap, _ := newGapValue(uint64(0), false).(gapValue[uint64])
|
||||
return zero_gap
|
||||
}
|
||||
|
||||
this_gap, _ := newGapValue(gap_size, gap_is_negative).(gapValue[uint64])
|
||||
return this_gap
|
||||
} // MakeGap
|
||||
|
||||
func makeFloatGap[Op operandConstraint](operand numericOperands[Op]) gapValue[float64] {
|
||||
switch this_op := any(operand).(type) {
|
||||
case numericOperands[float64]:
|
||||
left_val := this_op.Left
|
||||
right_val := this_op.Right
|
||||
gap_is_negative := false
|
||||
var gap_size float64
|
||||
if left_val < right_val {
|
||||
gap_is_negative = true
|
||||
gap_size = right_val - left_val
|
||||
} else {
|
||||
gap_size = left_val - right_val
|
||||
}
|
||||
|
||||
this_gap, _ := newGapValue(gap_size, gap_is_negative).(gapValue[float64])
|
||||
return this_gap
|
||||
|
||||
default:
|
||||
zero_gap, _ := newGapValue(float64(0.0), false).(gapValue[float64])
|
||||
return zero_gap
|
||||
}
|
||||
} // MakeFloatGap
|
||||
+330
@@ -0,0 +1,330 @@
|
||||
//go:build enable_antithesis_sdk
|
||||
|
||||
package assert
|
||||
|
||||
// A type for writing raw assertions.
|
||||
// guidanceFnType allows the assertion to provide guidance to
|
||||
// the Antithesis platform when testing in Antithesis.
|
||||
// Regular users of the assert package should not use it.
|
||||
type guidanceFnType int
|
||||
|
||||
const (
|
||||
guidanceFnMaximize guidanceFnType = iota // Maximize (left - right) values
|
||||
guidanceFnMinimize // Minimize (left - right) values
|
||||
guidanceFnWantAll // Encourages fuzzing explorations where boolean values are true
|
||||
guidanceFnWantNone // Encourages fuzzing explorations where boolean values are false
|
||||
guidanceFnExplore
|
||||
)
|
||||
|
||||
// guidanceFnExplore
|
||||
|
||||
func get_guidance_type_string(gt guidanceFnType) string {
|
||||
switch gt {
|
||||
case guidanceFnMaximize, guidanceFnMinimize:
|
||||
return "numeric"
|
||||
case guidanceFnWantAll, guidanceFnWantNone:
|
||||
return "boolean"
|
||||
case guidanceFnExplore:
|
||||
return "json"
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
type numericOperands[T operandConstraint] struct {
|
||||
Left T `json:"left"`
|
||||
Right T `json:"right"`
|
||||
}
|
||||
|
||||
type guidanceInfo struct {
|
||||
Data any `json:"guidance_data,omitempty"`
|
||||
Location *locationInfo `json:"location"`
|
||||
GuidanceType string `json:"guidance_type"`
|
||||
Message string `json:"message"`
|
||||
Id string `json:"id"`
|
||||
Maximize bool `json:"maximize"`
|
||||
Hit bool `json:"hit"`
|
||||
}
|
||||
|
||||
type booleanGuidanceInfo struct {
|
||||
Data any `json:"guidance_data,omitempty"`
|
||||
Location *locationInfo `json:"location"`
|
||||
GuidanceType string `json:"guidance_type"`
|
||||
Message string `json:"message"`
|
||||
Id string `json:"id"`
|
||||
Maximize bool `json:"maximize"`
|
||||
Hit bool `json:"hit"`
|
||||
}
|
||||
|
||||
func uses_maximize(gt guidanceFnType) bool {
|
||||
return gt == guidanceFnMaximize || gt == guidanceFnWantAll
|
||||
}
|
||||
|
||||
func newOperands[T Number](left, right T) any {
|
||||
switch any(left).(type) {
|
||||
case int8, int16, int32:
|
||||
return numericOperands[int32]{int32(left), int32(right)}
|
||||
case int, int64:
|
||||
return numericOperands[int64]{int64(left), int64(right)}
|
||||
case uint8, uint16, uint32, uint, uint64, uintptr:
|
||||
return numericOperands[uint64]{uint64(left), uint64(right)}
|
||||
case float32, float64:
|
||||
return numericOperands[float64]{float64(left), float64(right)}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func build_numeric_guidance[T Number](gt guidanceFnType, message string, left, right T, loc *locationInfo, id string, hit bool) *guidanceInfo {
|
||||
|
||||
operands := newOperands(left, right)
|
||||
if !hit {
|
||||
operands = nil
|
||||
}
|
||||
|
||||
gI := guidanceInfo{
|
||||
GuidanceType: get_guidance_type_string(gt),
|
||||
Message: message,
|
||||
Id: id,
|
||||
Location: loc,
|
||||
Maximize: uses_maximize(gt),
|
||||
Data: operands,
|
||||
Hit: hit,
|
||||
}
|
||||
|
||||
return &gI
|
||||
}
|
||||
|
||||
type namedBoolDictionary map[string]bool
|
||||
|
||||
func build_boolean_guidance(gt guidanceFnType, message string, named_bools []NamedBool,
|
||||
loc *locationInfo,
|
||||
id string, hit bool) *booleanGuidanceInfo {
|
||||
|
||||
var guidance_data any
|
||||
|
||||
// To ensure the sequence and naming for the named_bool values
|
||||
if hit {
|
||||
named_bool_dictionary := namedBoolDictionary{}
|
||||
for _, named_bool := range named_bools {
|
||||
named_bool_dictionary[named_bool.First] = named_bool.Second
|
||||
}
|
||||
guidance_data = named_bool_dictionary
|
||||
}
|
||||
|
||||
bgI := booleanGuidanceInfo{
|
||||
GuidanceType: get_guidance_type_string(gt),
|
||||
Message: message,
|
||||
Id: id,
|
||||
Location: loc,
|
||||
Maximize: uses_maximize(gt),
|
||||
Data: guidance_data,
|
||||
Hit: hit,
|
||||
}
|
||||
|
||||
return &bgI
|
||||
}
|
||||
|
||||
func behavior_to_guidance(behavior string) guidanceFnType {
|
||||
guidance := guidanceFnExplore
|
||||
switch behavior {
|
||||
case "maximize":
|
||||
guidance = guidanceFnMaximize
|
||||
case "minimize":
|
||||
guidance = guidanceFnMinimize
|
||||
case "all":
|
||||
guidance = guidanceFnWantAll
|
||||
case "none":
|
||||
guidance = guidanceFnWantNone
|
||||
}
|
||||
return guidance
|
||||
}
|
||||
|
||||
func numericGuidanceImpl[T Number](left, right T, message, id string, loc *locationInfo, guidanceFn guidanceFnType, hit bool) {
|
||||
tI := numeric_guidance_tracker.getTrackerEntry(id, gapTypeForOperand(left), uses_maximize(guidanceFn))
|
||||
gI := build_numeric_guidance(guidanceFn, message, left, right, loc, id, hit)
|
||||
send_value_if_needed(tI, gI)
|
||||
}
|
||||
|
||||
func booleanGuidanceImpl(named_bools []NamedBool, message, id string, loc *locationInfo, guidanceFn guidanceFnType, hit bool) {
|
||||
tI := boolean_guidance_tracker.getTrackerEntry(id)
|
||||
bgI := build_boolean_guidance(guidanceFn, message, named_bools, loc, id, hit)
|
||||
tI.send_value(bgI)
|
||||
}
|
||||
|
||||
// NumericGuidanceRaw is a low-level method designed to be used by third-party frameworks. Regular users of the assert package should not call it.
|
||||
func NumericGuidanceRaw[T Number](
|
||||
left, right T,
|
||||
message, id string,
|
||||
classname, funcname, filename string,
|
||||
line int,
|
||||
behavior string,
|
||||
hit bool,
|
||||
) {
|
||||
loc := &locationInfo{classname, funcname, filename, line, columnUnknown}
|
||||
guidanceFn := behavior_to_guidance(behavior)
|
||||
numericGuidanceImpl(left, right, message, id, loc, guidanceFn, hit)
|
||||
}
|
||||
|
||||
// BooleanGuidanceRaw is a low-level method designed to be used by third-party frameworks. Regular users of the assert package should not call it.
|
||||
func BooleanGuidanceRaw(
|
||||
named_bools []NamedBool,
|
||||
message, id string,
|
||||
classname, funcname, filename string,
|
||||
line int,
|
||||
behavior string,
|
||||
hit bool,
|
||||
) {
|
||||
loc := &locationInfo{classname, funcname, filename, line, columnUnknown}
|
||||
guidanceFn := behavior_to_guidance(behavior)
|
||||
booleanGuidanceImpl(named_bools, message, id, loc, guidanceFn, hit)
|
||||
}
|
||||
|
||||
func add_numeric_details[T Number](details map[string]any, left, right T) map[string]any {
|
||||
// ----------------------------------------------------
|
||||
// Can not use maps.Clone() until go 1.21.0 or above
|
||||
// enhancedDetails := maps.Clone(details)
|
||||
// ----------------------------------------------------
|
||||
enhancedDetails := map[string]any{}
|
||||
for k, v := range details {
|
||||
enhancedDetails[k] = v
|
||||
}
|
||||
enhancedDetails["left"] = left
|
||||
enhancedDetails["right"] = right
|
||||
return enhancedDetails
|
||||
}
|
||||
|
||||
func add_boolean_details(details map[string]any, named_bools []NamedBool) map[string]any {
|
||||
// ----------------------------------------------------
|
||||
// Can not use maps.Clone() until go 1.21.0 or above
|
||||
// enhancedDetails := maps.Clone(details)
|
||||
// ----------------------------------------------------
|
||||
enhancedDetails := map[string]any{}
|
||||
for k, v := range details {
|
||||
enhancedDetails[k] = v
|
||||
}
|
||||
for _, named_bool := range named_bools {
|
||||
enhancedDetails[named_bool.First] = named_bool.Second
|
||||
}
|
||||
return enhancedDetails
|
||||
}
|
||||
|
||||
// Equivalent to asserting Always(left > right, message, details). Information about left and right will automatically be added to the details parameter, with keys left and right. If you use this function for assertions that compare numeric quantities, you may help Antithesis find more bugs.
|
||||
func AlwaysGreaterThan[T Number](left, right T, message string, details map[string]any) {
|
||||
loc := newLocationInfo(offsetAPICaller)
|
||||
id := makeKey(message, loc)
|
||||
condition := left > right
|
||||
all_details := add_numeric_details(details, left, right)
|
||||
assertImpl(condition, message, all_details, loc, wasHit, mustBeHit, universalTest, alwaysDisplay, id)
|
||||
|
||||
numericGuidanceImpl(left, right, message, id, loc, guidanceFnMinimize, wasHit)
|
||||
}
|
||||
|
||||
// Equivalent to asserting Always(left >= right, message, details). Information about left and right will automatically be added to the details parameter, with keys left and right. If you use this function for assertions that compare numeric quantities, you may help Antithesis find more bugs.
|
||||
func AlwaysGreaterThanOrEqualTo[T Number](left, right T, message string, details map[string]any) {
|
||||
loc := newLocationInfo(offsetAPICaller)
|
||||
id := makeKey(message, loc)
|
||||
condition := left >= right
|
||||
all_details := add_numeric_details(details, left, right)
|
||||
assertImpl(condition, message, all_details, loc, wasHit, mustBeHit, universalTest, alwaysDisplay, id)
|
||||
|
||||
numericGuidanceImpl(left, right, message, id, loc, guidanceFnMinimize, wasHit)
|
||||
}
|
||||
|
||||
// Equivalent to asserting Sometimes(T left > T right, message, details). Information about left and right will automatically be added to the details parameter, with keys left and right. If you use this function for assertions that compare numeric quantities, you may help Antithesis find more bugs.
|
||||
func SometimesGreaterThan[T Number](left, right T, message string, details map[string]any) {
|
||||
loc := newLocationInfo(offsetAPICaller)
|
||||
id := makeKey(message, loc)
|
||||
condition := left > right
|
||||
all_details := add_numeric_details(details, left, right)
|
||||
assertImpl(condition, message, all_details, loc, wasHit, mustBeHit, existentialTest, sometimesDisplay, id)
|
||||
|
||||
numericGuidanceImpl(left, right, message, id, loc, guidanceFnMaximize, wasHit)
|
||||
}
|
||||
|
||||
// Equivalent to asserting Sometimes(T left >= T right, message, details). Information about left and right will automatically be added to the details parameter, with keys left and right. If you use this function for assertions that compare numeric quantities, you may help Antithesis find more bugs.
|
||||
func SometimesGreaterThanOrEqualTo[T Number](left, right T, message string, details map[string]any) {
|
||||
loc := newLocationInfo(offsetAPICaller)
|
||||
id := makeKey(message, loc)
|
||||
condition := left >= right
|
||||
all_details := add_numeric_details(details, left, right)
|
||||
assertImpl(condition, message, all_details, loc, wasHit, mustBeHit, existentialTest, sometimesDisplay, id)
|
||||
|
||||
numericGuidanceImpl(left, right, message, id, loc, guidanceFnMaximize, wasHit)
|
||||
}
|
||||
|
||||
// Equivalent to asserting Always(left < right, message, details). Information about left and right will automatically be added to the details parameter, with keys left and right. If you use this function for assertions that compare numeric quantities, you may help Antithesis find more bugs.
|
||||
func AlwaysLessThan[T Number](left, right T, message string, details map[string]any) {
|
||||
loc := newLocationInfo(offsetAPICaller)
|
||||
id := makeKey(message, loc)
|
||||
condition := left < right
|
||||
all_details := add_numeric_details(details, left, right)
|
||||
assertImpl(condition, message, all_details, loc, wasHit, mustBeHit, universalTest, alwaysDisplay, id)
|
||||
|
||||
numericGuidanceImpl(left, right, message, id, loc, guidanceFnMaximize, wasHit)
|
||||
}
|
||||
|
||||
// Equivalent to asserting Always(left <= right, message, details). Information about left and right will automatically be added to the details parameter, with keys left and right. If you use this function for assertions that compare numeric quantities, you may help Antithesis find more bugs.
|
||||
func AlwaysLessThanOrEqualTo[T Number](left, right T, message string, details map[string]any) {
|
||||
loc := newLocationInfo(offsetAPICaller)
|
||||
id := makeKey(message, loc)
|
||||
condition := left <= right
|
||||
all_details := add_numeric_details(details, left, right)
|
||||
assertImpl(condition, message, all_details, loc, wasHit, mustBeHit, universalTest, alwaysDisplay, id)
|
||||
|
||||
numericGuidanceImpl(left, right, message, id, loc, guidanceFnMaximize, wasHit)
|
||||
}
|
||||
|
||||
// Equivalent to asserting Sometimes(T left < T right, message, details). Information about left and right will automatically be added to the details parameter, with keys left and right. If you use this function for assertions that compare numeric quantities, you may help Antithesis find more bugs.
|
||||
func SometimesLessThan[T Number](left, right T, message string, details map[string]any) {
|
||||
loc := newLocationInfo(offsetAPICaller)
|
||||
id := makeKey(message, loc)
|
||||
condition := left < right
|
||||
all_details := add_numeric_details(details, left, right)
|
||||
assertImpl(condition, message, all_details, loc, wasHit, mustBeHit, existentialTest, sometimesDisplay, id)
|
||||
|
||||
numericGuidanceImpl(left, right, message, id, loc, guidanceFnMinimize, wasHit)
|
||||
}
|
||||
|
||||
// Equivalent to asserting Sometimes(T left <= T right, message, details). Information about left and right will automatically be added to the details parameter, with keys left and right. If you use this function for assertions that compare numeric quantities, you may help Antithesis find more bugs.
|
||||
func SometimesLessThanOrEqualTo[T Number](left, right T, message string, details map[string]any) {
|
||||
loc := newLocationInfo(offsetAPICaller)
|
||||
id := makeKey(message, loc)
|
||||
condition := left <= right
|
||||
all_details := add_numeric_details(details, left, right)
|
||||
assertImpl(condition, message, all_details, loc, wasHit, mustBeHit, existentialTest, sometimesDisplay, id)
|
||||
|
||||
numericGuidanceImpl(left, right, message, id, loc, guidanceFnMinimize, wasHit)
|
||||
}
|
||||
|
||||
// Asserts that every time this is called, at least one bool in named_bools is true. Equivalent to Always(named_bools[0].second || named_bools[1].second || ..., message, details). If you use this for assertions about the behavior of booleans, you may help Antithesis find more bugs. Information about named_bools will automatically be added to the details parameter, and the keys will be the names of the bools.
|
||||
func AlwaysSome(named_bools []NamedBool, message string, details map[string]any) {
|
||||
loc := newLocationInfo(offsetAPICaller)
|
||||
id := makeKey(message, loc)
|
||||
disjunction := false
|
||||
for _, named_bool := range named_bools {
|
||||
if named_bool.Second {
|
||||
disjunction = true
|
||||
break
|
||||
}
|
||||
}
|
||||
all_details := add_boolean_details(details, named_bools)
|
||||
assertImpl(disjunction, message, all_details, loc, wasHit, mustBeHit, universalTest, alwaysDisplay, id)
|
||||
|
||||
booleanGuidanceImpl(named_bools, message, id, loc, guidanceFnWantNone, wasHit)
|
||||
}
|
||||
|
||||
// Asserts that at least one time this is called, every bool in named_bools is true. Equivalent to Sometimes(named_bools[0].second && named_bools[1].second && ..., message, details). If you use this for assertions about the behavior of booleans, you may help Antithesis find more bugs. Information about named_bools will automatically be added to the details parameter, and the keys will be the names of the bools.
|
||||
func SometimesAll(named_bools []NamedBool, message string, details map[string]any) {
|
||||
loc := newLocationInfo(offsetAPICaller)
|
||||
id := makeKey(message, loc)
|
||||
conjunction := true
|
||||
for _, named_bool := range named_bools {
|
||||
if !named_bool.Second {
|
||||
conjunction = false
|
||||
break
|
||||
}
|
||||
}
|
||||
all_details := add_boolean_details(details, named_bools)
|
||||
assertImpl(conjunction, message, all_details, loc, wasHit, mustBeHit, existentialTest, sometimesDisplay, id)
|
||||
|
||||
booleanGuidanceImpl(named_bools, message, id, loc, guidanceFnWantAll, wasHit)
|
||||
}
|
||||
+34
@@ -0,0 +1,34 @@
|
||||
//go:build !enable_antithesis_sdk
|
||||
|
||||
package assert
|
||||
|
||||
func AlwaysGreaterThan[T Number](left, right T, message string, details map[string]any) {}
|
||||
func AlwaysGreaterThanOrEqualTo[T Number](left, right T, message string, details map[string]any) {}
|
||||
func SometimesGreaterThan[T Number](left, right T, message string, details map[string]any) {}
|
||||
func SometimesGreaterThanOrEqualTo[T Number](left, right T, message string, details map[string]any) {}
|
||||
func AlwaysLessThan[T Number](left, right T, message string, details map[string]any) {}
|
||||
func AlwaysLessThanOrEqualTo[T Number](left, right T, message string, details map[string]any) {}
|
||||
func SometimesLessThan[T Number](left, right T, message string, details map[string]any) {}
|
||||
func SometimesLessThanOrEqualTo[T Number](left, right T, message string, details map[string]any) {}
|
||||
|
||||
func AlwaysSome(named_bool []NamedBool, message string, details map[string]any) {}
|
||||
func SometimesAll(named_bool []NamedBool, message string, details map[string]any) {}
|
||||
|
||||
func NumericGuidanceRaw[T Number](left, right T,
|
||||
message, id string,
|
||||
classname, funcname, filename string,
|
||||
line int,
|
||||
behavior string,
|
||||
hit bool,
|
||||
) {
|
||||
}
|
||||
|
||||
func BooleanGuidanceRaw(
|
||||
named_bools []NamedBool,
|
||||
message, id string,
|
||||
classname, funcname, filename string,
|
||||
line int,
|
||||
behavior string,
|
||||
hit bool,
|
||||
) {
|
||||
}
|
||||
+111
@@ -0,0 +1,111 @@
|
||||
//go:build enable_antithesis_sdk
|
||||
|
||||
package assert
|
||||
|
||||
import (
|
||||
"runtime"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
|
||||
"github.com/antithesishq/antithesis-sdk-go/internal"
|
||||
)
|
||||
|
||||
type trackerInfo struct {
|
||||
Filename string
|
||||
Classname string
|
||||
PassCount int
|
||||
FailCount int
|
||||
}
|
||||
|
||||
type emitTracker map[string]*trackerInfo
|
||||
|
||||
// assert_tracker (global) keeps track of the unique asserts evaluated
|
||||
var (
|
||||
assertTracker emitTracker = make(emitTracker)
|
||||
trackerMutex sync.Mutex
|
||||
trackerInfoMutex sync.Mutex
|
||||
)
|
||||
|
||||
func (tracker emitTracker) getTrackerEntry(messageKey string, filename, classname string) *trackerInfo {
|
||||
var trackerEntry *trackerInfo
|
||||
var ok bool
|
||||
|
||||
if tracker == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
trackerMutex.Lock()
|
||||
defer trackerMutex.Unlock()
|
||||
if trackerEntry, ok = tracker[messageKey]; !ok {
|
||||
trackerEntry = newTrackerInfo(filename, classname)
|
||||
tracker[messageKey] = trackerEntry
|
||||
}
|
||||
return trackerEntry
|
||||
}
|
||||
|
||||
func newTrackerInfo(filename, classname string) *trackerInfo {
|
||||
trackerInfo := trackerInfo{
|
||||
PassCount: 0,
|
||||
FailCount: 0,
|
||||
Filename: filename,
|
||||
Classname: classname,
|
||||
}
|
||||
return &trackerInfo
|
||||
}
|
||||
|
||||
func (ti *trackerInfo) emit(ai *assertInfo) {
|
||||
if ti == nil || ai == nil {
|
||||
return
|
||||
}
|
||||
|
||||
// Registrations are just sent to voidstar
|
||||
hit := ai.Hit
|
||||
if !hit {
|
||||
emitAssert(ai)
|
||||
return
|
||||
}
|
||||
|
||||
var err error
|
||||
cond := ai.Condition
|
||||
|
||||
trackerInfoMutex.Lock()
|
||||
defer trackerInfoMutex.Unlock()
|
||||
if cond {
|
||||
if ti.PassCount == 0 {
|
||||
err = emitAssert(ai)
|
||||
}
|
||||
if err == nil {
|
||||
ti.PassCount++
|
||||
}
|
||||
return
|
||||
}
|
||||
if ti.FailCount == 0 {
|
||||
err = emitAssert(ai)
|
||||
}
|
||||
if err == nil {
|
||||
ti.FailCount++
|
||||
}
|
||||
}
|
||||
|
||||
func versionMessage() {
|
||||
languageBlock := map[string]any{
|
||||
"name": "Go",
|
||||
"version": runtime.Version(),
|
||||
}
|
||||
versionBlock := map[string]any{
|
||||
"language": languageBlock,
|
||||
"sdk_version": internal.SDK_Version,
|
||||
"protocol_version": internal.Protocol_Version,
|
||||
}
|
||||
internal.Json_data(map[string]any{"antithesis_sdk": versionBlock})
|
||||
}
|
||||
|
||||
// package-level flag
|
||||
var hasEmitted atomic.Bool // initialzed to false
|
||||
|
||||
func emitAssert(ai *assertInfo) error {
|
||||
if hasEmitted.CompareAndSwap(false, true) {
|
||||
versionMessage()
|
||||
}
|
||||
return internal.Json_data(wrappedAssertInfo{ai})
|
||||
}
|
||||
+100
@@ -0,0 +1,100 @@
|
||||
//go:build enable_antithesis_sdk
|
||||
|
||||
package internal
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"log"
|
||||
"math/rand"
|
||||
"os"
|
||||
)
|
||||
|
||||
func Json_data(v any) error {
|
||||
if data, err := json.Marshal(v); err != nil {
|
||||
return err
|
||||
} else {
|
||||
handler.output(string(data))
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
func Get_random() uint64 {
|
||||
return handler.random()
|
||||
}
|
||||
|
||||
func Notify(edge uint64) bool {
|
||||
return handler.notify(edge)
|
||||
}
|
||||
|
||||
func InitCoverage(num_edges uint64, symbols string) uint64 {
|
||||
return handler.init_coverage(num_edges, symbols)
|
||||
}
|
||||
|
||||
type libHandler interface {
|
||||
output(message string)
|
||||
random() uint64
|
||||
notify(edge uint64) bool
|
||||
init_coverage(num_edges uint64, symbols string) uint64
|
||||
}
|
||||
|
||||
const (
|
||||
errorLogLinePrefix = "[* antithesis-sdk-go *]"
|
||||
)
|
||||
|
||||
var handler libHandler
|
||||
|
||||
type localHandler struct {
|
||||
outputFile *os.File // can be nil
|
||||
}
|
||||
|
||||
func (h *localHandler) output(message string) {
|
||||
msg_len := len(message)
|
||||
if msg_len == 0 {
|
||||
return
|
||||
}
|
||||
if h.outputFile != nil {
|
||||
h.outputFile.WriteString(message + "\n")
|
||||
}
|
||||
}
|
||||
|
||||
func (h *localHandler) random() uint64 {
|
||||
return rand.Uint64()
|
||||
}
|
||||
|
||||
func (h *localHandler) notify(edge uint64) bool {
|
||||
return false
|
||||
}
|
||||
|
||||
func (h *localHandler) init_coverage(num_edges uint64, symbols string) uint64 {
|
||||
return 0
|
||||
}
|
||||
|
||||
func init() {
|
||||
handler = init_in_antithesis()
|
||||
if handler == nil {
|
||||
// Otherwise fallback to the local handler.
|
||||
handler = openLocalHandler()
|
||||
}
|
||||
}
|
||||
|
||||
// If `localOutputEnvVar` is set to a non-empty path, attempt to open that path and truncate the file
|
||||
// to serve as the log file of the local handler.
|
||||
// Otherwise, we don't have a log file, and logging is a no-op in the local handler.
|
||||
func openLocalHandler() *localHandler {
|
||||
path, is_set := os.LookupEnv(localOutputEnvVar)
|
||||
if !is_set || len(path) == 0 {
|
||||
return &localHandler{nil}
|
||||
}
|
||||
|
||||
// Open the file R/W (create if needed and possible)
|
||||
file, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE, 0644)
|
||||
if err != nil {
|
||||
log.Printf("%s Failed to open path %s: %v", errorLogLinePrefix, path, err)
|
||||
file = nil
|
||||
} else if err = file.Truncate(0); err != nil {
|
||||
log.Printf("%s Failed to truncate file at %s: %v", errorLogLinePrefix, path, err)
|
||||
file = nil
|
||||
}
|
||||
|
||||
return &localHandler{file}
|
||||
}
|
||||
+12
@@ -0,0 +1,12 @@
|
||||
package internal
|
||||
|
||||
// --------------------------------------------------------------------------------
|
||||
// Versions
|
||||
// --------------------------------------------------------------------------------
|
||||
const SDK_Version = "0.7.0"
|
||||
const Protocol_Version = "1.1.0"
|
||||
|
||||
// --------------------------------------------------------------------------------
|
||||
// Environment Vars
|
||||
// --------------------------------------------------------------------------------
|
||||
const localOutputEnvVar = "ANTITHESIS_SDK_LOCAL_OUTPUT"
|
||||
+160
@@ -0,0 +1,160 @@
|
||||
//go:build enable_antithesis_sdk && linux && amd64 && cgo
|
||||
|
||||
package internal
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"unsafe"
|
||||
"os"
|
||||
)
|
||||
|
||||
// --------------------------------------------------------------------------------
|
||||
// To build and run an executable with this package
|
||||
//
|
||||
// CC=clang CGO_ENABLED=1 go run ./main.go
|
||||
// --------------------------------------------------------------------------------
|
||||
|
||||
// \/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/
|
||||
//
|
||||
// The commented lines below, and the `import "C"` line which must directly follow
|
||||
// the commented lines are used by CGO. They are load-bearing, and should not be
|
||||
// changed without first understanding how CGO uses them.
|
||||
//
|
||||
// \/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/
|
||||
|
||||
// #cgo LDFLAGS: -ldl
|
||||
//
|
||||
// #include <dlfcn.h>
|
||||
// #include <stdbool.h>
|
||||
// #include <stdint.h>
|
||||
// #include <stdlib.h>
|
||||
//
|
||||
// typedef void (*go_fuzz_json_data_fn)(const char *data, size_t size);
|
||||
// void
|
||||
// go_fuzz_json_data(void *f, const char *data, size_t size) {
|
||||
// ((go_fuzz_json_data_fn)f)(data, size);
|
||||
// }
|
||||
//
|
||||
// typedef void (*go_fuzz_flush_fn)(void);
|
||||
// void
|
||||
// go_fuzz_flush(void *f) {
|
||||
// ((go_fuzz_flush_fn)f)();
|
||||
// }
|
||||
//
|
||||
// typedef uint64_t (*go_fuzz_get_random_fn)(void);
|
||||
// uint64_t
|
||||
// go_fuzz_get_random(void *f) {
|
||||
// return ((go_fuzz_get_random_fn)f)();
|
||||
// }
|
||||
//
|
||||
// typedef bool (*go_notify_coverage_fn)(size_t);
|
||||
// int
|
||||
// go_notify_coverage(void *f, size_t edges) {
|
||||
// bool b = ((go_notify_coverage_fn)f)(edges);
|
||||
// return b ? 1 : 0;
|
||||
// }
|
||||
//
|
||||
// typedef uint64_t (*go_init_coverage_fn)(size_t num_edges, const char *symbols);
|
||||
// uint64_t
|
||||
// go_init_coverage(void *f, size_t num_edges, const char *symbols) {
|
||||
// return ((go_init_coverage_fn)f)(num_edges, symbols);
|
||||
// }
|
||||
//
|
||||
import "C"
|
||||
|
||||
const (
|
||||
defaultNativeLibraryPath = "/usr/lib/libvoidstar.so"
|
||||
)
|
||||
|
||||
type voidstarHandler struct {
|
||||
fuzzJsonData unsafe.Pointer
|
||||
fuzzFlush unsafe.Pointer
|
||||
fuzzGetRandom unsafe.Pointer
|
||||
initCoverage unsafe.Pointer
|
||||
notifyCoverage unsafe.Pointer
|
||||
}
|
||||
|
||||
func (h *voidstarHandler) output(message string) {
|
||||
msg_len := len(message)
|
||||
if msg_len == 0 {
|
||||
return
|
||||
}
|
||||
cstrMessage := C.CString(message)
|
||||
defer C.free(unsafe.Pointer(cstrMessage))
|
||||
C.go_fuzz_json_data(h.fuzzJsonData, cstrMessage, C.ulong(msg_len))
|
||||
C.go_fuzz_flush(h.fuzzFlush)
|
||||
}
|
||||
|
||||
func (h *voidstarHandler) random() uint64 {
|
||||
return uint64(C.go_fuzz_get_random(h.fuzzGetRandom))
|
||||
}
|
||||
|
||||
func (h *voidstarHandler) init_coverage(num_edge uint64, symbols string) uint64 {
|
||||
cstrSymbols := C.CString(symbols)
|
||||
defer C.free(unsafe.Pointer(cstrSymbols))
|
||||
return uint64(C.go_init_coverage(h.initCoverage, C.ulong(num_edge), cstrSymbols))
|
||||
}
|
||||
|
||||
func (h *voidstarHandler) notify(edge uint64) bool {
|
||||
ival := int(C.go_notify_coverage(h.notifyCoverage, C.ulong(edge)))
|
||||
return ival == 1
|
||||
}
|
||||
|
||||
// Attempt to load libvoidstar and some symbols from `path`
|
||||
func openSharedLib(path string) (*voidstarHandler, error) {
|
||||
cstrPath := C.CString(path)
|
||||
defer C.free(unsafe.Pointer(cstrPath))
|
||||
|
||||
dlError := func(message string) error {
|
||||
return fmt.Errorf("%s: (%s)", message, C.GoString(C.dlerror()))
|
||||
}
|
||||
|
||||
sharedLib := C.dlopen(cstrPath, C.int(C.RTLD_NOW))
|
||||
if sharedLib == nil {
|
||||
return nil, dlError("Can not load the Antithesis native library")
|
||||
}
|
||||
|
||||
loadFunc := func(name string) (symbol unsafe.Pointer, err error) {
|
||||
cstrName := C.CString(name)
|
||||
defer C.free(unsafe.Pointer(cstrName))
|
||||
if symbol = C.dlsym(sharedLib, cstrName); symbol == nil {
|
||||
err = dlError(fmt.Sprintf("Can not access symbol %s", name))
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
fuzzJsonData, err := loadFunc("fuzz_json_data")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
fuzzFlush, err := loadFunc("fuzz_flush")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
fuzzGetRandom, err := loadFunc("fuzz_get_random")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
notifyCoverage, err := loadFunc("notify_coverage")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
initCoverage, err := loadFunc("init_coverage_module")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &voidstarHandler{fuzzJsonData, fuzzFlush, fuzzGetRandom, initCoverage, notifyCoverage}, nil
|
||||
}
|
||||
|
||||
// If we have a file at `defaultNativeLibraryPath`, we load the shared library
|
||||
// (and panic on any error encountered during load).
|
||||
func init_in_antithesis() libHandler {
|
||||
if _, err := os.Stat(defaultNativeLibraryPath); err == nil {
|
||||
handler, err := openSharedLib(defaultNativeLibraryPath)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return handler
|
||||
}
|
||||
return nil
|
||||
}
|
||||
Generated
Vendored
+7
@@ -0,0 +1,7 @@
|
||||
//go:build enable_antithesis_sdk && (!linux || !amd64 || !cgo)
|
||||
|
||||
package internal
|
||||
|
||||
func init_in_antithesis() libHandler {
|
||||
return nil
|
||||
}
|
||||
+202
@@ -0,0 +1,202 @@
|
||||
|
||||
Apache License
|
||||
Version 2.0, January 2004
|
||||
http://www.apache.org/licenses/
|
||||
|
||||
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
||||
|
||||
1. Definitions.
|
||||
|
||||
"License" shall mean the terms and conditions for use, reproduction,
|
||||
and distribution as defined by Sections 1 through 9 of this document.
|
||||
|
||||
"Licensor" shall mean the copyright owner or entity authorized by
|
||||
the copyright owner that is granting the License.
|
||||
|
||||
"Legal Entity" shall mean the union of the acting entity and all
|
||||
other entities that control, are controlled by, or are under common
|
||||
control with that entity. For the purposes of this definition,
|
||||
"control" means (i) the power, direct or indirect, to cause the
|
||||
direction or management of such entity, whether by contract or
|
||||
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
||||
outstanding shares, or (iii) beneficial ownership of such entity.
|
||||
|
||||
"You" (or "Your") shall mean an individual or Legal Entity
|
||||
exercising permissions granted by this License.
|
||||
|
||||
"Source" form shall mean the preferred form for making modifications,
|
||||
including but not limited to software source code, documentation
|
||||
source, and configuration files.
|
||||
|
||||
"Object" form shall mean any form resulting from mechanical
|
||||
transformation or translation of a Source form, including but
|
||||
not limited to compiled object code, generated documentation,
|
||||
and conversions to other media types.
|
||||
|
||||
"Work" shall mean the work of authorship, whether in Source or
|
||||
Object form, made available under the License, as indicated by a
|
||||
copyright notice that is included in or attached to the work
|
||||
(an example is provided in the Appendix below).
|
||||
|
||||
"Derivative Works" shall mean any work, whether in Source or Object
|
||||
form, that is based on (or derived from) the Work and for which the
|
||||
editorial revisions, annotations, elaborations, or other modifications
|
||||
represent, as a whole, an original work of authorship. For the purposes
|
||||
of this License, Derivative Works shall not include works that remain
|
||||
separable from, or merely link (or bind by name) to the interfaces of,
|
||||
the Work and Derivative Works thereof.
|
||||
|
||||
"Contribution" shall mean any work of authorship, including
|
||||
the original version of the Work and any modifications or additions
|
||||
to that Work or Derivative Works thereof, that is intentionally
|
||||
submitted to Licensor for inclusion in the Work by the copyright owner
|
||||
or by an individual or Legal Entity authorized to submit on behalf of
|
||||
the copyright owner. For the purposes of this definition, "submitted"
|
||||
means any form of electronic, verbal, or written communication sent
|
||||
to the Licensor or its representatives, including but not limited to
|
||||
communication on electronic mailing lists, source code control systems,
|
||||
and issue tracking systems that are managed by, or on behalf of, the
|
||||
Licensor for the purpose of discussing and improving the Work, but
|
||||
excluding communication that is conspicuously marked or otherwise
|
||||
designated in writing by the copyright owner as "Not a Contribution."
|
||||
|
||||
"Contributor" shall mean Licensor and any individual or Legal Entity
|
||||
on behalf of whom a Contribution has been received by Licensor and
|
||||
subsequently incorporated within the Work.
|
||||
|
||||
2. Grant of Copyright License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
copyright license to reproduce, prepare Derivative Works of,
|
||||
publicly display, publicly perform, sublicense, and distribute the
|
||||
Work and such Derivative Works in Source or Object form.
|
||||
|
||||
3. Grant of Patent License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
(except as stated in this section) patent license to make, have made,
|
||||
use, offer to sell, sell, import, and otherwise transfer the Work,
|
||||
where such license applies only to those patent claims licensable
|
||||
by such Contributor that are necessarily infringed by their
|
||||
Contribution(s) alone or by combination of their Contribution(s)
|
||||
with the Work to which such Contribution(s) was submitted. If You
|
||||
institute patent litigation against any entity (including a
|
||||
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
||||
or a Contribution incorporated within the Work constitutes direct
|
||||
or contributory patent infringement, then any patent licenses
|
||||
granted to You under this License for that Work shall terminate
|
||||
as of the date such litigation is filed.
|
||||
|
||||
4. Redistribution. You may reproduce and distribute copies of the
|
||||
Work or Derivative Works thereof in any medium, with or without
|
||||
modifications, and in Source or Object form, provided that You
|
||||
meet the following conditions:
|
||||
|
||||
(a) You must give any other recipients of the Work or
|
||||
Derivative Works a copy of this License; and
|
||||
|
||||
(b) You must cause any modified files to carry prominent notices
|
||||
stating that You changed the files; and
|
||||
|
||||
(c) You must retain, in the Source form of any Derivative Works
|
||||
that You distribute, all copyright, patent, trademark, and
|
||||
attribution notices from the Source form of the Work,
|
||||
excluding those notices that do not pertain to any part of
|
||||
the Derivative Works; and
|
||||
|
||||
(d) If the Work includes a "NOTICE" text file as part of its
|
||||
distribution, then any Derivative Works that You distribute must
|
||||
include a readable copy of the attribution notices contained
|
||||
within such NOTICE file, excluding those notices that do not
|
||||
pertain to any part of the Derivative Works, in at least one
|
||||
of the following places: within a NOTICE text file distributed
|
||||
as part of the Derivative Works; within the Source form or
|
||||
documentation, if provided along with the Derivative Works; or,
|
||||
within a display generated by the Derivative Works, if and
|
||||
wherever such third-party notices normally appear. The contents
|
||||
of the NOTICE file are for informational purposes only and
|
||||
do not modify the License. You may add Your own attribution
|
||||
notices within Derivative Works that You distribute, alongside
|
||||
or as an addendum to the NOTICE text from the Work, provided
|
||||
that such additional attribution notices cannot be construed
|
||||
as modifying the License.
|
||||
|
||||
You may add Your own copyright statement to Your modifications and
|
||||
may provide additional or different license terms and conditions
|
||||
for use, reproduction, or distribution of Your modifications, or
|
||||
for any such Derivative Works as a whole, provided Your use,
|
||||
reproduction, and distribution of the Work otherwise complies with
|
||||
the conditions stated in this License.
|
||||
|
||||
5. Submission of Contributions. Unless You explicitly state otherwise,
|
||||
any Contribution intentionally submitted for inclusion in the Work
|
||||
by You to the Licensor shall be under the terms and conditions of
|
||||
this License, without any additional terms or conditions.
|
||||
Notwithstanding the above, nothing herein shall supersede or modify
|
||||
the terms of any separate license agreement you may have executed
|
||||
with Licensor regarding such Contributions.
|
||||
|
||||
6. Trademarks. This License does not grant permission to use the trade
|
||||
names, trademarks, service marks, or product names of the Licensor,
|
||||
except as required for reasonable and customary use in describing the
|
||||
origin of the Work and reproducing the content of the NOTICE file.
|
||||
|
||||
7. Disclaimer of Warranty. Unless required by applicable law or
|
||||
agreed to in writing, Licensor provides the Work (and each
|
||||
Contributor provides its Contributions) on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
implied, including, without limitation, any warranties or conditions
|
||||
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
||||
PARTICULAR PURPOSE. You are solely responsible for determining the
|
||||
appropriateness of using or redistributing the Work and assume any
|
||||
risks associated with Your exercise of permissions under this License.
|
||||
|
||||
8. Limitation of Liability. In no event and under no legal theory,
|
||||
whether in tort (including negligence), contract, or otherwise,
|
||||
unless required by applicable law (such as deliberate and grossly
|
||||
negligent acts) or agreed to in writing, shall any Contributor be
|
||||
liable to You for damages, including any direct, indirect, special,
|
||||
incidental, or consequential damages of any character arising as a
|
||||
result of this License or out of the use or inability to use the
|
||||
Work (including but not limited to damages for loss of goodwill,
|
||||
work stoppage, computer failure or malfunction, or any and all
|
||||
other commercial damages or losses), even if such Contributor
|
||||
has been advised of the possibility of such damages.
|
||||
|
||||
9. Accepting Warranty or Additional Liability. While redistributing
|
||||
the Work or Derivative Works thereof, You may choose to offer,
|
||||
and charge a fee for, acceptance of support, warranty, indemnity,
|
||||
or other liability obligations and/or rights consistent with this
|
||||
License. However, in accepting such obligations, You may act only
|
||||
on Your own behalf and on Your sole responsibility, not on behalf
|
||||
of any other Contributor, and only if You agree to indemnify,
|
||||
defend, and hold each Contributor harmless for any liability
|
||||
incurred by, or claims asserted against, such Contributor by reason
|
||||
of your accepting any such warranty or additional liability.
|
||||
|
||||
END OF TERMS AND CONDITIONS
|
||||
|
||||
APPENDIX: How to apply the Apache License to your work.
|
||||
|
||||
To apply the Apache License to your work, attach the following
|
||||
boilerplate notice, with the fields enclosed by brackets "[]"
|
||||
replaced with your own identifying information. (Don't include
|
||||
the brackets!) The text should be enclosed in the appropriate
|
||||
comment syntax for the file format. We also recommend that a
|
||||
file or class name and description of purpose be included on the
|
||||
same "printed page" as the copyright notice for easier
|
||||
identification within third-party archives.
|
||||
|
||||
Copyright [yyyy] [name of copyright owner]
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
+35
@@ -0,0 +1,35 @@
|
||||
# TPM 2.0 client library
|
||||
|
||||
## Tests
|
||||
|
||||
This library contains unit tests in `github.com/google/go-tpm/tpm2`, which just
|
||||
tests that various encoding and error checking functions work correctly. It also
|
||||
contains more comprehensive integration tests in
|
||||
`github.com/google/go-tpm/tpm2/test`, which run actual commands on a TPM.
|
||||
|
||||
By default, these integration tests are run against the
|
||||
[`go-tpm-tools`](https://github.com/google/go-tpm-tools)
|
||||
simulator, which is baesed on the
|
||||
[Microsoft Reference TPM2 code](https://github.com/microsoft/ms-tpm-20-ref). To
|
||||
run both the unit and integration tests, run (in this directory)
|
||||
```bash
|
||||
go test . ./test
|
||||
```
|
||||
|
||||
These integration tests can also be run against a real TPM device. This is
|
||||
slightly more complex as the tests often need to be built as a normal user and
|
||||
then executed as root. For example,
|
||||
```bash
|
||||
# Build the test binary without running it
|
||||
go test -c github.com/google/go-tpm/tpm2/test
|
||||
# Execute the test binary as root
|
||||
sudo ./test.test --tpm-path=/dev/tpmrm0
|
||||
```
|
||||
On Linux, The `--tpm-path` causes the integration tests to be run against a
|
||||
real TPM located at that path (usually `/dev/tpmrm0` or `/dev/tpm0`). On Windows, the story is similar, execept that
|
||||
the `--use-tbs` flag is used instead.
|
||||
|
||||
Tip: if your TPM host is remote and you don't want to install Go on it, this
|
||||
same two-step process can be used. The test binary can be copied to a remote
|
||||
host and run without extra installation (as the test binary has very few
|
||||
*runtime* dependancies).
|
||||
+576
@@ -0,0 +1,576 @@
|
||||
// Copyright (c) 2018, Google LLC All rights reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package tpm2
|
||||
|
||||
import (
|
||||
"crypto"
|
||||
"crypto/elliptic"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
// Register the relevant hash implementations to prevent a runtime failure.
|
||||
_ "crypto/sha1"
|
||||
_ "crypto/sha256"
|
||||
_ "crypto/sha512"
|
||||
|
||||
"github.com/google/go-tpm/tpmutil"
|
||||
)
|
||||
|
||||
var hashInfo = []struct {
|
||||
alg Algorithm
|
||||
hash crypto.Hash
|
||||
}{
|
||||
{AlgSHA1, crypto.SHA1},
|
||||
{AlgSHA256, crypto.SHA256},
|
||||
{AlgSHA384, crypto.SHA384},
|
||||
{AlgSHA512, crypto.SHA512},
|
||||
{AlgSHA3_256, crypto.SHA3_256},
|
||||
{AlgSHA3_384, crypto.SHA3_384},
|
||||
{AlgSHA3_512, crypto.SHA3_512},
|
||||
}
|
||||
|
||||
// MAX_DIGEST_BUFFER is the maximum size of []byte request or response fields.
|
||||
// Typically used for chunking of big blobs of data (such as for hashing or
|
||||
// encryption).
|
||||
const maxDigestBuffer = 1024
|
||||
|
||||
// Algorithm represents a TPM_ALG_ID value.
|
||||
type Algorithm uint16
|
||||
|
||||
// HashToAlgorithm looks up the TPM2 algorithm corresponding to the provided crypto.Hash
|
||||
func HashToAlgorithm(hash crypto.Hash) (Algorithm, error) {
|
||||
for _, info := range hashInfo {
|
||||
if info.hash == hash {
|
||||
return info.alg, nil
|
||||
}
|
||||
}
|
||||
return AlgUnknown, fmt.Errorf("go hash algorithm #%d has no TPM2 algorithm", hash)
|
||||
}
|
||||
|
||||
// IsNull returns true if a is AlgNull or zero (unset).
|
||||
func (a Algorithm) IsNull() bool {
|
||||
return a == AlgNull || a == AlgUnknown
|
||||
}
|
||||
|
||||
// UsesCount returns true if a signature algorithm uses count value.
|
||||
func (a Algorithm) UsesCount() bool {
|
||||
return a == AlgECDAA
|
||||
}
|
||||
|
||||
// UsesHash returns true if the algorithm requires the use of a hash.
|
||||
func (a Algorithm) UsesHash() bool {
|
||||
return a == AlgOAEP
|
||||
}
|
||||
|
||||
// Hash returns a crypto.Hash based on the given TPM_ALG_ID.
|
||||
// An error is returned if the given algorithm is not a hash algorithm or is not available.
|
||||
func (a Algorithm) Hash() (crypto.Hash, error) {
|
||||
for _, info := range hashInfo {
|
||||
if info.alg == a {
|
||||
if !info.hash.Available() {
|
||||
return crypto.Hash(0), fmt.Errorf("go hash algorithm #%d not available", info.hash)
|
||||
}
|
||||
return info.hash, nil
|
||||
}
|
||||
}
|
||||
return crypto.Hash(0), fmt.Errorf("hash algorithm not supported: 0x%x", a)
|
||||
}
|
||||
|
||||
func (a Algorithm) String() string {
|
||||
var s strings.Builder
|
||||
var err error
|
||||
switch a {
|
||||
case AlgUnknown:
|
||||
_, err = s.WriteString("AlgUnknown")
|
||||
case AlgRSA:
|
||||
_, err = s.WriteString("RSA")
|
||||
case AlgSHA1:
|
||||
_, err = s.WriteString("SHA1")
|
||||
case AlgHMAC:
|
||||
_, err = s.WriteString("HMAC")
|
||||
case AlgAES:
|
||||
_, err = s.WriteString("AES")
|
||||
case AlgKeyedHash:
|
||||
_, err = s.WriteString("KeyedHash")
|
||||
case AlgXOR:
|
||||
_, err = s.WriteString("XOR")
|
||||
case AlgSHA256:
|
||||
_, err = s.WriteString("SHA256")
|
||||
case AlgSHA384:
|
||||
_, err = s.WriteString("SHA384")
|
||||
case AlgSHA512:
|
||||
_, err = s.WriteString("SHA512")
|
||||
case AlgNull:
|
||||
_, err = s.WriteString("AlgNull")
|
||||
case AlgRSASSA:
|
||||
_, err = s.WriteString("RSASSA")
|
||||
case AlgRSAES:
|
||||
_, err = s.WriteString("RSAES")
|
||||
case AlgRSAPSS:
|
||||
_, err = s.WriteString("RSAPSS")
|
||||
case AlgOAEP:
|
||||
_, err = s.WriteString("OAEP")
|
||||
case AlgECDSA:
|
||||
_, err = s.WriteString("ECDSA")
|
||||
case AlgECDH:
|
||||
_, err = s.WriteString("ECDH")
|
||||
case AlgECDAA:
|
||||
_, err = s.WriteString("ECDAA")
|
||||
case AlgKDF2:
|
||||
_, err = s.WriteString("KDF2")
|
||||
case AlgECC:
|
||||
_, err = s.WriteString("ECC")
|
||||
case AlgSymCipher:
|
||||
_, err = s.WriteString("SymCipher")
|
||||
case AlgSHA3_256:
|
||||
_, err = s.WriteString("SHA3_256")
|
||||
case AlgSHA3_384:
|
||||
_, err = s.WriteString("SHA3_384")
|
||||
case AlgSHA3_512:
|
||||
_, err = s.WriteString("SHA3_512")
|
||||
case AlgCTR:
|
||||
_, err = s.WriteString("CTR")
|
||||
case AlgOFB:
|
||||
_, err = s.WriteString("OFB")
|
||||
case AlgCBC:
|
||||
_, err = s.WriteString("CBC")
|
||||
case AlgCFB:
|
||||
_, err = s.WriteString("CFB")
|
||||
case AlgECB:
|
||||
_, err = s.WriteString("ECB")
|
||||
default:
|
||||
return fmt.Sprintf("Alg?<%d>", int(a))
|
||||
}
|
||||
if err != nil {
|
||||
return fmt.Sprintf("Writing to string builder failed: %v", err)
|
||||
}
|
||||
return s.String()
|
||||
}
|
||||
|
||||
// Supported Algorithms.
|
||||
const (
|
||||
AlgUnknown Algorithm = 0x0000
|
||||
AlgRSA Algorithm = 0x0001
|
||||
AlgSHA1 Algorithm = 0x0004
|
||||
AlgHMAC Algorithm = 0x0005
|
||||
AlgAES Algorithm = 0x0006
|
||||
AlgKeyedHash Algorithm = 0x0008
|
||||
AlgXOR Algorithm = 0x000A
|
||||
AlgSHA256 Algorithm = 0x000B
|
||||
AlgSHA384 Algorithm = 0x000C
|
||||
AlgSHA512 Algorithm = 0x000D
|
||||
AlgNull Algorithm = 0x0010
|
||||
AlgRSASSA Algorithm = 0x0014
|
||||
AlgRSAES Algorithm = 0x0015
|
||||
AlgRSAPSS Algorithm = 0x0016
|
||||
AlgOAEP Algorithm = 0x0017
|
||||
AlgECDSA Algorithm = 0x0018
|
||||
AlgECDH Algorithm = 0x0019
|
||||
AlgECDAA Algorithm = 0x001A
|
||||
AlgKDF2 Algorithm = 0x0021
|
||||
AlgECC Algorithm = 0x0023
|
||||
AlgSymCipher Algorithm = 0x0025
|
||||
AlgSHA3_256 Algorithm = 0x0027
|
||||
AlgSHA3_384 Algorithm = 0x0028
|
||||
AlgSHA3_512 Algorithm = 0x0029
|
||||
AlgCTR Algorithm = 0x0040
|
||||
AlgOFB Algorithm = 0x0041
|
||||
AlgCBC Algorithm = 0x0042
|
||||
AlgCFB Algorithm = 0x0043
|
||||
AlgECB Algorithm = 0x0044
|
||||
)
|
||||
|
||||
// HandleType defines a type of handle.
|
||||
type HandleType uint8
|
||||
|
||||
// Supported handle types
|
||||
const (
|
||||
HandleTypePCR HandleType = 0x00
|
||||
HandleTypeNVIndex HandleType = 0x01
|
||||
HandleTypeHMACSession HandleType = 0x02
|
||||
HandleTypeLoadedSession HandleType = 0x02
|
||||
HandleTypePolicySession HandleType = 0x03
|
||||
HandleTypeSavedSession HandleType = 0x03
|
||||
HandleTypePermanent HandleType = 0x40
|
||||
HandleTypeTransient HandleType = 0x80
|
||||
HandleTypePersistent HandleType = 0x81
|
||||
)
|
||||
|
||||
// SessionType defines the type of session created in StartAuthSession.
|
||||
type SessionType uint8
|
||||
|
||||
// Supported session types.
|
||||
const (
|
||||
SessionHMAC SessionType = 0x00
|
||||
SessionPolicy SessionType = 0x01
|
||||
SessionTrial SessionType = 0x03
|
||||
)
|
||||
|
||||
// SessionAttributes represents an attribute of a session.
|
||||
type SessionAttributes byte
|
||||
|
||||
// Session Attributes (Structures 8.4 TPMA_SESSION)
|
||||
const (
|
||||
AttrContinueSession SessionAttributes = 1 << iota
|
||||
AttrAuditExclusive
|
||||
AttrAuditReset
|
||||
_ // bit 3 reserved
|
||||
_ // bit 4 reserved
|
||||
AttrDecrypt
|
||||
AttrEcrypt
|
||||
AttrAudit
|
||||
)
|
||||
|
||||
// EmptyAuth represents the empty authorization value.
|
||||
var EmptyAuth []byte
|
||||
|
||||
// KeyProp is a bitmask used in Attributes field of key templates. Individual
|
||||
// flags should be OR-ed to form a full mask.
|
||||
type KeyProp uint32
|
||||
|
||||
// Key properties.
|
||||
const (
|
||||
FlagFixedTPM KeyProp = 0x00000002
|
||||
FlagStClear KeyProp = 0x00000004
|
||||
FlagFixedParent KeyProp = 0x00000010
|
||||
FlagSensitiveDataOrigin KeyProp = 0x00000020
|
||||
FlagUserWithAuth KeyProp = 0x00000040
|
||||
FlagAdminWithPolicy KeyProp = 0x00000080
|
||||
FlagNoDA KeyProp = 0x00000400
|
||||
FlagRestricted KeyProp = 0x00010000
|
||||
FlagDecrypt KeyProp = 0x00020000
|
||||
FlagSign KeyProp = 0x00040000
|
||||
|
||||
FlagSealDefault = FlagFixedTPM | FlagFixedParent
|
||||
FlagSignerDefault = FlagSign | FlagRestricted | FlagFixedTPM |
|
||||
FlagFixedParent | FlagSensitiveDataOrigin | FlagUserWithAuth
|
||||
FlagStorageDefault = FlagDecrypt | FlagRestricted | FlagFixedTPM |
|
||||
FlagFixedParent | FlagSensitiveDataOrigin | FlagUserWithAuth
|
||||
)
|
||||
|
||||
// TPMProp represents a Property Tag (TPM_PT) used with calls to GetCapability(CapabilityTPMProperties).
|
||||
type TPMProp uint32
|
||||
|
||||
// TPM Capability Properties, see TPM 2.0 Spec, Rev 1.38, Table 23.
|
||||
// Fixed TPM Properties (PT_FIXED)
|
||||
const (
|
||||
FamilyIndicator TPMProp = 0x100 + iota
|
||||
SpecLevel
|
||||
SpecRevision
|
||||
SpecDayOfYear
|
||||
SpecYear
|
||||
Manufacturer
|
||||
VendorString1
|
||||
VendorString2
|
||||
VendorString3
|
||||
VendorString4
|
||||
VendorTPMType
|
||||
FirmwareVersion1
|
||||
FirmwareVersion2
|
||||
InputMaxBufferSize
|
||||
TransientObjectsMin
|
||||
PersistentObjectsMin
|
||||
LoadedObjectsMin
|
||||
ActiveSessionsMax
|
||||
PCRCount
|
||||
PCRSelectMin
|
||||
ContextGapMax
|
||||
_ // (PT_FIXED + 21) is skipped
|
||||
NVCountersMax
|
||||
NVIndexMax
|
||||
MemoryMethod
|
||||
ClockUpdate
|
||||
ContextHash
|
||||
ContextSym
|
||||
ContextSymSize
|
||||
OrderlyCount
|
||||
CommandMaxSize
|
||||
ResponseMaxSize
|
||||
DigestMaxSize
|
||||
ObjectContextMaxSize
|
||||
SessionContextMaxSize
|
||||
PSFamilyIndicator
|
||||
PSSpecLevel
|
||||
PSSpecRevision
|
||||
PSSpecDayOfYear
|
||||
PSSpecYear
|
||||
SplitSigningMax
|
||||
TotalCommands
|
||||
LibraryCommands
|
||||
VendorCommands
|
||||
NVMaxBufferSize
|
||||
TPMModes
|
||||
CapabilityMaxBufferSize
|
||||
)
|
||||
|
||||
// Variable TPM Properties (PT_VAR)
|
||||
const (
|
||||
TPMAPermanent TPMProp = 0x200 + iota
|
||||
TPMAStartupClear
|
||||
HRNVIndex
|
||||
HRLoaded
|
||||
HRLoadedAvail
|
||||
HRActive
|
||||
HRActiveAvail
|
||||
HRTransientAvail
|
||||
CurrentPersistent
|
||||
AvailPersistent
|
||||
NVCounters
|
||||
NVCountersAvail
|
||||
AlgorithmSet
|
||||
LoadedCurves
|
||||
LockoutCounter
|
||||
MaxAuthFail
|
||||
LockoutInterval
|
||||
LockoutRecovery
|
||||
NVWriteRecovery
|
||||
AuditCounter0
|
||||
AuditCounter1
|
||||
)
|
||||
|
||||
// Allowed ranges of different kinds of Handles (TPM_HANDLE)
|
||||
// These constants have type TPMProp for backwards compatibility.
|
||||
const (
|
||||
PCRFirst TPMProp = 0x00000000
|
||||
HMACSessionFirst TPMProp = 0x02000000
|
||||
LoadedSessionFirst TPMProp = 0x02000000
|
||||
PolicySessionFirst TPMProp = 0x03000000
|
||||
ActiveSessionFirst TPMProp = 0x03000000
|
||||
TransientFirst TPMProp = 0x80000000
|
||||
PersistentFirst TPMProp = 0x81000000
|
||||
PersistentLast TPMProp = 0x81FFFFFF
|
||||
PlatformPersistent TPMProp = 0x81800000
|
||||
NVIndexFirst TPMProp = 0x01000000
|
||||
NVIndexLast TPMProp = 0x01FFFFFF
|
||||
PermanentFirst TPMProp = 0x40000000
|
||||
PermanentLast TPMProp = 0x4000010F
|
||||
)
|
||||
|
||||
// Reserved Handles.
|
||||
const (
|
||||
HandleOwner tpmutil.Handle = 0x40000001 + iota
|
||||
HandleRevoke
|
||||
HandleTransport
|
||||
HandleOperator
|
||||
HandleAdmin
|
||||
HandleEK
|
||||
HandleNull
|
||||
HandleUnassigned
|
||||
HandlePasswordSession
|
||||
HandleLockout
|
||||
HandleEndorsement
|
||||
HandlePlatform
|
||||
)
|
||||
|
||||
// Capability identifies some TPM property or state type.
|
||||
type Capability uint32
|
||||
|
||||
// TPM Capabilities.
|
||||
const (
|
||||
CapabilityAlgs Capability = iota
|
||||
CapabilityHandles
|
||||
CapabilityCommands
|
||||
CapabilityPPCommands
|
||||
CapabilityAuditCommands
|
||||
CapabilityPCRs
|
||||
CapabilityTPMProperties
|
||||
CapabilityPCRProperties
|
||||
CapabilityECCCurves
|
||||
CapabilityAuthPolicies
|
||||
)
|
||||
|
||||
// TPM Structure Tags. Tags are used to disambiguate structures, similar to Alg
|
||||
// values: tag value defines what kind of data lives in a nested field.
|
||||
const (
|
||||
TagNull tpmutil.Tag = 0x8000
|
||||
TagNoSessions tpmutil.Tag = 0x8001
|
||||
TagSessions tpmutil.Tag = 0x8002
|
||||
TagAttestCertify tpmutil.Tag = 0x8017
|
||||
TagAttestQuote tpmutil.Tag = 0x8018
|
||||
TagAttestCreation tpmutil.Tag = 0x801a
|
||||
TagAuthSecret tpmutil.Tag = 0x8023
|
||||
TagHashCheck tpmutil.Tag = 0x8024
|
||||
TagAuthSigned tpmutil.Tag = 0x8025
|
||||
)
|
||||
|
||||
// StartupType instructs the TPM on how to handle its state during Shutdown or
|
||||
// Startup.
|
||||
type StartupType uint16
|
||||
|
||||
// Startup types
|
||||
const (
|
||||
StartupClear StartupType = iota
|
||||
StartupState
|
||||
)
|
||||
|
||||
// EllipticCurve identifies specific EC curves.
|
||||
type EllipticCurve uint16
|
||||
|
||||
// ECC curves supported by TPM 2.0 spec.
|
||||
const (
|
||||
CurveNISTP192 = EllipticCurve(iota + 1)
|
||||
CurveNISTP224
|
||||
CurveNISTP256
|
||||
CurveNISTP384
|
||||
CurveNISTP521
|
||||
|
||||
CurveBNP256 = EllipticCurve(iota + 10)
|
||||
CurveBNP638
|
||||
|
||||
CurveSM2P256 = EllipticCurve(0x0020)
|
||||
)
|
||||
|
||||
var toGoCurve = map[EllipticCurve]elliptic.Curve{
|
||||
CurveNISTP224: elliptic.P224(),
|
||||
CurveNISTP256: elliptic.P256(),
|
||||
CurveNISTP384: elliptic.P384(),
|
||||
CurveNISTP521: elliptic.P521(),
|
||||
}
|
||||
|
||||
// Supported TPM operations.
|
||||
const (
|
||||
CmdNVUndefineSpaceSpecial tpmutil.Command = 0x0000011F
|
||||
CmdEvictControl tpmutil.Command = 0x00000120
|
||||
CmdUndefineSpace tpmutil.Command = 0x00000122
|
||||
CmdClear tpmutil.Command = 0x00000126
|
||||
CmdHierarchyChangeAuth tpmutil.Command = 0x00000129
|
||||
CmdDefineSpace tpmutil.Command = 0x0000012A
|
||||
CmdPCRAllocate tpmutil.Command = 0x0000012B
|
||||
CmdCreatePrimary tpmutil.Command = 0x00000131
|
||||
CmdIncrementNVCounter tpmutil.Command = 0x00000134
|
||||
CmdWriteNV tpmutil.Command = 0x00000137
|
||||
CmdWriteLockNV tpmutil.Command = 0x00000138
|
||||
CmdDictionaryAttackLockReset tpmutil.Command = 0x00000139
|
||||
CmdDictionaryAttackParameters tpmutil.Command = 0x0000013A
|
||||
CmdPCREvent tpmutil.Command = 0x0000013C
|
||||
CmdPCRReset tpmutil.Command = 0x0000013D
|
||||
CmdSequenceComplete tpmutil.Command = 0x0000013E
|
||||
CmdStartup tpmutil.Command = 0x00000144
|
||||
CmdShutdown tpmutil.Command = 0x00000145
|
||||
CmdActivateCredential tpmutil.Command = 0x00000147
|
||||
CmdCertify tpmutil.Command = 0x00000148
|
||||
CmdCertifyCreation tpmutil.Command = 0x0000014A
|
||||
CmdReadNV tpmutil.Command = 0x0000014E
|
||||
CmdReadLockNV tpmutil.Command = 0x0000014F
|
||||
CmdPolicySecret tpmutil.Command = 0x00000151
|
||||
CmdCreate tpmutil.Command = 0x00000153
|
||||
CmdECDHZGen tpmutil.Command = 0x00000154
|
||||
CmdImport tpmutil.Command = 0x00000156
|
||||
CmdLoad tpmutil.Command = 0x00000157
|
||||
CmdQuote tpmutil.Command = 0x00000158
|
||||
CmdRSADecrypt tpmutil.Command = 0x00000159
|
||||
CmdSequenceUpdate tpmutil.Command = 0x0000015C
|
||||
CmdSign tpmutil.Command = 0x0000015D
|
||||
CmdUnseal tpmutil.Command = 0x0000015E
|
||||
CmdPolicySigned tpmutil.Command = 0x00000160
|
||||
CmdContextLoad tpmutil.Command = 0x00000161
|
||||
CmdContextSave tpmutil.Command = 0x00000162
|
||||
CmdECDHKeyGen tpmutil.Command = 0x00000163
|
||||
CmdEncryptDecrypt tpmutil.Command = 0x00000164
|
||||
CmdFlushContext tpmutil.Command = 0x00000165
|
||||
CmdLoadExternal tpmutil.Command = 0x00000167
|
||||
CmdMakeCredential tpmutil.Command = 0x00000168
|
||||
CmdReadPublicNV tpmutil.Command = 0x00000169
|
||||
CmdPolicyCommandCode tpmutil.Command = 0x0000016C
|
||||
CmdPolicyOr tpmutil.Command = 0x00000171
|
||||
CmdReadPublic tpmutil.Command = 0x00000173
|
||||
CmdRSAEncrypt tpmutil.Command = 0x00000174
|
||||
CmdStartAuthSession tpmutil.Command = 0x00000176
|
||||
CmdGetCapability tpmutil.Command = 0x0000017A
|
||||
CmdGetRandom tpmutil.Command = 0x0000017B
|
||||
CmdHash tpmutil.Command = 0x0000017D
|
||||
CmdPCRRead tpmutil.Command = 0x0000017E
|
||||
CmdPolicyPCR tpmutil.Command = 0x0000017F
|
||||
CmdReadClock tpmutil.Command = 0x00000181
|
||||
CmdPCRExtend tpmutil.Command = 0x00000182
|
||||
CmdEventSequenceComplete tpmutil.Command = 0x00000185
|
||||
CmdHashSequenceStart tpmutil.Command = 0x00000186
|
||||
CmdPolicyGetDigest tpmutil.Command = 0x00000189
|
||||
CmdPolicyPassword tpmutil.Command = 0x0000018C
|
||||
CmdEncryptDecrypt2 tpmutil.Command = 0x00000193
|
||||
)
|
||||
|
||||
// Regular TPM 2.0 devices use 24-bit mask (3 bytes) for PCR selection.
|
||||
const sizeOfPCRSelect = 3
|
||||
|
||||
const defaultRSAExponent = 1<<16 + 1
|
||||
|
||||
// NVAttr is a bitmask used in Attributes field of NV indexes. Individual
|
||||
// flags should be OR-ed to form a full mask.
|
||||
type NVAttr uint32
|
||||
|
||||
// NV Attributes
|
||||
const (
|
||||
AttrPPWrite NVAttr = 0x00000001
|
||||
AttrOwnerWrite NVAttr = 0x00000002
|
||||
AttrAuthWrite NVAttr = 0x00000004
|
||||
AttrPolicyWrite NVAttr = 0x00000008
|
||||
AttrPolicyDelete NVAttr = 0x00000400
|
||||
AttrWriteLocked NVAttr = 0x00000800
|
||||
AttrWriteAll NVAttr = 0x00001000
|
||||
AttrWriteDefine NVAttr = 0x00002000
|
||||
AttrWriteSTClear NVAttr = 0x00004000
|
||||
AttrGlobalLock NVAttr = 0x00008000
|
||||
AttrPPRead NVAttr = 0x00010000
|
||||
AttrOwnerRead NVAttr = 0x00020000
|
||||
AttrAuthRead NVAttr = 0x00040000
|
||||
AttrPolicyRead NVAttr = 0x00080000
|
||||
AttrNoDA NVAttr = 0x02000000
|
||||
AttrOrderly NVAttr = 0x04000000
|
||||
AttrClearSTClear NVAttr = 0x08000000
|
||||
AttrReadLocked NVAttr = 0x10000000
|
||||
AttrWritten NVAttr = 0x20000000
|
||||
AttrPlatformCreate NVAttr = 0x40000000
|
||||
AttrReadSTClear NVAttr = 0x80000000
|
||||
)
|
||||
|
||||
var permMap = map[NVAttr]string{
|
||||
AttrPPWrite: "PPWrite",
|
||||
AttrOwnerWrite: "OwnerWrite",
|
||||
AttrAuthWrite: "AuthWrite",
|
||||
AttrPolicyWrite: "PolicyWrite",
|
||||
AttrPolicyDelete: "PolicyDelete",
|
||||
AttrWriteLocked: "WriteLocked",
|
||||
AttrWriteAll: "WriteAll",
|
||||
AttrWriteDefine: "WriteDefine",
|
||||
AttrWriteSTClear: "WriteSTClear",
|
||||
AttrGlobalLock: "GlobalLock",
|
||||
AttrPPRead: "PPRead",
|
||||
AttrOwnerRead: "OwnerRead",
|
||||
AttrAuthRead: "AuthRead",
|
||||
AttrPolicyRead: "PolicyRead",
|
||||
AttrNoDA: "No Do",
|
||||
AttrOrderly: "Oderly",
|
||||
AttrClearSTClear: "ClearSTClear",
|
||||
AttrReadLocked: "ReadLocked",
|
||||
AttrWritten: "Writte",
|
||||
AttrPlatformCreate: "PlatformCreate",
|
||||
AttrReadSTClear: "ReadSTClear",
|
||||
}
|
||||
|
||||
// String returns a textual representation of the set of NVAttr
|
||||
func (p NVAttr) String() string {
|
||||
var retString strings.Builder
|
||||
for iterator, item := range permMap {
|
||||
if (p & iterator) != 0 {
|
||||
retString.WriteString(item + " + ")
|
||||
}
|
||||
}
|
||||
if retString.String() == "" {
|
||||
return "Permission/s not found"
|
||||
}
|
||||
return strings.TrimSuffix(retString.String(), " + ")
|
||||
|
||||
}
|
||||
+362
@@ -0,0 +1,362 @@
|
||||
package tpm2
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"github.com/google/go-tpm/tpmutil"
|
||||
)
|
||||
|
||||
type (
|
||||
// RCFmt0 holds Format 0 error codes
|
||||
RCFmt0 uint8
|
||||
|
||||
// RCFmt1 holds Format 1 error codes
|
||||
RCFmt1 uint8
|
||||
|
||||
// RCWarn holds error codes used in warnings
|
||||
RCWarn uint8
|
||||
|
||||
// RCIndex is used to reference arguments, handles and sessions in errors
|
||||
RCIndex uint8
|
||||
)
|
||||
|
||||
// Format 0 error codes.
|
||||
const (
|
||||
RCInitialize RCFmt0 = 0x00
|
||||
RCFailure RCFmt0 = 0x01
|
||||
RCSequence RCFmt0 = 0x03
|
||||
RCPrivate RCFmt0 = 0x0B
|
||||
RCHMAC RCFmt0 = 0x19
|
||||
RCDisabled RCFmt0 = 0x20
|
||||
RCExclusive RCFmt0 = 0x21
|
||||
RCAuthType RCFmt0 = 0x24
|
||||
RCAuthMissing RCFmt0 = 0x25
|
||||
RCPolicy RCFmt0 = 0x26
|
||||
RCPCR RCFmt0 = 0x27
|
||||
RCPCRChanged RCFmt0 = 0x28
|
||||
RCUpgrade RCFmt0 = 0x2D
|
||||
RCTooManyContexts RCFmt0 = 0x2E
|
||||
RCAuthUnavailable RCFmt0 = 0x2F
|
||||
RCReboot RCFmt0 = 0x30
|
||||
RCUnbalanced RCFmt0 = 0x31
|
||||
RCCommandSize RCFmt0 = 0x42
|
||||
RCCommandCode RCFmt0 = 0x43
|
||||
RCAuthSize RCFmt0 = 0x44
|
||||
RCAuthContext RCFmt0 = 0x45
|
||||
RCNVRange RCFmt0 = 0x46
|
||||
RCNVSize RCFmt0 = 0x47
|
||||
RCNVLocked RCFmt0 = 0x48
|
||||
RCNVAuthorization RCFmt0 = 0x49
|
||||
RCNVUninitialized RCFmt0 = 0x4A
|
||||
RCNVSpace RCFmt0 = 0x4B
|
||||
RCNVDefined RCFmt0 = 0x4C
|
||||
RCBadContext RCFmt0 = 0x50
|
||||
RCCPHash RCFmt0 = 0x51
|
||||
RCParent RCFmt0 = 0x52
|
||||
RCNeedsTest RCFmt0 = 0x53
|
||||
RCNoResult RCFmt0 = 0x54
|
||||
RCSensitive RCFmt0 = 0x55
|
||||
)
|
||||
|
||||
var fmt0Msg = map[RCFmt0]string{
|
||||
RCInitialize: "TPM not initialized by TPM2_Startup or already initialized",
|
||||
RCFailure: "commands not being accepted because of a TPM failure",
|
||||
RCSequence: "improper use of a sequence handle",
|
||||
RCPrivate: "not currently used",
|
||||
RCHMAC: "not currently used",
|
||||
RCDisabled: "the command is disabled",
|
||||
RCExclusive: "command failed because audit sequence required exclusivity",
|
||||
RCAuthType: "authorization handle is not correct for command",
|
||||
RCAuthMissing: "5 command requires an authorization session for handle and it is not present",
|
||||
RCPolicy: "policy failure in math operation or an invalid authPolicy value",
|
||||
RCPCR: "PCR check fail",
|
||||
RCPCRChanged: "PCR have changed since checked",
|
||||
RCUpgrade: "TPM is in field upgrade mode unless called via TPM2_FieldUpgradeData(), then it is not in field upgrade mode",
|
||||
RCTooManyContexts: "context ID counter is at maximum",
|
||||
RCAuthUnavailable: "authValue or authPolicy is not available for selected entity",
|
||||
RCReboot: "a _TPM_Init and Startup(CLEAR) is required before the TPM can resume operation",
|
||||
RCUnbalanced: "the protection algorithms (hash and symmetric) are not reasonably balanced; the digest size of the hash must be larger than the key size of the symmetric algorithm",
|
||||
RCCommandSize: "command commandSize value is inconsistent with contents of the command buffer; either the size is not the same as the octets loaded by the hardware interface layer or the value is not large enough to hold a command header",
|
||||
RCCommandCode: "command code not supported",
|
||||
RCAuthSize: "the value of authorizationSize is out of range or the number of octets in the Authorization Area is greater than required",
|
||||
RCAuthContext: "use of an authorization session with a context command or another command that cannot have an authorization session",
|
||||
RCNVRange: "NV offset+size is out of range",
|
||||
RCNVSize: "Requested allocation size is larger than allowed",
|
||||
RCNVLocked: "NV access locked",
|
||||
RCNVAuthorization: "NV access authorization fails in command actions",
|
||||
RCNVUninitialized: "an NV Index is used before being initialized or the state saved by TPM2_Shutdown(STATE) could not be restored",
|
||||
RCNVSpace: "insufficient space for NV allocation",
|
||||
RCNVDefined: "NV Index or persistent object already defined",
|
||||
RCBadContext: "context in TPM2_ContextLoad() is not valid",
|
||||
RCCPHash: "cpHash value already set or not correct for use",
|
||||
RCParent: "handle for parent is not a valid parent",
|
||||
RCNeedsTest: "some function needs testing",
|
||||
RCNoResult: "returned when an internal function cannot process a request due to an unspecified problem; this code is usually related to invalid parameters that are not properly filtered by the input unmarshaling code",
|
||||
RCSensitive: "the sensitive area did not unmarshal correctly after decryption",
|
||||
}
|
||||
|
||||
// Format 1 error codes.
|
||||
const (
|
||||
RCAsymmetric = 0x01
|
||||
RCAttributes = 0x02
|
||||
RCHash = 0x03
|
||||
RCValue = 0x04
|
||||
RCHierarchy = 0x05
|
||||
RCKeySize = 0x07
|
||||
RCMGF = 0x08
|
||||
RCMode = 0x09
|
||||
RCType = 0x0A
|
||||
RCHandle = 0x0B
|
||||
RCKDF = 0x0C
|
||||
RCRange = 0x0D
|
||||
RCAuthFail = 0x0E
|
||||
RCNonce = 0x0F
|
||||
RCPP = 0x10
|
||||
RCScheme = 0x12
|
||||
RCSize = 0x15
|
||||
RCSymmetric = 0x16
|
||||
RCTag = 0x17
|
||||
RCSelector = 0x18
|
||||
RCInsufficient = 0x1A
|
||||
RCSignature = 0x1B
|
||||
RCKey = 0x1C
|
||||
RCPolicyFail = 0x1D
|
||||
RCIntegrity = 0x1F
|
||||
RCTicket = 0x20
|
||||
RCReservedBits = 0x21
|
||||
RCBadAuth = 0x22
|
||||
RCExpired = 0x23
|
||||
RCPolicyCC = 0x24
|
||||
RCBinding = 0x25
|
||||
RCCurve = 0x26
|
||||
RCECCPoint = 0x27
|
||||
)
|
||||
|
||||
var fmt1Msg = map[RCFmt1]string{
|
||||
RCAsymmetric: "asymmetric algorithm not supported or not correct",
|
||||
RCAttributes: "inconsistent attributes",
|
||||
RCHash: "hash algorithm not supported or not appropriate",
|
||||
RCValue: "value is out of range or is not correct for the context",
|
||||
RCHierarchy: "hierarchy is not enabled or is not correct for the use",
|
||||
RCKeySize: "key size is not supported",
|
||||
RCMGF: "mask generation function not supported",
|
||||
RCMode: "mode of operation not supported",
|
||||
RCType: "the type of the value is not appropriate for the use",
|
||||
RCHandle: "the handle is not correct for the use",
|
||||
RCKDF: "unsupported key derivation function or function not appropriate for use",
|
||||
RCRange: "value was out of allowed range",
|
||||
RCAuthFail: "the authorization HMAC check failed and DA counter incremented",
|
||||
RCNonce: "invalid nonce size or nonce value mismatch",
|
||||
RCPP: "authorization requires assertion of PP",
|
||||
RCScheme: "unsupported or incompatible scheme",
|
||||
RCSize: "structure is the wrong size",
|
||||
RCSymmetric: "unsupported symmetric algorithm or key size, or not appropriate for instance",
|
||||
RCTag: "incorrect structure tag",
|
||||
RCSelector: "union selector is incorrect",
|
||||
RCInsufficient: "the TPM was unable to unmarshal a value because there were not enough octets in the input buffer",
|
||||
RCSignature: "the signature is not valid",
|
||||
RCKey: "key fields are not compatible with the selected use",
|
||||
RCPolicyFail: "a policy check failed",
|
||||
RCIntegrity: "integrity check failed",
|
||||
RCTicket: "invalid ticket",
|
||||
RCReservedBits: "reserved bits not set to zero as required",
|
||||
RCBadAuth: "authorization failure without DA implications",
|
||||
RCExpired: "the policy has expired",
|
||||
RCPolicyCC: "the commandCode in the policy is not the commandCode of the command or the command code in a policy command references a command that is not implemented",
|
||||
RCBinding: "public and sensitive portions of an object are not cryptographically bound",
|
||||
RCCurve: "curve not supported",
|
||||
RCECCPoint: "point is not on the required curve",
|
||||
}
|
||||
|
||||
// Warning codes.
|
||||
const (
|
||||
RCContextGap RCWarn = 0x01
|
||||
RCObjectMemory RCWarn = 0x02
|
||||
RCSessionMemory RCWarn = 0x03
|
||||
RCMemory RCWarn = 0x04
|
||||
RCSessionHandles RCWarn = 0x05
|
||||
RCObjectHandles RCWarn = 0x06
|
||||
RCLocality RCWarn = 0x07
|
||||
RCYielded RCWarn = 0x08
|
||||
RCCanceled RCWarn = 0x09
|
||||
RCTesting RCWarn = 0x0A
|
||||
RCReferenceH0 RCWarn = 0x10
|
||||
RCReferenceH1 RCWarn = 0x11
|
||||
RCReferenceH2 RCWarn = 0x12
|
||||
RCReferenceH3 RCWarn = 0x13
|
||||
RCReferenceH4 RCWarn = 0x14
|
||||
RCReferenceH5 RCWarn = 0x15
|
||||
RCReferenceH6 RCWarn = 0x16
|
||||
RCReferenceS0 RCWarn = 0x18
|
||||
RCReferenceS1 RCWarn = 0x19
|
||||
RCReferenceS2 RCWarn = 0x1A
|
||||
RCReferenceS3 RCWarn = 0x1B
|
||||
RCReferenceS4 RCWarn = 0x1C
|
||||
RCReferenceS5 RCWarn = 0x1D
|
||||
RCReferenceS6 RCWarn = 0x1E
|
||||
RCNVRate RCWarn = 0x20
|
||||
RCLockout RCWarn = 0x21
|
||||
RCRetry RCWarn = 0x22
|
||||
RCNVUnavailable RCWarn = 0x23
|
||||
)
|
||||
|
||||
var warnMsg = map[RCWarn]string{
|
||||
RCContextGap: "gap for context ID is too large",
|
||||
RCObjectMemory: "out of memory for object contexts",
|
||||
RCSessionMemory: "out of memory for session contexts",
|
||||
RCMemory: "out of shared object/session memory or need space for internal operations",
|
||||
RCSessionHandles: "out of session handles",
|
||||
RCObjectHandles: "out of object handles",
|
||||
RCLocality: "bad locality",
|
||||
RCYielded: "the TPM has suspended operation on the command; forward progress was made and the command may be retried",
|
||||
RCCanceled: "the command was canceled",
|
||||
RCTesting: "TPM is performing self-tests",
|
||||
RCReferenceH0: "the 1st handle in the handle area references a transient object or session that is not loaded",
|
||||
RCReferenceH1: "the 2nd handle in the handle area references a transient object or session that is not loaded",
|
||||
RCReferenceH2: "the 3rd handle in the handle area references a transient object or session that is not loaded",
|
||||
RCReferenceH3: "the 4th handle in the handle area references a transient object or session that is not loaded",
|
||||
RCReferenceH4: "the 5th handle in the handle area references a transient object or session that is not loaded",
|
||||
RCReferenceH5: "the 6th handle in the handle area references a transient object or session that is not loaded",
|
||||
RCReferenceH6: "the 7th handle in the handle area references a transient object or session that is not loaded",
|
||||
RCReferenceS0: "the 1st authorization session handle references a session that is not loaded",
|
||||
RCReferenceS1: "the 2nd authorization session handle references a session that is not loaded",
|
||||
RCReferenceS2: "the 3rd authorization session handle references a session that is not loaded",
|
||||
RCReferenceS3: "the 4th authorization session handle references a session that is not loaded",
|
||||
RCReferenceS4: "the 5th authorization session handle references a session that is not loaded",
|
||||
RCReferenceS5: "the 6th authorization session handle references a session that is not loaded",
|
||||
RCReferenceS6: "the 7th authorization session handle references a session that is not loaded",
|
||||
RCNVRate: "the TPM is rate-limiting accesses to prevent wearout of NV",
|
||||
RCLockout: "authorizations for objects subject to DA protection are not allowed at this time because the TPM is in DA lockout mode",
|
||||
RCRetry: "the TPM was not able to start the command",
|
||||
RCNVUnavailable: "the command may require writing of NV and NV is not current accessible",
|
||||
}
|
||||
|
||||
// Indexes for arguments, handles and sessions.
|
||||
const (
|
||||
RC1 RCIndex = iota + 1
|
||||
RC2
|
||||
RC3
|
||||
RC4
|
||||
RC5
|
||||
RC6
|
||||
RC7
|
||||
RC8
|
||||
RC9
|
||||
RCA
|
||||
RCB
|
||||
RCC
|
||||
RCD
|
||||
RCE
|
||||
RCF
|
||||
)
|
||||
|
||||
const unknownCode = "unknown error code"
|
||||
|
||||
// Error is returned for all Format 0 errors from the TPM. It is used for general
|
||||
// errors not specific to a parameter, handle or session.
|
||||
type Error struct {
|
||||
Code RCFmt0
|
||||
}
|
||||
|
||||
func (e Error) Error() string {
|
||||
msg := fmt0Msg[e.Code]
|
||||
if msg == "" {
|
||||
msg = unknownCode
|
||||
}
|
||||
return fmt.Sprintf("error code 0x%x : %s", e.Code, msg)
|
||||
}
|
||||
|
||||
// VendorError represents a vendor-specific error response. These types of responses
|
||||
// are not decoded and Code contains the complete response code.
|
||||
type VendorError struct {
|
||||
Code uint32
|
||||
}
|
||||
|
||||
func (e VendorError) Error() string {
|
||||
return fmt.Sprintf("vendor error code 0x%x", e.Code)
|
||||
}
|
||||
|
||||
// Warning is typically used to report transient errors.
|
||||
type Warning struct {
|
||||
Code RCWarn
|
||||
}
|
||||
|
||||
func (w Warning) Error() string {
|
||||
msg := warnMsg[w.Code]
|
||||
if msg == "" {
|
||||
msg = unknownCode
|
||||
}
|
||||
return fmt.Sprintf("warning code 0x%x : %s", w.Code, msg)
|
||||
}
|
||||
|
||||
// ParameterError describes an error related to a parameter, and the parameter number.
|
||||
type ParameterError struct {
|
||||
Code RCFmt1
|
||||
Parameter RCIndex
|
||||
}
|
||||
|
||||
func (e ParameterError) Error() string {
|
||||
msg := fmt1Msg[e.Code]
|
||||
if msg == "" {
|
||||
msg = unknownCode
|
||||
}
|
||||
return fmt.Sprintf("parameter %d, error code 0x%x : %s", e.Parameter, e.Code, msg)
|
||||
}
|
||||
|
||||
// HandleError describes an error related to a handle, and the handle number.
|
||||
type HandleError struct {
|
||||
Code RCFmt1
|
||||
Handle RCIndex
|
||||
}
|
||||
|
||||
func (e HandleError) Error() string {
|
||||
msg := fmt1Msg[e.Code]
|
||||
if msg == "" {
|
||||
msg = unknownCode
|
||||
}
|
||||
return fmt.Sprintf("handle %d, error code 0x%x : %s", e.Handle, e.Code, msg)
|
||||
}
|
||||
|
||||
// SessionError describes an error related to a session, and the session number.
|
||||
type SessionError struct {
|
||||
Code RCFmt1
|
||||
Session RCIndex
|
||||
}
|
||||
|
||||
func (e SessionError) Error() string {
|
||||
msg := fmt1Msg[e.Code]
|
||||
if msg == "" {
|
||||
msg = unknownCode
|
||||
}
|
||||
return fmt.Sprintf("session %d, error code 0x%x : %s", e.Session, e.Code, msg)
|
||||
}
|
||||
|
||||
// Decode a TPM2 response code and return the appropriate error. Logic
|
||||
// according to the "Response Code Evaluation" chart in Part 1 of the TPM 2.0
|
||||
// spec.
|
||||
func decodeResponse(code tpmutil.ResponseCode) error {
|
||||
if code == tpmutil.RCSuccess {
|
||||
return nil
|
||||
}
|
||||
if code&0x180 == 0 { // Bits 7:8 == 0 is a TPM1 error
|
||||
return fmt.Errorf("response status 0x%x", code)
|
||||
}
|
||||
if code&0x80 == 0 { // Bit 7 unset
|
||||
if code&0x400 > 0 { // Bit 10 set, vendor specific code
|
||||
return VendorError{uint32(code)}
|
||||
}
|
||||
if code&0x800 > 0 { // Bit 11 set, warning with code in bit 0:6
|
||||
return Warning{RCWarn(code & 0x7f)}
|
||||
}
|
||||
// error with code in bit 0:6
|
||||
return Error{RCFmt0(code & 0x7f)}
|
||||
}
|
||||
if code&0x40 > 0 { // Bit 6 set, code in 0:5, parameter number in 8:11
|
||||
return ParameterError{RCFmt1(code & 0x3f), RCIndex((code & 0xf00) >> 8)}
|
||||
}
|
||||
if code&0x800 == 0 { // Bit 11 unset, code in 0:5, handle in 8:10
|
||||
return HandleError{RCFmt1(code & 0x3f), RCIndex((code & 0x700) >> 8)}
|
||||
}
|
||||
// Code in 0:5, Session in 8:10
|
||||
return SessionError{RCFmt1(code & 0x3f), RCIndex((code & 0x700) >> 8)}
|
||||
}
|
||||
+116
@@ -0,0 +1,116 @@
|
||||
// Copyright (c) 2018, Google LLC All rights reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package tpm2
|
||||
|
||||
import (
|
||||
"crypto"
|
||||
"crypto/hmac"
|
||||
"encoding/binary"
|
||||
"hash"
|
||||
)
|
||||
|
||||
// KDFa implements TPM 2.0's default key derivation function, as defined in
|
||||
// section 11.4.9.2 of the TPM revision 2 specification part 1.
|
||||
// See: https://trustedcomputinggroup.org/resource/tpm-library-specification/
|
||||
// The key & label parameters must not be zero length.
|
||||
// The label parameter is a non-null-terminated string.
|
||||
// The contextU & contextV parameters are optional.
|
||||
// Deprecated: Use KDFaHash.
|
||||
func KDFa(hashAlg Algorithm, key []byte, label string, contextU, contextV []byte, bits int) ([]byte, error) {
|
||||
h, err := hashAlg.Hash()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return KDFaHash(h, key, label, contextU, contextV, bits), nil
|
||||
}
|
||||
|
||||
// KDFe implements TPM 2.0's ECDH key derivation function, as defined in
|
||||
// section 11.4.9.3 of the TPM revision 2 specification part 1.
|
||||
// See: https://trustedcomputinggroup.org/resource/tpm-library-specification/
|
||||
// The z parameter is the x coordinate of one party's private ECC key multiplied
|
||||
// by the other party's public ECC point.
|
||||
// The use parameter is a non-null-terminated string.
|
||||
// The partyUInfo and partyVInfo are the x coordinates of the initiator's and
|
||||
// Deprecated: Use KDFeHash.
|
||||
func KDFe(hashAlg Algorithm, z []byte, use string, partyUInfo, partyVInfo []byte, bits int) ([]byte, error) {
|
||||
h, err := hashAlg.Hash()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return KDFeHash(h, z, use, partyUInfo, partyVInfo, bits), nil
|
||||
}
|
||||
|
||||
// KDFaHash implements TPM 2.0's default key derivation function, as defined in
|
||||
// section 11.4.9.2 of the TPM revision 2 specification part 1.
|
||||
// See: https://trustedcomputinggroup.org/resource/tpm-library-specification/
|
||||
// The key & label parameters must not be zero length.
|
||||
// The label parameter is a non-null-terminated string.
|
||||
// The contextU & contextV parameters are optional.
|
||||
func KDFaHash(h crypto.Hash, key []byte, label string, contextU, contextV []byte, bits int) []byte {
|
||||
mac := hmac.New(h.New, key)
|
||||
|
||||
out := kdf(mac, bits, func() {
|
||||
mac.Write([]byte(label))
|
||||
mac.Write([]byte{0}) // Terminating null character for C-string.
|
||||
mac.Write(contextU)
|
||||
mac.Write(contextV)
|
||||
binary.Write(mac, binary.BigEndian, uint32(bits))
|
||||
})
|
||||
return out
|
||||
}
|
||||
|
||||
// KDFeHash implements TPM 2.0's ECDH key derivation function, as defined in
|
||||
// section 11.4.9.3 of the TPM revision 2 specification part 1.
|
||||
// See: https://trustedcomputinggroup.org/resource/tpm-library-specification/
|
||||
// The z parameter is the x coordinate of one party's private ECC key multiplied
|
||||
// by the other party's public ECC point.
|
||||
// The use parameter is a non-null-terminated string.
|
||||
// The partyUInfo and partyVInfo are the x coordinates of the initiator's and
|
||||
// the responder's ECC points, respectively.
|
||||
func KDFeHash(h crypto.Hash, z []byte, use string, partyUInfo, partyVInfo []byte, bits int) []byte {
|
||||
hash := h.New()
|
||||
|
||||
out := kdf(hash, bits, func() {
|
||||
hash.Write(z)
|
||||
hash.Write([]byte(use))
|
||||
hash.Write([]byte{0}) // Terminating null character for C-string.
|
||||
hash.Write(partyUInfo)
|
||||
hash.Write(partyVInfo)
|
||||
})
|
||||
return out
|
||||
}
|
||||
|
||||
func kdf(h hash.Hash, bits int, update func()) []byte {
|
||||
bytes := (bits + 7) / 8
|
||||
out := []byte{}
|
||||
|
||||
for counter := 1; len(out) < bytes; counter++ {
|
||||
h.Reset()
|
||||
binary.Write(h, binary.BigEndian, uint32(counter))
|
||||
update()
|
||||
|
||||
out = h.Sum(out)
|
||||
}
|
||||
// out's length is a multiple of hash size, so there will be excess
|
||||
// bytes if bytes isn't a multiple of hash size.
|
||||
out = out[:bytes]
|
||||
|
||||
// As mentioned in the KDFa and KDFe specs mentioned above,
|
||||
// the unused bits of the most significant octet are masked off.
|
||||
if maskBits := uint8(bits % 8); maskBits > 0 {
|
||||
out[0] &= (1 << maskBits) - 1
|
||||
}
|
||||
return out
|
||||
}
|
||||
+57
@@ -0,0 +1,57 @@
|
||||
//go:build !windows
|
||||
|
||||
// Copyright (c) 2019, Google LLC All rights reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package tpm2
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
|
||||
"github.com/google/go-tpm/tpmutil"
|
||||
)
|
||||
|
||||
// OpenTPM opens a channel to the TPM at the given path. If the file is a
|
||||
// device, then it treats it like a normal TPM device, and if the file is a
|
||||
// Unix domain socket, then it opens a connection to the socket.
|
||||
//
|
||||
// This function may also be invoked with no paths, as tpm2.OpenTPM(). In this
|
||||
// case, the default paths on Linux (/dev/tpmrm0 then /dev/tpm0), will be used.
|
||||
func OpenTPM(path ...string) (tpm io.ReadWriteCloser, err error) {
|
||||
switch len(path) {
|
||||
case 0:
|
||||
tpm, err = tpmutil.OpenTPM("/dev/tpmrm0")
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
tpm, err = tpmutil.OpenTPM("/dev/tpm0")
|
||||
}
|
||||
case 1:
|
||||
tpm, err = tpmutil.OpenTPM(path[0])
|
||||
default:
|
||||
return nil, errors.New("cannot specify multiple paths to tpm2.OpenTPM")
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Make sure this is a TPM 2.0
|
||||
_, err = GetManufacturer(tpm)
|
||||
if err != nil {
|
||||
tpm.Close()
|
||||
return nil, fmt.Errorf("open %s: device is not a TPM 2.0", path)
|
||||
}
|
||||
return tpm, nil
|
||||
}
|
||||
+39
@@ -0,0 +1,39 @@
|
||||
//go:build windows
|
||||
|
||||
// Copyright (c) 2018, Google LLC All rights reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package tpm2
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
|
||||
"github.com/google/go-tpm/tpmutil"
|
||||
"github.com/google/go-tpm/tpmutil/tbs"
|
||||
)
|
||||
|
||||
// OpenTPM opens a channel to the TPM.
|
||||
func OpenTPM() (io.ReadWriteCloser, error) {
|
||||
info, err := tbs.GetDeviceInfo()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if info.TPMVersion != tbs.TPMVersion20 {
|
||||
return nil, fmt.Errorf("openTPM: device is not a TPM 2.0")
|
||||
}
|
||||
|
||||
return tpmutil.OpenTPM()
|
||||
}
|
||||
+1112
File diff suppressed because it is too large
Load Diff
+2354
File diff suppressed because it is too large
Load Diff
+211
@@ -0,0 +1,211 @@
|
||||
// Copyright (c) 2018, Google LLC All rights reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package tpmutil
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/binary"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"reflect"
|
||||
)
|
||||
|
||||
var (
|
||||
selfMarshalerType = reflect.TypeOf((*SelfMarshaler)(nil)).Elem()
|
||||
handlesAreaType = reflect.TypeOf((*[]Handle)(nil))
|
||||
)
|
||||
|
||||
// packWithHeader takes a header and a sequence of elements that are either of
|
||||
// fixed length or slices of fixed-length types and packs them into a single
|
||||
// byte array using binary.Write. It updates the CommandHeader to have the right
|
||||
// length.
|
||||
func packWithHeader(ch commandHeader, cmd ...interface{}) ([]byte, error) {
|
||||
hdrSize := binary.Size(ch)
|
||||
body, err := Pack(cmd...)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("couldn't pack message body: %v", err)
|
||||
}
|
||||
bodySize := len(body)
|
||||
ch.Size = uint32(hdrSize + bodySize)
|
||||
header, err := Pack(ch)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("couldn't pack message header: %v", err)
|
||||
}
|
||||
return append(header, body...), nil
|
||||
}
|
||||
|
||||
// Pack encodes a set of elements into a single byte array, using
|
||||
// encoding/binary. This means that all the elements must be encodeable
|
||||
// according to the rules of encoding/binary.
|
||||
//
|
||||
// It has one difference from encoding/binary: it encodes byte slices with a
|
||||
// prepended length, to match how the TPM encodes variable-length arrays. If
|
||||
// you wish to add a byte slice without length prefix, use RawBytes.
|
||||
func Pack(elts ...interface{}) ([]byte, error) {
|
||||
buf := new(bytes.Buffer)
|
||||
if err := packType(buf, elts...); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return buf.Bytes(), nil
|
||||
}
|
||||
|
||||
// tryMarshal attempts to use a TPMMarshal() method defined on the type
|
||||
// to pack v into buf. True is returned if the method exists and the
|
||||
// marshal was attempted.
|
||||
func tryMarshal(buf io.Writer, v reflect.Value) (bool, error) {
|
||||
t := v.Type()
|
||||
if t.Implements(selfMarshalerType) {
|
||||
if v.Kind() == reflect.Ptr && v.IsNil() {
|
||||
return true, fmt.Errorf("cannot call TPMMarshal on a nil pointer of type %T", v)
|
||||
}
|
||||
return true, v.Interface().(SelfMarshaler).TPMMarshal(buf)
|
||||
}
|
||||
|
||||
// We might have a non-pointer struct field, but we dont have a
|
||||
// pointer with which to implement the interface.
|
||||
// If the pointer of the type implements the interface, we should be
|
||||
// able to construct a value to call TPMMarshal() with.
|
||||
// TODO(awly): Try and avoid blowing away private data by using Addr() instead of Set()
|
||||
if reflect.PtrTo(t).Implements(selfMarshalerType) {
|
||||
tmp := reflect.New(t)
|
||||
tmp.Elem().Set(v)
|
||||
return true, tmp.Interface().(SelfMarshaler).TPMMarshal(buf)
|
||||
}
|
||||
|
||||
return false, nil
|
||||
}
|
||||
|
||||
func packValue(buf io.Writer, v reflect.Value) error {
|
||||
if v.Type() == handlesAreaType {
|
||||
v = v.Convert(reflect.TypeOf((*handleList)(nil)))
|
||||
}
|
||||
if canMarshal, err := tryMarshal(buf, v); canMarshal {
|
||||
return err
|
||||
}
|
||||
|
||||
switch v.Kind() {
|
||||
case reflect.Ptr:
|
||||
if v.IsNil() {
|
||||
return fmt.Errorf("cannot pack nil %s", v.Type().String())
|
||||
}
|
||||
return packValue(buf, v.Elem())
|
||||
case reflect.Struct:
|
||||
for i := 0; i < v.NumField(); i++ {
|
||||
f := v.Field(i)
|
||||
if err := packValue(buf, f); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
default:
|
||||
return binary.Write(buf, binary.BigEndian, v.Interface())
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func packType(buf io.Writer, elts ...interface{}) error {
|
||||
for _, e := range elts {
|
||||
if err := packValue(buf, reflect.ValueOf(e)); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// tryUnmarshal attempts to use TPMUnmarshal() to perform the
|
||||
// unpack, if the given value implements SelfMarshaler.
|
||||
// True is returned if v implements SelfMarshaler & TPMUnmarshal
|
||||
// was called, along with an error returned from TPMUnmarshal.
|
||||
func tryUnmarshal(buf io.Reader, v reflect.Value) (bool, error) {
|
||||
t := v.Type()
|
||||
if t.Implements(selfMarshalerType) {
|
||||
if v.Kind() == reflect.Ptr && v.IsNil() {
|
||||
return true, fmt.Errorf("cannot call TPMUnmarshal on a nil pointer")
|
||||
}
|
||||
return true, v.Interface().(SelfMarshaler).TPMUnmarshal(buf)
|
||||
}
|
||||
|
||||
// We might have a non-pointer struct field, which is addressable,
|
||||
// If the pointer of the type implements the interface, and the
|
||||
// value is addressable, we should be able to call TPMUnmarshal().
|
||||
if v.CanAddr() && reflect.PtrTo(t).Implements(selfMarshalerType) {
|
||||
return true, v.Addr().Interface().(SelfMarshaler).TPMUnmarshal(buf)
|
||||
}
|
||||
|
||||
return false, nil
|
||||
}
|
||||
|
||||
// Unpack is a convenience wrapper around UnpackBuf. Unpack returns the number
|
||||
// of bytes read from b to fill elts and error, if any.
|
||||
func Unpack(b []byte, elts ...interface{}) (int, error) {
|
||||
buf := bytes.NewBuffer(b)
|
||||
err := UnpackBuf(buf, elts...)
|
||||
read := len(b) - buf.Len()
|
||||
return read, err
|
||||
}
|
||||
|
||||
func unpackValue(buf io.Reader, v reflect.Value) error {
|
||||
if v.Type() == handlesAreaType {
|
||||
v = v.Convert(reflect.TypeOf((*handleList)(nil)))
|
||||
}
|
||||
if didUnmarshal, err := tryUnmarshal(buf, v); didUnmarshal {
|
||||
return err
|
||||
}
|
||||
|
||||
switch v.Kind() {
|
||||
case reflect.Ptr:
|
||||
if v.IsNil() {
|
||||
return fmt.Errorf("cannot unpack nil %s", v.Type().String())
|
||||
}
|
||||
return unpackValue(buf, v.Elem())
|
||||
case reflect.Struct:
|
||||
for i := 0; i < v.NumField(); i++ {
|
||||
f := v.Field(i)
|
||||
if err := unpackValue(buf, f); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
default:
|
||||
// binary.Read can only set pointer values, so we need to take the address.
|
||||
if !v.CanAddr() {
|
||||
return fmt.Errorf("cannot unpack unaddressable leaf type %q", v.Type().String())
|
||||
}
|
||||
return binary.Read(buf, binary.BigEndian, v.Addr().Interface())
|
||||
}
|
||||
}
|
||||
|
||||
// UnpackBuf recursively unpacks types from a reader just as encoding/binary
|
||||
// does under binary.BigEndian, but with one difference: it unpacks a byte
|
||||
// slice by first reading an integer with lengthPrefixSize bytes, then reading
|
||||
// that many bytes. It assumes that incoming values are pointers to values so
|
||||
// that, e.g., underlying slices can be resized as needed.
|
||||
func UnpackBuf(buf io.Reader, elts ...interface{}) error {
|
||||
for _, e := range elts {
|
||||
v := reflect.ValueOf(e)
|
||||
if v.Kind() != reflect.Ptr {
|
||||
return fmt.Errorf("non-pointer value %q passed to UnpackBuf", v.Type().String())
|
||||
}
|
||||
if v.IsNil() {
|
||||
return errors.New("nil pointer passed to UnpackBuf")
|
||||
}
|
||||
|
||||
if err := unpackValue(buf, v); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
+10
@@ -0,0 +1,10 @@
|
||||
//go:build !linux && !darwin
|
||||
|
||||
package tpmutil
|
||||
|
||||
import (
|
||||
"os"
|
||||
)
|
||||
|
||||
// Not implemented on Windows.
|
||||
func poll(_ *os.File) error { return nil }
|
||||
+32
@@ -0,0 +1,32 @@
|
||||
//go:build linux || darwin
|
||||
|
||||
package tpmutil
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
|
||||
"golang.org/x/sys/unix"
|
||||
)
|
||||
|
||||
// poll blocks until the file descriptor is ready for reading or an error occurs.
|
||||
func poll(f *os.File) error {
|
||||
var (
|
||||
fds = []unix.PollFd{{
|
||||
Fd: int32(f.Fd()),
|
||||
Events: 0x1, // POLLIN
|
||||
}}
|
||||
timeout = -1 // Indefinite timeout
|
||||
)
|
||||
|
||||
if _, err := unix.Poll(fds, timeout); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Revents is filled in by the kernel.
|
||||
// If the expected event happened, Revents should match Events.
|
||||
if fds[0].Revents != fds[0].Events {
|
||||
return fmt.Errorf("unexpected poll Revents 0x%x", fds[0].Revents)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
+113
@@ -0,0 +1,113 @@
|
||||
// Copyright (c) 2018, Google LLC All rights reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
// Package tpmutil provides common utility functions for both TPM 1.2 and TPM
|
||||
// 2.0 devices.
|
||||
package tpmutil
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"io"
|
||||
"os"
|
||||
"time"
|
||||
)
|
||||
|
||||
// maxTPMResponse is the largest possible response from the TPM. We need to know
|
||||
// this because we don't always know the length of the TPM response, and
|
||||
// /dev/tpm insists on giving it all back in a single value rather than
|
||||
// returning a header and a body in separate responses.
|
||||
const maxTPMResponse = 4096
|
||||
|
||||
// RunCommandRaw executes the given raw command and returns the raw response.
|
||||
// Does not check the response code except to execute retry logic.
|
||||
func RunCommandRaw(rw io.ReadWriter, inb []byte) ([]byte, error) {
|
||||
if rw == nil {
|
||||
return nil, errors.New("nil TPM handle")
|
||||
}
|
||||
|
||||
// f(t) = (2^t)ms, up to 2s
|
||||
var backoffFac uint
|
||||
var rh responseHeader
|
||||
var outb []byte
|
||||
|
||||
for {
|
||||
if _, err := rw.Write(inb); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// If the TPM is a real device, it may not be ready for reading
|
||||
// immediately after writing the command. Wait until the file
|
||||
// descriptor is ready to be read from.
|
||||
if f, ok := rw.(*os.File); ok {
|
||||
if err := poll(f); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
outb = make([]byte, maxTPMResponse)
|
||||
outlen, err := rw.Read(outb)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// Resize the buffer to match the amount read from the TPM.
|
||||
outb = outb[:outlen]
|
||||
|
||||
_, err = Unpack(outb, &rh)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// If TPM is busy, retry the command after waiting a few ms.
|
||||
if rh.Res == RCRetry {
|
||||
if backoffFac < 11 {
|
||||
dur := (1 << backoffFac) * time.Millisecond
|
||||
time.Sleep(dur)
|
||||
backoffFac++
|
||||
} else {
|
||||
return nil, err
|
||||
}
|
||||
} else {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
return outb, nil
|
||||
}
|
||||
|
||||
// RunCommand executes cmd with given tag and arguments. Returns TPM response
|
||||
// body (without response header) and response code from the header. Returned
|
||||
// error may be nil if response code is not RCSuccess; caller should check
|
||||
// both.
|
||||
func RunCommand(rw io.ReadWriter, tag Tag, cmd Command, in ...interface{}) ([]byte, ResponseCode, error) {
|
||||
inb, err := packWithHeader(commandHeader{tag, 0, cmd}, in...)
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
|
||||
outb, err := RunCommandRaw(rw, inb)
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
|
||||
var rh responseHeader
|
||||
read, err := Unpack(outb, &rh)
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
if rh.Res != RCSuccess {
|
||||
return nil, rh.Res, nil
|
||||
}
|
||||
|
||||
return outb[read:], rh.Res, nil
|
||||
}
|
||||
+111
@@ -0,0 +1,111 @@
|
||||
//go:build !windows
|
||||
|
||||
// Copyright (c) 2018, Google LLC All rights reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package tpmutil
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"os"
|
||||
)
|
||||
|
||||
// OpenTPM opens a channel to the TPM at the given path. If the file is a
|
||||
// device, then it treats it like a normal TPM device, and if the file is a
|
||||
// Unix domain socket, then it opens a connection to the socket.
|
||||
func OpenTPM(path string) (io.ReadWriteCloser, error) {
|
||||
// If it's a regular file, then open it
|
||||
var rwc io.ReadWriteCloser
|
||||
fi, err := os.Stat(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if fi.Mode()&os.ModeDevice != 0 {
|
||||
var f *os.File
|
||||
f, err = os.OpenFile(path, os.O_RDWR, 0600)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
rwc = io.ReadWriteCloser(f)
|
||||
} else if fi.Mode()&os.ModeSocket != 0 {
|
||||
rwc = NewEmulatorReadWriteCloser(path)
|
||||
} else {
|
||||
return nil, fmt.Errorf("unsupported TPM file mode %s", fi.Mode().String())
|
||||
}
|
||||
|
||||
return rwc, nil
|
||||
}
|
||||
|
||||
// dialer abstracts the net.Dial call so test code can provide its own net.Conn
|
||||
// implementation.
|
||||
type dialer func(network, path string) (net.Conn, error)
|
||||
|
||||
// EmulatorReadWriteCloser manages connections with a TPM emulator over a Unix
|
||||
// domain socket. These emulators often operate in a write/read/disconnect
|
||||
// sequence, so the Write method always connects, and the Read method always
|
||||
// closes. EmulatorReadWriteCloser is not thread safe.
|
||||
type EmulatorReadWriteCloser struct {
|
||||
path string
|
||||
conn net.Conn
|
||||
dialer dialer
|
||||
}
|
||||
|
||||
// NewEmulatorReadWriteCloser stores information about a Unix domain socket to
|
||||
// write to and read from.
|
||||
func NewEmulatorReadWriteCloser(path string) *EmulatorReadWriteCloser {
|
||||
return &EmulatorReadWriteCloser{
|
||||
path: path,
|
||||
dialer: net.Dial,
|
||||
}
|
||||
}
|
||||
|
||||
// Read implements io.Reader by reading from the Unix domain socket and closing
|
||||
// it.
|
||||
func (erw *EmulatorReadWriteCloser) Read(p []byte) (int, error) {
|
||||
// Read is always the second operation in a Write/Read sequence.
|
||||
if erw.conn == nil {
|
||||
return 0, fmt.Errorf("must call Write then Read in an alternating sequence")
|
||||
}
|
||||
n, err := erw.conn.Read(p)
|
||||
erw.conn.Close()
|
||||
erw.conn = nil
|
||||
return n, err
|
||||
}
|
||||
|
||||
// Write implements io.Writer by connecting to the Unix domain socket and
|
||||
// writing.
|
||||
func (erw *EmulatorReadWriteCloser) Write(p []byte) (int, error) {
|
||||
if erw.conn != nil {
|
||||
return 0, fmt.Errorf("must call Write then Read in an alternating sequence")
|
||||
}
|
||||
var err error
|
||||
erw.conn, err = erw.dialer("unix", erw.path)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return erw.conn.Write(p)
|
||||
}
|
||||
|
||||
// Close implements io.Closer by closing the Unix domain socket if one is open.
|
||||
func (erw *EmulatorReadWriteCloser) Close() error {
|
||||
if erw.conn == nil {
|
||||
return fmt.Errorf("cannot call Close when no connection is open")
|
||||
}
|
||||
err := erw.conn.Close()
|
||||
erw.conn = nil
|
||||
return err
|
||||
}
|
||||
+84
@@ -0,0 +1,84 @@
|
||||
// Copyright (c) 2018, Google LLC All rights reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package tpmutil
|
||||
|
||||
import (
|
||||
"io"
|
||||
|
||||
"github.com/google/go-tpm/tpmutil/tbs"
|
||||
)
|
||||
|
||||
// winTPMBuffer is a ReadWriteCloser to access the TPM in Windows.
|
||||
type winTPMBuffer struct {
|
||||
context tbs.Context
|
||||
outBuffer []byte
|
||||
}
|
||||
|
||||
// Executes the TPM command specified by commandBuffer (at Normal Priority), returning the number
|
||||
// of bytes in the command and any error code returned by executing the TPM command. Command
|
||||
// response can be read by calling Read().
|
||||
func (rwc *winTPMBuffer) Write(commandBuffer []byte) (int, error) {
|
||||
// TPM spec defines longest possible response to be maxTPMResponse.
|
||||
rwc.outBuffer = rwc.outBuffer[:maxTPMResponse]
|
||||
|
||||
outBufferLen, err := rwc.context.SubmitCommand(
|
||||
tbs.NormalPriority,
|
||||
commandBuffer,
|
||||
rwc.outBuffer,
|
||||
)
|
||||
|
||||
if err != nil {
|
||||
rwc.outBuffer = rwc.outBuffer[:0]
|
||||
return 0, err
|
||||
}
|
||||
// Shrink outBuffer so it is length of response.
|
||||
rwc.outBuffer = rwc.outBuffer[:outBufferLen]
|
||||
return len(commandBuffer), nil
|
||||
}
|
||||
|
||||
// Provides TPM response from the command called in the last Write call.
|
||||
func (rwc *winTPMBuffer) Read(responseBuffer []byte) (int, error) {
|
||||
if len(rwc.outBuffer) == 0 {
|
||||
return 0, io.EOF
|
||||
}
|
||||
lenCopied := copy(responseBuffer, rwc.outBuffer)
|
||||
// Cut out the piece of slice which was just read out, maintaining original slice capacity.
|
||||
rwc.outBuffer = append(rwc.outBuffer[:0], rwc.outBuffer[lenCopied:]...)
|
||||
return lenCopied, nil
|
||||
}
|
||||
|
||||
func (rwc *winTPMBuffer) Close() error {
|
||||
return rwc.context.Close()
|
||||
}
|
||||
|
||||
// OpenTPM creates a new instance of a ReadWriteCloser which can interact with a
|
||||
// Windows TPM.
|
||||
func OpenTPM() (io.ReadWriteCloser, error) {
|
||||
tpmContext, err := tbs.CreateContext(tbs.TPMVersion20, tbs.IncludeTPM12|tbs.IncludeTPM20)
|
||||
rwc := &winTPMBuffer{
|
||||
context: tpmContext,
|
||||
outBuffer: make([]byte, 0, maxTPMResponse),
|
||||
}
|
||||
return rwc, err
|
||||
}
|
||||
|
||||
// FromContext creates a new instance of a ReadWriteCloser which can
|
||||
// interact with a Windows TPM, using the specified TBS handle.
|
||||
func FromContext(ctx tbs.Context) io.ReadWriteCloser {
|
||||
return &winTPMBuffer{
|
||||
context: ctx,
|
||||
outBuffer: make([]byte, 0, maxTPMResponse),
|
||||
}
|
||||
}
|
||||
+195
@@ -0,0 +1,195 @@
|
||||
// Copyright (c) 2018, Google LLC All rights reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package tpmutil
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/binary"
|
||||
"fmt"
|
||||
"io"
|
||||
)
|
||||
|
||||
// maxBytesBufferSize sets a sane upper bound on the size of a U32Bytes
|
||||
// buffer. This limit exists to prevent a maliciously large size prefix
|
||||
// from resulting in a massive memory allocation, potentially causing
|
||||
// an OOM condition on the system.
|
||||
// We expect no buffer from a TPM to approach 1Mb in size.
|
||||
const maxBytesBufferSize uint32 = 1024 * 1024 // 1Mb.
|
||||
|
||||
// RawBytes is for Pack and RunCommand arguments that are already encoded.
|
||||
// Compared to []byte, RawBytes will not be prepended with slice length during
|
||||
// encoding.
|
||||
type RawBytes []byte
|
||||
|
||||
// U16Bytes is a byte slice with a 16-bit header
|
||||
type U16Bytes []byte
|
||||
|
||||
// TPMMarshal packs U16Bytes
|
||||
func (b *U16Bytes) TPMMarshal(out io.Writer) error {
|
||||
size := len([]byte(*b))
|
||||
if err := binary.Write(out, binary.BigEndian, uint16(size)); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
n, err := out.Write(*b)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if n != size {
|
||||
return fmt.Errorf("unable to write all contents of U16Bytes")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// TPMUnmarshal unpacks a U16Bytes
|
||||
func (b *U16Bytes) TPMUnmarshal(in io.Reader) error {
|
||||
var tmpSize uint16
|
||||
if err := binary.Read(in, binary.BigEndian, &tmpSize); err != nil {
|
||||
return err
|
||||
}
|
||||
size := int(tmpSize)
|
||||
|
||||
if len(*b) >= size {
|
||||
*b = (*b)[:size]
|
||||
} else {
|
||||
*b = append(*b, make([]byte, size-len(*b))...)
|
||||
}
|
||||
|
||||
n, err := in.Read(*b)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if n != size {
|
||||
return io.ErrUnexpectedEOF
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// U32Bytes is a byte slice with a 32-bit header
|
||||
type U32Bytes []byte
|
||||
|
||||
// TPMMarshal packs U32Bytes
|
||||
func (b *U32Bytes) TPMMarshal(out io.Writer) error {
|
||||
size := len([]byte(*b))
|
||||
if err := binary.Write(out, binary.BigEndian, uint32(size)); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
n, err := out.Write(*b)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if n != size {
|
||||
return fmt.Errorf("unable to write all contents of U32Bytes")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// TPMUnmarshal unpacks a U32Bytes
|
||||
func (b *U32Bytes) TPMUnmarshal(in io.Reader) error {
|
||||
var tmpSize uint32
|
||||
if err := binary.Read(in, binary.BigEndian, &tmpSize); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if tmpSize > maxBytesBufferSize {
|
||||
return bytes.ErrTooLarge
|
||||
}
|
||||
// We can now safely cast to an int on 32-bit or 64-bit machines
|
||||
size := int(tmpSize)
|
||||
|
||||
if len(*b) >= size {
|
||||
*b = (*b)[:size]
|
||||
} else {
|
||||
*b = append(*b, make([]byte, size-len(*b))...)
|
||||
}
|
||||
|
||||
n, err := in.Read(*b)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if n != size {
|
||||
return fmt.Errorf("unable to read all contents in to U32Bytes")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Tag is a command tag.
|
||||
type Tag uint16
|
||||
|
||||
// Command is an identifier of a TPM command.
|
||||
type Command uint32
|
||||
|
||||
// A commandHeader is the header for a TPM command.
|
||||
type commandHeader struct {
|
||||
Tag Tag
|
||||
Size uint32
|
||||
Cmd Command
|
||||
}
|
||||
|
||||
// ResponseCode is a response code returned by TPM.
|
||||
type ResponseCode uint32
|
||||
|
||||
// RCSuccess is response code for successful command. Identical for TPM 1.2 and
|
||||
// 2.0.
|
||||
const RCSuccess ResponseCode = 0x000
|
||||
|
||||
// RCRetry is response code for TPM is busy.
|
||||
const RCRetry ResponseCode = 0x922
|
||||
|
||||
// A responseHeader is a header for TPM responses.
|
||||
type responseHeader struct {
|
||||
Tag Tag
|
||||
Size uint32
|
||||
Res ResponseCode
|
||||
}
|
||||
|
||||
// A Handle is a reference to a TPM object.
|
||||
type Handle uint32
|
||||
|
||||
// HandleValue returns the handle value. This behavior is intended to satisfy
|
||||
// an interface that can be implemented by other, more complex types as well.
|
||||
func (h Handle) HandleValue() uint32 {
|
||||
return uint32(h)
|
||||
}
|
||||
|
||||
type handleList []Handle
|
||||
|
||||
func (l *handleList) TPMMarshal(_ io.Writer) error {
|
||||
return fmt.Errorf("TPMMarhsal on []Handle is not supported yet")
|
||||
}
|
||||
|
||||
func (l *handleList) TPMUnmarshal(in io.Reader) error {
|
||||
var numHandles uint16
|
||||
if err := binary.Read(in, binary.BigEndian, &numHandles); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Make len(e) match size exactly.
|
||||
size := int(numHandles)
|
||||
if len(*l) >= size {
|
||||
*l = (*l)[:size]
|
||||
} else {
|
||||
*l = append(*l, make([]Handle, size-len(*l))...)
|
||||
}
|
||||
return binary.Read(in, binary.BigEndian, *l)
|
||||
}
|
||||
|
||||
// SelfMarshaler allows custom types to override default encoding/decoding
|
||||
// behavior in Pack, Unpack and UnpackBuf.
|
||||
type SelfMarshaler interface {
|
||||
TPMMarshal(out io.Writer) error
|
||||
TPMUnmarshal(in io.Reader) error
|
||||
}
|
||||
+267
@@ -0,0 +1,267 @@
|
||||
// Copyright (c) 2018, Google LLC All rights reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
// Package tbs provides an low-level interface directly mapping to Windows
|
||||
// Tbs.dll system library commands:
|
||||
// https://docs.microsoft.com/en-us/windows/desktop/TBS/tpm-base-services-portal
|
||||
// Public field descriptions contain links to the high-level Windows documentation.
|
||||
package tbs
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"syscall"
|
||||
"unsafe"
|
||||
)
|
||||
|
||||
// Context references the current TPM context
|
||||
type Context uintptr
|
||||
|
||||
// Version of TPM being used by the application.
|
||||
type Version uint32
|
||||
|
||||
// Flag indicates TPM versions that are supported by the application.
|
||||
type Flag uint32
|
||||
|
||||
// CommandPriority is used to determine which pending command to submit whenever the TPM is free.
|
||||
type CommandPriority uint32
|
||||
|
||||
// Command parameters:
|
||||
// https://github.com/tpn/winsdk-10/blob/master/Include/10.0.10240.0/shared/tbs.h
|
||||
const (
|
||||
// https://docs.microsoft.com/en-us/windows/desktop/api/Tbs/ns-tbs-tdtbs_context_params2
|
||||
// OR flags to use multiple.
|
||||
RequestRaw Flag = 1 << iota // Add flag to request raw context
|
||||
IncludeTPM12 // Add flag to support TPM 1.2
|
||||
IncludeTPM20 // Add flag to support TPM 2
|
||||
|
||||
TPMVersion12 Version = 1 // For TPM 1.2 applications
|
||||
TPMVersion20 Version = 2 // For TPM 2 applications or applications using multiple TPM versions
|
||||
|
||||
// https://docs.microsoft.com/en-us/windows/desktop/tbs/command-scheduling
|
||||
// https://docs.microsoft.com/en-us/windows/desktop/api/Tbs/nf-tbs-tbsip_submit_command#parameters
|
||||
LowPriority CommandPriority = 100 // For low priority application use
|
||||
NormalPriority CommandPriority = 200 // For normal priority application use
|
||||
HighPriority CommandPriority = 300 // For high priority application use
|
||||
SystemPriority CommandPriority = 400 // For system tasks that access the TPM
|
||||
|
||||
commandLocalityZero uint32 = 0 // Windows currently only supports TBS_COMMAND_LOCALITY_ZERO.
|
||||
)
|
||||
|
||||
// Error is the return type of all functions in this package.
|
||||
type Error uint32
|
||||
|
||||
func (err Error) Error() string {
|
||||
if description, ok := errorDescriptions[err]; ok {
|
||||
return fmt.Sprintf("TBS Error 0x%X: %s", uint32(err), description)
|
||||
}
|
||||
return fmt.Sprintf("Unrecognized TBS Error 0x%X", uint32(err))
|
||||
}
|
||||
|
||||
func getError(err uintptr) error {
|
||||
// tbs.dll uses 0x0 as the return value for success.
|
||||
if err == 0 {
|
||||
return nil
|
||||
}
|
||||
return Error(err)
|
||||
}
|
||||
|
||||
// TBS Return Codes:
|
||||
// https://docs.microsoft.com/en-us/windows/desktop/TBS/tbs-return-codes
|
||||
const (
|
||||
ErrInternalError Error = 0x80284001
|
||||
ErrBadParameter Error = 0x80284002
|
||||
ErrInvalidOutputPointer Error = 0x80284003
|
||||
ErrInvalidContext Error = 0x80284004
|
||||
ErrInsufficientBuffer Error = 0x80284005
|
||||
ErrIOError Error = 0x80284006
|
||||
ErrInvalidContextParam Error = 0x80284007
|
||||
ErrServiceNotRunning Error = 0x80284008
|
||||
ErrTooManyTBSContexts Error = 0x80284009
|
||||
ErrTooManyResources Error = 0x8028400A
|
||||
ErrServiceStartPending Error = 0x8028400B
|
||||
ErrPPINotSupported Error = 0x8028400C
|
||||
ErrCommandCanceled Error = 0x8028400D
|
||||
ErrBufferTooLarge Error = 0x8028400E
|
||||
ErrTPMNotFound Error = 0x8028400F
|
||||
ErrServiceDisabled Error = 0x80284010
|
||||
ErrNoEventLog Error = 0x80284011
|
||||
ErrAccessDenied Error = 0x80284012
|
||||
ErrProvisioningNotAllowed Error = 0x80284013
|
||||
ErrPPIFunctionUnsupported Error = 0x80284014
|
||||
ErrOwnerauthNotFound Error = 0x80284015
|
||||
)
|
||||
|
||||
var errorDescriptions = map[Error]string{
|
||||
ErrInternalError: "An internal software error occurred.",
|
||||
ErrBadParameter: "One or more parameter values are not valid.",
|
||||
ErrInvalidOutputPointer: "A specified output pointer is bad.",
|
||||
ErrInvalidContext: "The specified context handle does not refer to a valid context.",
|
||||
ErrInsufficientBuffer: "The specified output buffer is too small.",
|
||||
ErrIOError: "An error occurred while communicating with the TPM.",
|
||||
ErrInvalidContextParam: "A context parameter that is not valid was passed when attempting to create a TBS context.",
|
||||
ErrServiceNotRunning: "The TBS service is not running and could not be started.",
|
||||
ErrTooManyTBSContexts: "A new context could not be created because there are too many open contexts.",
|
||||
ErrTooManyResources: "A new virtual resource could not be created because there are too many open virtual resources.",
|
||||
ErrServiceStartPending: "The TBS service has been started but is not yet running.",
|
||||
ErrPPINotSupported: "The physical presence interface is not supported.",
|
||||
ErrCommandCanceled: "The command was canceled.",
|
||||
ErrBufferTooLarge: "The input or output buffer is too large.",
|
||||
ErrTPMNotFound: "A compatible Trusted Platform Module (TPM) Security Device cannot be found on this computer.",
|
||||
ErrServiceDisabled: "The TBS service has been disabled.",
|
||||
ErrNoEventLog: "The TBS event log is not available.",
|
||||
ErrAccessDenied: "The caller does not have the appropriate rights to perform the requested operation.",
|
||||
ErrProvisioningNotAllowed: "The TPM provisioning action is not allowed by the specified flags.",
|
||||
ErrPPIFunctionUnsupported: "The Physical Presence Interface of this firmware does not support the requested method.",
|
||||
ErrOwnerauthNotFound: "The requested TPM OwnerAuth value was not found.",
|
||||
}
|
||||
|
||||
// Tbs.dll provides an API for making calls to the TPM:
|
||||
// https://docs.microsoft.com/en-us/windows/desktop/TBS/tpm-base-services-portal
|
||||
var (
|
||||
tbsDLL = syscall.NewLazyDLL("Tbs.dll")
|
||||
tbsGetDeviceInfo = tbsDLL.NewProc("Tbsi_GetDeviceInfo")
|
||||
tbsCreateContext = tbsDLL.NewProc("Tbsi_Context_Create")
|
||||
tbsContextClose = tbsDLL.NewProc("Tbsip_Context_Close")
|
||||
tbsSubmitCommand = tbsDLL.NewProc("Tbsip_Submit_Command")
|
||||
tbsGetTCGLog = tbsDLL.NewProc("Tbsi_Get_TCG_Log")
|
||||
)
|
||||
|
||||
// Returns the address of the beginning of a slice or 0 for a nil slice.
|
||||
func sliceAddress(s []byte) uintptr {
|
||||
if len(s) == 0 {
|
||||
return 0
|
||||
}
|
||||
return uintptr(unsafe.Pointer(&(s[0])))
|
||||
}
|
||||
|
||||
// DeviceInfo is TPM_DEVICE_INFO from tbs.h
|
||||
type DeviceInfo struct {
|
||||
StructVersion uint32
|
||||
TPMVersion Version
|
||||
TPMInterfaceType uint32
|
||||
TPMImpRevision uint32
|
||||
}
|
||||
|
||||
// GetDeviceInfo gets the DeviceInfo of the current TPM:
|
||||
// https://docs.microsoft.com/en-us/windows/win32/api/tbs/nf-tbs-tbsi_getdeviceinfo
|
||||
func GetDeviceInfo() (*DeviceInfo, error) {
|
||||
info := DeviceInfo{}
|
||||
// TBS_RESULT Tbsi_GetDeviceInfo(
|
||||
// UINT32 Size,
|
||||
// PVOID Info
|
||||
// );
|
||||
if err := tbsGetDeviceInfo.Find(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
result, _, _ := tbsGetDeviceInfo.Call(
|
||||
unsafe.Sizeof(info),
|
||||
uintptr(unsafe.Pointer(&info)),
|
||||
)
|
||||
return &info, getError(result)
|
||||
}
|
||||
|
||||
// CreateContext creates a new TPM context:
|
||||
// https://docs.microsoft.com/en-us/windows/desktop/api/Tbs/nf-tbs-tbsi_context_create
|
||||
func CreateContext(version Version, flag Flag) (Context, error) {
|
||||
var context Context
|
||||
params := struct {
|
||||
Version
|
||||
Flag
|
||||
}{version, flag}
|
||||
// TBS_RESULT Tbsi_Context_Create(
|
||||
// _In_ PCTBS_CONTEXT_PARAMS pContextParams,
|
||||
// _Out_ PTBS_HCONTEXT *phContext
|
||||
// );
|
||||
if err := tbsCreateContext.Find(); err != nil {
|
||||
return context, err
|
||||
}
|
||||
result, _, _ := tbsCreateContext.Call(
|
||||
uintptr(unsafe.Pointer(¶ms)),
|
||||
uintptr(unsafe.Pointer(&context)),
|
||||
)
|
||||
return context, getError(result)
|
||||
}
|
||||
|
||||
// Close closes an existing TPM context:
|
||||
// https://docs.microsoft.com/en-us/windows/desktop/api/Tbs/nf-tbs-tbsip_context_close
|
||||
func (context Context) Close() error {
|
||||
// TBS_RESULT Tbsip_Context_Close(
|
||||
// _In_ TBS_HCONTEXT hContext
|
||||
// );
|
||||
if err := tbsContextClose.Find(); err != nil {
|
||||
return err
|
||||
}
|
||||
result, _, _ := tbsContextClose.Call(uintptr(context))
|
||||
return getError(result)
|
||||
}
|
||||
|
||||
// SubmitCommand sends commandBuffer to the TPM, returning the number of bytes
|
||||
// written to responseBuffer. ErrInsufficientBuffer is returned if the
|
||||
// responseBuffer is too short. ErrInvalidOutputPointer is returned if the
|
||||
// responseBuffer is nil. On failure, the returned length is unspecified.
|
||||
// https://docs.microsoft.com/en-us/windows/desktop/api/Tbs/nf-tbs-tbsip_submit_command
|
||||
func (context Context) SubmitCommand(
|
||||
priority CommandPriority,
|
||||
commandBuffer []byte,
|
||||
responseBuffer []byte,
|
||||
) (uint32, error) {
|
||||
responseBufferLen := uint32(len(responseBuffer))
|
||||
|
||||
// TBS_RESULT Tbsip_Submit_Command(
|
||||
// _In_ TBS_HCONTEXT hContext,
|
||||
// _In_ TBS_COMMAND_LOCALITY Locality,
|
||||
// _In_ TBS_COMMAND_PRIORITY Priority,
|
||||
// _In_ const PCBYTE *pabCommand,
|
||||
// _In_ UINT32 cbCommand,
|
||||
// _Out_ PBYTE *pabResult,
|
||||
// _Inout_ UINT32 *pcbOutput
|
||||
// );
|
||||
if err := tbsSubmitCommand.Find(); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
result, _, _ := tbsSubmitCommand.Call(
|
||||
uintptr(context),
|
||||
uintptr(commandLocalityZero),
|
||||
uintptr(priority),
|
||||
sliceAddress(commandBuffer),
|
||||
uintptr(len(commandBuffer)),
|
||||
sliceAddress(responseBuffer),
|
||||
uintptr(unsafe.Pointer(&responseBufferLen)),
|
||||
)
|
||||
return responseBufferLen, getError(result)
|
||||
}
|
||||
|
||||
// GetTCGLog gets the system event log, returning the number of bytes written
|
||||
// to logBuffer. If logBuffer is nil, the size of the TCG log is returned.
|
||||
// ErrInsufficientBuffer is returned if the logBuffer is too short. On failure,
|
||||
// the returned length is unspecified.
|
||||
// https://docs.microsoft.com/en-us/windows/desktop/api/Tbs/nf-tbs-tbsi_get_tcg_log
|
||||
func (context Context) GetTCGLog(logBuffer []byte) (uint32, error) {
|
||||
logBufferLen := uint32(len(logBuffer))
|
||||
|
||||
// TBS_RESULT Tbsi_Get_TCG_Log(
|
||||
// TBS_HCONTEXT hContext,
|
||||
// PBYTE pOutputBuf,
|
||||
// PUINT32 pOutputBufLen
|
||||
// );
|
||||
if err := tbsGetTCGLog.Find(); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
result, _, _ := tbsGetTCGLog.Call(
|
||||
uintptr(context),
|
||||
sliceAddress(logBuffer),
|
||||
uintptr(unsafe.Pointer(&logBufferLen)),
|
||||
)
|
||||
return logBufferLen, getError(result)
|
||||
}
|
||||
+13
@@ -0,0 +1,13 @@
|
||||
// Copyright 2015 The Go Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
//go:build !race
|
||||
|
||||
package race
|
||||
|
||||
func ReadSlice[T any](s []T) {
|
||||
}
|
||||
|
||||
func WriteSlice[T any](s []T) {
|
||||
}
|
||||
+26
@@ -0,0 +1,26 @@
|
||||
// Copyright 2015 The Go Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
//go:build race
|
||||
|
||||
package race
|
||||
|
||||
import (
|
||||
"runtime"
|
||||
"unsafe"
|
||||
)
|
||||
|
||||
func ReadSlice[T any](s []T) {
|
||||
if len(s) == 0 {
|
||||
return
|
||||
}
|
||||
runtime.RaceReadRange(unsafe.Pointer(&s[0]), len(s)*int(unsafe.Sizeof(s[0])))
|
||||
}
|
||||
|
||||
func WriteSlice[T any](s []T) {
|
||||
if len(s) == 0 {
|
||||
return
|
||||
}
|
||||
runtime.RaceWriteRange(unsafe.Pointer(&s[0]), len(s)*int(unsafe.Sizeof(s[0])))
|
||||
}
|
||||
+15
@@ -0,0 +1,15 @@
|
||||
testdata/bench
|
||||
|
||||
# These explicitly listed benchmark data files are for an obsolete version of
|
||||
# snappy_test.go.
|
||||
testdata/alice29.txt
|
||||
testdata/asyoulik.txt
|
||||
testdata/fireworks.jpeg
|
||||
testdata/geo.protodata
|
||||
testdata/html
|
||||
testdata/html_x_4
|
||||
testdata/kppkn.gtb
|
||||
testdata/lcet10.txt
|
||||
testdata/paper-100k.pdf
|
||||
testdata/plrabn12.txt
|
||||
testdata/urls.10K
|
||||
+28
@@ -0,0 +1,28 @@
|
||||
Copyright (c) 2011 The Snappy-Go Authors. All rights reserved.
|
||||
Copyright (c) 2019 Klaus Post. All rights reserved.
|
||||
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
modification, are permitted provided that the following conditions are
|
||||
met:
|
||||
|
||||
* Redistributions of source code must retain the above copyright
|
||||
notice, this list of conditions and the following disclaimer.
|
||||
* Redistributions in binary form must reproduce the above
|
||||
copyright notice, this list of conditions and the following disclaimer
|
||||
in the documentation and/or other materials provided with the
|
||||
distribution.
|
||||
* Neither the name of Google Inc. nor the names of its
|
||||
contributors may be used to endorse or promote products derived from
|
||||
this software without specific prior written permission.
|
||||
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||||
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
+1134
File diff suppressed because it is too large
Load Diff
+443
@@ -0,0 +1,443 @@
|
||||
// Copyright 2011 The Snappy-Go Authors. All rights reserved.
|
||||
// Copyright (c) 2019 Klaus Post. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package s2
|
||||
|
||||
import (
|
||||
"encoding/binary"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strconv"
|
||||
|
||||
"github.com/klauspost/compress/internal/race"
|
||||
)
|
||||
|
||||
var (
|
||||
// ErrCorrupt reports that the input is invalid.
|
||||
ErrCorrupt = errors.New("s2: corrupt input")
|
||||
// ErrCRC reports that the input failed CRC validation (streams only)
|
||||
ErrCRC = errors.New("s2: corrupt input, crc mismatch")
|
||||
// ErrTooLarge reports that the uncompressed length is too large.
|
||||
ErrTooLarge = errors.New("s2: decoded block is too large")
|
||||
// ErrUnsupported reports that the input isn't supported.
|
||||
ErrUnsupported = errors.New("s2: unsupported input")
|
||||
)
|
||||
|
||||
// DecodedLen returns the length of the decoded block.
|
||||
func DecodedLen(src []byte) (int, error) {
|
||||
v, _, err := decodedLen(src)
|
||||
return v, err
|
||||
}
|
||||
|
||||
// decodedLen returns the length of the decoded block and the number of bytes
|
||||
// that the length header occupied.
|
||||
func decodedLen(src []byte) (blockLen, headerLen int, err error) {
|
||||
v, n := binary.Uvarint(src)
|
||||
if n <= 0 || n > 5 || v > 0xffffffff {
|
||||
return 0, 0, ErrCorrupt
|
||||
}
|
||||
|
||||
const wordSize = 32 << (^uint(0) >> 32 & 1)
|
||||
if wordSize == 32 && v > 0x7fffffff {
|
||||
return 0, 0, ErrTooLarge
|
||||
}
|
||||
return int(v), n, nil
|
||||
}
|
||||
|
||||
const (
|
||||
decodeErrCodeCorrupt = 1
|
||||
)
|
||||
|
||||
// Decode returns the decoded form of src. The returned slice may be a sub-
|
||||
// slice of dst if dst was large enough to hold the entire decoded block.
|
||||
// Otherwise, a newly allocated slice will be returned.
|
||||
//
|
||||
// The dst and src must not overlap. It is valid to pass a nil dst.
|
||||
func Decode(dst, src []byte) ([]byte, error) {
|
||||
dLen, s, err := decodedLen(src)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if dLen <= cap(dst) {
|
||||
dst = dst[:dLen]
|
||||
} else {
|
||||
dst = make([]byte, dLen)
|
||||
}
|
||||
|
||||
race.WriteSlice(dst)
|
||||
race.ReadSlice(src[s:])
|
||||
|
||||
if s2Decode(dst, src[s:]) != 0 {
|
||||
return nil, ErrCorrupt
|
||||
}
|
||||
return dst, nil
|
||||
}
|
||||
|
||||
// s2DecodeDict writes the decoding of src to dst. It assumes that the varint-encoded
|
||||
// length of the decompressed bytes has already been read, and that len(dst)
|
||||
// equals that length.
|
||||
//
|
||||
// It returns 0 on success or a decodeErrCodeXxx error code on failure.
|
||||
func s2DecodeDict(dst, src []byte, dict *Dict) int {
|
||||
if dict == nil {
|
||||
return s2Decode(dst, src)
|
||||
}
|
||||
const debug = false
|
||||
const debugErrs = debug
|
||||
|
||||
if debug {
|
||||
fmt.Println("Starting decode, dst len:", len(dst))
|
||||
}
|
||||
var d, s, length int
|
||||
offset := len(dict.dict) - dict.repeat
|
||||
|
||||
// As long as we can read at least 5 bytes...
|
||||
for s < len(src)-5 {
|
||||
// Removing bounds checks is SLOWER, when if doing
|
||||
// in := src[s:s+5]
|
||||
// Checked on Go 1.18
|
||||
switch src[s] & 0x03 {
|
||||
case tagLiteral:
|
||||
x := uint32(src[s] >> 2)
|
||||
switch {
|
||||
case x < 60:
|
||||
s++
|
||||
case x == 60:
|
||||
s += 2
|
||||
x = uint32(src[s-1])
|
||||
case x == 61:
|
||||
in := src[s : s+3]
|
||||
x = uint32(in[1]) | uint32(in[2])<<8
|
||||
s += 3
|
||||
case x == 62:
|
||||
in := src[s : s+4]
|
||||
// Load as 32 bit and shift down.
|
||||
x = uint32(in[0]) | uint32(in[1])<<8 | uint32(in[2])<<16 | uint32(in[3])<<24
|
||||
x >>= 8
|
||||
s += 4
|
||||
case x == 63:
|
||||
in := src[s : s+5]
|
||||
x = uint32(in[1]) | uint32(in[2])<<8 | uint32(in[3])<<16 | uint32(in[4])<<24
|
||||
s += 5
|
||||
}
|
||||
length = int(x) + 1
|
||||
if debug {
|
||||
fmt.Println("literals, length:", length, "d-after:", d+length)
|
||||
}
|
||||
if length > len(dst)-d || length > len(src)-s || (strconv.IntSize == 32 && length <= 0) {
|
||||
if debugErrs {
|
||||
fmt.Println("corrupt literal: length:", length, "d-left:", len(dst)-d, "src-left:", len(src)-s)
|
||||
}
|
||||
return decodeErrCodeCorrupt
|
||||
}
|
||||
|
||||
copy(dst[d:], src[s:s+length])
|
||||
d += length
|
||||
s += length
|
||||
continue
|
||||
|
||||
case tagCopy1:
|
||||
s += 2
|
||||
toffset := int(uint32(src[s-2])&0xe0<<3 | uint32(src[s-1]))
|
||||
length = int(src[s-2]) >> 2 & 0x7
|
||||
if toffset == 0 {
|
||||
if debug {
|
||||
fmt.Print("(repeat) ")
|
||||
}
|
||||
// keep last offset
|
||||
switch length {
|
||||
case 5:
|
||||
length = int(src[s]) + 4
|
||||
s += 1
|
||||
case 6:
|
||||
in := src[s : s+2]
|
||||
length = int(uint32(in[0])|(uint32(in[1])<<8)) + (1 << 8)
|
||||
s += 2
|
||||
case 7:
|
||||
in := src[s : s+3]
|
||||
length = int((uint32(in[2])<<16)|(uint32(in[1])<<8)|uint32(in[0])) + (1 << 16)
|
||||
s += 3
|
||||
default: // 0-> 4
|
||||
}
|
||||
} else {
|
||||
offset = toffset
|
||||
}
|
||||
length += 4
|
||||
case tagCopy2:
|
||||
in := src[s : s+3]
|
||||
offset = int(uint32(in[1]) | uint32(in[2])<<8)
|
||||
length = 1 + int(in[0])>>2
|
||||
s += 3
|
||||
|
||||
case tagCopy4:
|
||||
in := src[s : s+5]
|
||||
offset = int(uint32(in[1]) | uint32(in[2])<<8 | uint32(in[3])<<16 | uint32(in[4])<<24)
|
||||
length = 1 + int(in[0])>>2
|
||||
s += 5
|
||||
}
|
||||
|
||||
if offset <= 0 || length > len(dst)-d {
|
||||
if debugErrs {
|
||||
fmt.Println("match error; offset:", offset, "length:", length, "dst-left:", len(dst)-d)
|
||||
}
|
||||
return decodeErrCodeCorrupt
|
||||
}
|
||||
|
||||
// copy from dict
|
||||
if d < offset {
|
||||
if d > MaxDictSrcOffset {
|
||||
if debugErrs {
|
||||
fmt.Println("dict after", MaxDictSrcOffset, "d:", d, "offset:", offset, "length:", length)
|
||||
}
|
||||
return decodeErrCodeCorrupt
|
||||
}
|
||||
startOff := len(dict.dict) - offset + d
|
||||
if startOff < 0 || startOff+length > len(dict.dict) {
|
||||
if debugErrs {
|
||||
fmt.Printf("offset (%d) + length (%d) bigger than dict (%d)\n", offset, length, len(dict.dict))
|
||||
}
|
||||
return decodeErrCodeCorrupt
|
||||
}
|
||||
if debug {
|
||||
fmt.Println("dict copy, length:", length, "offset:", offset, "d-after:", d+length, "dict start offset:", startOff)
|
||||
}
|
||||
copy(dst[d:d+length], dict.dict[startOff:])
|
||||
d += length
|
||||
continue
|
||||
}
|
||||
|
||||
if debug {
|
||||
fmt.Println("copy, length:", length, "offset:", offset, "d-after:", d+length)
|
||||
}
|
||||
|
||||
// Copy from an earlier sub-slice of dst to a later sub-slice.
|
||||
// If no overlap, use the built-in copy:
|
||||
if offset > length {
|
||||
copy(dst[d:d+length], dst[d-offset:])
|
||||
d += length
|
||||
continue
|
||||
}
|
||||
|
||||
// Unlike the built-in copy function, this byte-by-byte copy always runs
|
||||
// forwards, even if the slices overlap. Conceptually, this is:
|
||||
//
|
||||
// d += forwardCopy(dst[d:d+length], dst[d-offset:])
|
||||
//
|
||||
// We align the slices into a and b and show the compiler they are the same size.
|
||||
// This allows the loop to run without bounds checks.
|
||||
a := dst[d : d+length]
|
||||
b := dst[d-offset:]
|
||||
b = b[:len(a)]
|
||||
for i := range a {
|
||||
a[i] = b[i]
|
||||
}
|
||||
d += length
|
||||
}
|
||||
|
||||
// Remaining with extra checks...
|
||||
for s < len(src) {
|
||||
switch src[s] & 0x03 {
|
||||
case tagLiteral:
|
||||
x := uint32(src[s] >> 2)
|
||||
switch {
|
||||
case x < 60:
|
||||
s++
|
||||
case x == 60:
|
||||
s += 2
|
||||
if uint(s) > uint(len(src)) { // The uint conversions catch overflow from the previous line.
|
||||
if debugErrs {
|
||||
fmt.Println("src went oob")
|
||||
}
|
||||
return decodeErrCodeCorrupt
|
||||
}
|
||||
x = uint32(src[s-1])
|
||||
case x == 61:
|
||||
s += 3
|
||||
if uint(s) > uint(len(src)) { // The uint conversions catch overflow from the previous line.
|
||||
if debugErrs {
|
||||
fmt.Println("src went oob")
|
||||
}
|
||||
return decodeErrCodeCorrupt
|
||||
}
|
||||
x = uint32(src[s-2]) | uint32(src[s-1])<<8
|
||||
case x == 62:
|
||||
s += 4
|
||||
if uint(s) > uint(len(src)) { // The uint conversions catch overflow from the previous line.
|
||||
if debugErrs {
|
||||
fmt.Println("src went oob")
|
||||
}
|
||||
return decodeErrCodeCorrupt
|
||||
}
|
||||
x = uint32(src[s-3]) | uint32(src[s-2])<<8 | uint32(src[s-1])<<16
|
||||
case x == 63:
|
||||
s += 5
|
||||
if uint(s) > uint(len(src)) { // The uint conversions catch overflow from the previous line.
|
||||
if debugErrs {
|
||||
fmt.Println("src went oob")
|
||||
}
|
||||
return decodeErrCodeCorrupt
|
||||
}
|
||||
x = uint32(src[s-4]) | uint32(src[s-3])<<8 | uint32(src[s-2])<<16 | uint32(src[s-1])<<24
|
||||
}
|
||||
length = int(x) + 1
|
||||
if length > len(dst)-d || length > len(src)-s || (strconv.IntSize == 32 && length <= 0) {
|
||||
if debugErrs {
|
||||
fmt.Println("corrupt literal: length:", length, "d-left:", len(dst)-d, "src-left:", len(src)-s)
|
||||
}
|
||||
return decodeErrCodeCorrupt
|
||||
}
|
||||
if debug {
|
||||
fmt.Println("literals, length:", length, "d-after:", d+length)
|
||||
}
|
||||
|
||||
copy(dst[d:], src[s:s+length])
|
||||
d += length
|
||||
s += length
|
||||
continue
|
||||
|
||||
case tagCopy1:
|
||||
s += 2
|
||||
if uint(s) > uint(len(src)) { // The uint conversions catch overflow from the previous line.
|
||||
if debugErrs {
|
||||
fmt.Println("src went oob")
|
||||
}
|
||||
return decodeErrCodeCorrupt
|
||||
}
|
||||
length = int(src[s-2]) >> 2 & 0x7
|
||||
toffset := int(uint32(src[s-2])&0xe0<<3 | uint32(src[s-1]))
|
||||
if toffset == 0 {
|
||||
if debug {
|
||||
fmt.Print("(repeat) ")
|
||||
}
|
||||
// keep last offset
|
||||
switch length {
|
||||
case 5:
|
||||
s += 1
|
||||
if uint(s) > uint(len(src)) { // The uint conversions catch overflow from the previous line.
|
||||
if debugErrs {
|
||||
fmt.Println("src went oob")
|
||||
}
|
||||
return decodeErrCodeCorrupt
|
||||
}
|
||||
length = int(uint32(src[s-1])) + 4
|
||||
case 6:
|
||||
s += 2
|
||||
if uint(s) > uint(len(src)) { // The uint conversions catch overflow from the previous line.
|
||||
if debugErrs {
|
||||
fmt.Println("src went oob")
|
||||
}
|
||||
return decodeErrCodeCorrupt
|
||||
}
|
||||
length = int(uint32(src[s-2])|(uint32(src[s-1])<<8)) + (1 << 8)
|
||||
case 7:
|
||||
s += 3
|
||||
if uint(s) > uint(len(src)) { // The uint conversions catch overflow from the previous line.
|
||||
if debugErrs {
|
||||
fmt.Println("src went oob")
|
||||
}
|
||||
return decodeErrCodeCorrupt
|
||||
}
|
||||
length = int(uint32(src[s-3])|(uint32(src[s-2])<<8)|(uint32(src[s-1])<<16)) + (1 << 16)
|
||||
default: // 0-> 4
|
||||
}
|
||||
} else {
|
||||
offset = toffset
|
||||
}
|
||||
length += 4
|
||||
case tagCopy2:
|
||||
s += 3
|
||||
if uint(s) > uint(len(src)) { // The uint conversions catch overflow from the previous line.
|
||||
if debugErrs {
|
||||
fmt.Println("src went oob")
|
||||
}
|
||||
return decodeErrCodeCorrupt
|
||||
}
|
||||
length = 1 + int(src[s-3])>>2
|
||||
offset = int(uint32(src[s-2]) | uint32(src[s-1])<<8)
|
||||
|
||||
case tagCopy4:
|
||||
s += 5
|
||||
if uint(s) > uint(len(src)) { // The uint conversions catch overflow from the previous line.
|
||||
if debugErrs {
|
||||
fmt.Println("src went oob")
|
||||
}
|
||||
return decodeErrCodeCorrupt
|
||||
}
|
||||
length = 1 + int(src[s-5])>>2
|
||||
offset = int(uint32(src[s-4]) | uint32(src[s-3])<<8 | uint32(src[s-2])<<16 | uint32(src[s-1])<<24)
|
||||
}
|
||||
|
||||
if offset <= 0 || length > len(dst)-d {
|
||||
if debugErrs {
|
||||
fmt.Println("match error; offset:", offset, "length:", length, "dst-left:", len(dst)-d)
|
||||
}
|
||||
return decodeErrCodeCorrupt
|
||||
}
|
||||
|
||||
// copy from dict
|
||||
if d < offset {
|
||||
if d > MaxDictSrcOffset {
|
||||
if debugErrs {
|
||||
fmt.Println("dict after", MaxDictSrcOffset, "d:", d, "offset:", offset, "length:", length)
|
||||
}
|
||||
return decodeErrCodeCorrupt
|
||||
}
|
||||
rOff := len(dict.dict) - (offset - d)
|
||||
if debug {
|
||||
fmt.Println("starting dict entry from dict offset", len(dict.dict)-rOff)
|
||||
}
|
||||
if rOff+length > len(dict.dict) {
|
||||
if debugErrs {
|
||||
fmt.Println("err: END offset", rOff+length, "bigger than dict", len(dict.dict), "dict offset:", rOff, "length:", length)
|
||||
}
|
||||
return decodeErrCodeCorrupt
|
||||
}
|
||||
if rOff < 0 {
|
||||
if debugErrs {
|
||||
fmt.Println("err: START offset", rOff, "less than 0", len(dict.dict), "dict offset:", rOff, "length:", length)
|
||||
}
|
||||
return decodeErrCodeCorrupt
|
||||
}
|
||||
copy(dst[d:d+length], dict.dict[rOff:])
|
||||
d += length
|
||||
continue
|
||||
}
|
||||
|
||||
if debug {
|
||||
fmt.Println("copy, length:", length, "offset:", offset, "d-after:", d+length)
|
||||
}
|
||||
|
||||
// Copy from an earlier sub-slice of dst to a later sub-slice.
|
||||
// If no overlap, use the built-in copy:
|
||||
if offset > length {
|
||||
copy(dst[d:d+length], dst[d-offset:])
|
||||
d += length
|
||||
continue
|
||||
}
|
||||
|
||||
// Unlike the built-in copy function, this byte-by-byte copy always runs
|
||||
// forwards, even if the slices overlap. Conceptually, this is:
|
||||
//
|
||||
// d += forwardCopy(dst[d:d+length], dst[d-offset:])
|
||||
//
|
||||
// We align the slices into a and b and show the compiler they are the same size.
|
||||
// This allows the loop to run without bounds checks.
|
||||
a := dst[d : d+length]
|
||||
b := dst[d-offset:]
|
||||
b = b[:len(a)]
|
||||
for i := range a {
|
||||
a[i] = b[i]
|
||||
}
|
||||
d += length
|
||||
}
|
||||
|
||||
if d != len(dst) {
|
||||
if debugErrs {
|
||||
fmt.Println("wanted length", len(dst), "got", d)
|
||||
}
|
||||
return decodeErrCodeCorrupt
|
||||
}
|
||||
return 0
|
||||
}
|
||||
+568
@@ -0,0 +1,568 @@
|
||||
// Copyright 2016 The Go Authors. All rights reserved.
|
||||
// Copyright (c) 2019 Klaus Post. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
// +build !appengine
|
||||
// +build gc
|
||||
// +build !noasm
|
||||
|
||||
#include "textflag.h"
|
||||
|
||||
#define R_TMP0 AX
|
||||
#define R_TMP1 BX
|
||||
#define R_LEN CX
|
||||
#define R_OFF DX
|
||||
#define R_SRC SI
|
||||
#define R_DST DI
|
||||
#define R_DBASE R8
|
||||
#define R_DLEN R9
|
||||
#define R_DEND R10
|
||||
#define R_SBASE R11
|
||||
#define R_SLEN R12
|
||||
#define R_SEND R13
|
||||
#define R_TMP2 R14
|
||||
#define R_TMP3 R15
|
||||
|
||||
// The asm code generally follows the pure Go code in decode_other.go, except
|
||||
// where marked with a "!!!".
|
||||
|
||||
// func decode(dst, src []byte) int
|
||||
//
|
||||
// All local variables fit into registers. The non-zero stack size is only to
|
||||
// spill registers and push args when issuing a CALL. The register allocation:
|
||||
// - R_TMP0 scratch
|
||||
// - R_TMP1 scratch
|
||||
// - R_LEN length or x (shared)
|
||||
// - R_OFF offset
|
||||
// - R_SRC &src[s]
|
||||
// - R_DST &dst[d]
|
||||
// + R_DBASE dst_base
|
||||
// + R_DLEN dst_len
|
||||
// + R_DEND dst_base + dst_len
|
||||
// + R_SBASE src_base
|
||||
// + R_SLEN src_len
|
||||
// + R_SEND src_base + src_len
|
||||
// - R_TMP2 used by doCopy
|
||||
// - R_TMP3 used by doCopy
|
||||
//
|
||||
// The registers R_DBASE-R_SEND (marked with a "+") are set at the start of the
|
||||
// function, and after a CALL returns, and are not otherwise modified.
|
||||
//
|
||||
// The d variable is implicitly R_DST - R_DBASE, and len(dst)-d is R_DEND - R_DST.
|
||||
// The s variable is implicitly R_SRC - R_SBASE, and len(src)-s is R_SEND - R_SRC.
|
||||
TEXT ·s2Decode(SB), NOSPLIT, $56-56
|
||||
// Initialize R_SRC, R_DST and R_DBASE-R_SEND.
|
||||
MOVQ dst_base+0(FP), R_DBASE
|
||||
MOVQ dst_len+8(FP), R_DLEN
|
||||
MOVQ R_DBASE, R_DST
|
||||
MOVQ R_DBASE, R_DEND
|
||||
ADDQ R_DLEN, R_DEND
|
||||
MOVQ src_base+24(FP), R_SBASE
|
||||
MOVQ src_len+32(FP), R_SLEN
|
||||
MOVQ R_SBASE, R_SRC
|
||||
MOVQ R_SBASE, R_SEND
|
||||
ADDQ R_SLEN, R_SEND
|
||||
XORQ R_OFF, R_OFF
|
||||
|
||||
loop:
|
||||
// for s < len(src)
|
||||
CMPQ R_SRC, R_SEND
|
||||
JEQ end
|
||||
|
||||
// R_LEN = uint32(src[s])
|
||||
//
|
||||
// switch src[s] & 0x03
|
||||
MOVBLZX (R_SRC), R_LEN
|
||||
MOVL R_LEN, R_TMP1
|
||||
ANDL $3, R_TMP1
|
||||
CMPL R_TMP1, $1
|
||||
JAE tagCopy
|
||||
|
||||
// ----------------------------------------
|
||||
// The code below handles literal tags.
|
||||
|
||||
// case tagLiteral:
|
||||
// x := uint32(src[s] >> 2)
|
||||
// switch
|
||||
SHRL $2, R_LEN
|
||||
CMPL R_LEN, $60
|
||||
JAE tagLit60Plus
|
||||
|
||||
// case x < 60:
|
||||
// s++
|
||||
INCQ R_SRC
|
||||
|
||||
doLit:
|
||||
// This is the end of the inner "switch", when we have a literal tag.
|
||||
//
|
||||
// We assume that R_LEN == x and x fits in a uint32, where x is the variable
|
||||
// used in the pure Go decode_other.go code.
|
||||
|
||||
// length = int(x) + 1
|
||||
//
|
||||
// Unlike the pure Go code, we don't need to check if length <= 0 because
|
||||
// R_LEN can hold 64 bits, so the increment cannot overflow.
|
||||
INCQ R_LEN
|
||||
|
||||
// Prepare to check if copying length bytes will run past the end of dst or
|
||||
// src.
|
||||
//
|
||||
// R_TMP0 = len(dst) - d
|
||||
// R_TMP1 = len(src) - s
|
||||
MOVQ R_DEND, R_TMP0
|
||||
SUBQ R_DST, R_TMP0
|
||||
MOVQ R_SEND, R_TMP1
|
||||
SUBQ R_SRC, R_TMP1
|
||||
|
||||
// !!! Try a faster technique for short (16 or fewer bytes) copies.
|
||||
//
|
||||
// if length > 16 || len(dst)-d < 16 || len(src)-s < 16 {
|
||||
// goto callMemmove // Fall back on calling runtime·memmove.
|
||||
// }
|
||||
//
|
||||
// The C++ snappy code calls this TryFastAppend. It also checks len(src)-s
|
||||
// against 21 instead of 16, because it cannot assume that all of its input
|
||||
// is contiguous in memory and so it needs to leave enough source bytes to
|
||||
// read the next tag without refilling buffers, but Go's Decode assumes
|
||||
// contiguousness (the src argument is a []byte).
|
||||
CMPQ R_LEN, $16
|
||||
JGT callMemmove
|
||||
CMPQ R_TMP0, $16
|
||||
JLT callMemmove
|
||||
CMPQ R_TMP1, $16
|
||||
JLT callMemmove
|
||||
|
||||
// !!! Implement the copy from src to dst as a 16-byte load and store.
|
||||
// (Decode's documentation says that dst and src must not overlap.)
|
||||
//
|
||||
// This always copies 16 bytes, instead of only length bytes, but that's
|
||||
// OK. If the input is a valid Snappy encoding then subsequent iterations
|
||||
// will fix up the overrun. Otherwise, Decode returns a nil []byte (and a
|
||||
// non-nil error), so the overrun will be ignored.
|
||||
//
|
||||
// Note that on amd64, it is legal and cheap to issue unaligned 8-byte or
|
||||
// 16-byte loads and stores. This technique probably wouldn't be as
|
||||
// effective on architectures that are fussier about alignment.
|
||||
MOVOU 0(R_SRC), X0
|
||||
MOVOU X0, 0(R_DST)
|
||||
|
||||
// d += length
|
||||
// s += length
|
||||
ADDQ R_LEN, R_DST
|
||||
ADDQ R_LEN, R_SRC
|
||||
JMP loop
|
||||
|
||||
callMemmove:
|
||||
// if length > len(dst)-d || length > len(src)-s { etc }
|
||||
CMPQ R_LEN, R_TMP0
|
||||
JGT errCorrupt
|
||||
CMPQ R_LEN, R_TMP1
|
||||
JGT errCorrupt
|
||||
|
||||
// copy(dst[d:], src[s:s+length])
|
||||
//
|
||||
// This means calling runtime·memmove(&dst[d], &src[s], length), so we push
|
||||
// R_DST, R_SRC and R_LEN as arguments. Coincidentally, we also need to spill those
|
||||
// three registers to the stack, to save local variables across the CALL.
|
||||
MOVQ R_DST, 0(SP)
|
||||
MOVQ R_SRC, 8(SP)
|
||||
MOVQ R_LEN, 16(SP)
|
||||
MOVQ R_DST, 24(SP)
|
||||
MOVQ R_SRC, 32(SP)
|
||||
MOVQ R_LEN, 40(SP)
|
||||
MOVQ R_OFF, 48(SP)
|
||||
CALL runtime·memmove(SB)
|
||||
|
||||
// Restore local variables: unspill registers from the stack and
|
||||
// re-calculate R_DBASE-R_SEND.
|
||||
MOVQ 24(SP), R_DST
|
||||
MOVQ 32(SP), R_SRC
|
||||
MOVQ 40(SP), R_LEN
|
||||
MOVQ 48(SP), R_OFF
|
||||
MOVQ dst_base+0(FP), R_DBASE
|
||||
MOVQ dst_len+8(FP), R_DLEN
|
||||
MOVQ R_DBASE, R_DEND
|
||||
ADDQ R_DLEN, R_DEND
|
||||
MOVQ src_base+24(FP), R_SBASE
|
||||
MOVQ src_len+32(FP), R_SLEN
|
||||
MOVQ R_SBASE, R_SEND
|
||||
ADDQ R_SLEN, R_SEND
|
||||
|
||||
// d += length
|
||||
// s += length
|
||||
ADDQ R_LEN, R_DST
|
||||
ADDQ R_LEN, R_SRC
|
||||
JMP loop
|
||||
|
||||
tagLit60Plus:
|
||||
// !!! This fragment does the
|
||||
//
|
||||
// s += x - 58; if uint(s) > uint(len(src)) { etc }
|
||||
//
|
||||
// checks. In the asm version, we code it once instead of once per switch case.
|
||||
ADDQ R_LEN, R_SRC
|
||||
SUBQ $58, R_SRC
|
||||
CMPQ R_SRC, R_SEND
|
||||
JA errCorrupt
|
||||
|
||||
// case x == 60:
|
||||
CMPL R_LEN, $61
|
||||
JEQ tagLit61
|
||||
JA tagLit62Plus
|
||||
|
||||
// x = uint32(src[s-1])
|
||||
MOVBLZX -1(R_SRC), R_LEN
|
||||
JMP doLit
|
||||
|
||||
tagLit61:
|
||||
// case x == 61:
|
||||
// x = uint32(src[s-2]) | uint32(src[s-1])<<8
|
||||
MOVWLZX -2(R_SRC), R_LEN
|
||||
JMP doLit
|
||||
|
||||
tagLit62Plus:
|
||||
CMPL R_LEN, $62
|
||||
JA tagLit63
|
||||
|
||||
// case x == 62:
|
||||
// x = uint32(src[s-3]) | uint32(src[s-2])<<8 | uint32(src[s-1])<<16
|
||||
// We read one byte, safe to read one back, since we are just reading tag.
|
||||
// x = binary.LittleEndian.Uint32(src[s-1:]) >> 8
|
||||
MOVL -4(R_SRC), R_LEN
|
||||
SHRL $8, R_LEN
|
||||
JMP doLit
|
||||
|
||||
tagLit63:
|
||||
// case x == 63:
|
||||
// x = uint32(src[s-4]) | uint32(src[s-3])<<8 | uint32(src[s-2])<<16 | uint32(src[s-1])<<24
|
||||
MOVL -4(R_SRC), R_LEN
|
||||
JMP doLit
|
||||
|
||||
// The code above handles literal tags.
|
||||
// ----------------------------------------
|
||||
// The code below handles copy tags.
|
||||
|
||||
tagCopy4:
|
||||
// case tagCopy4:
|
||||
// s += 5
|
||||
ADDQ $5, R_SRC
|
||||
|
||||
// if uint(s) > uint(len(src)) { etc }
|
||||
CMPQ R_SRC, R_SEND
|
||||
JA errCorrupt
|
||||
|
||||
// length = 1 + int(src[s-5])>>2
|
||||
SHRQ $2, R_LEN
|
||||
INCQ R_LEN
|
||||
|
||||
// offset = int(uint32(src[s-4]) | uint32(src[s-3])<<8 | uint32(src[s-2])<<16 | uint32(src[s-1])<<24)
|
||||
MOVLQZX -4(R_SRC), R_OFF
|
||||
JMP doCopy
|
||||
|
||||
tagCopy2:
|
||||
// case tagCopy2:
|
||||
// s += 3
|
||||
ADDQ $3, R_SRC
|
||||
|
||||
// if uint(s) > uint(len(src)) { etc }
|
||||
CMPQ R_SRC, R_SEND
|
||||
JA errCorrupt
|
||||
|
||||
// length = 1 + int(src[s-3])>>2
|
||||
SHRQ $2, R_LEN
|
||||
INCQ R_LEN
|
||||
|
||||
// offset = int(uint32(src[s-2]) | uint32(src[s-1])<<8)
|
||||
MOVWQZX -2(R_SRC), R_OFF
|
||||
JMP doCopy
|
||||
|
||||
tagCopy:
|
||||
// We have a copy tag. We assume that:
|
||||
// - R_TMP1 == src[s] & 0x03
|
||||
// - R_LEN == src[s]
|
||||
CMPQ R_TMP1, $2
|
||||
JEQ tagCopy2
|
||||
JA tagCopy4
|
||||
|
||||
// case tagCopy1:
|
||||
// s += 2
|
||||
ADDQ $2, R_SRC
|
||||
|
||||
// if uint(s) > uint(len(src)) { etc }
|
||||
CMPQ R_SRC, R_SEND
|
||||
JA errCorrupt
|
||||
|
||||
// offset = int(uint32(src[s-2])&0xe0<<3 | uint32(src[s-1]))
|
||||
// length = 4 + int(src[s-2])>>2&0x7
|
||||
MOVBQZX -1(R_SRC), R_TMP1
|
||||
MOVQ R_LEN, R_TMP0
|
||||
SHRQ $2, R_LEN
|
||||
ANDQ $0xe0, R_TMP0
|
||||
ANDQ $7, R_LEN
|
||||
SHLQ $3, R_TMP0
|
||||
ADDQ $4, R_LEN
|
||||
ORQ R_TMP1, R_TMP0
|
||||
|
||||
// check if repeat code, ZF set by ORQ.
|
||||
JZ repeatCode
|
||||
|
||||
// This is a regular copy, transfer our temporary value to R_OFF (length)
|
||||
MOVQ R_TMP0, R_OFF
|
||||
JMP doCopy
|
||||
|
||||
// This is a repeat code.
|
||||
repeatCode:
|
||||
// If length < 9, reuse last offset, with the length already calculated.
|
||||
CMPQ R_LEN, $9
|
||||
JL doCopyRepeat
|
||||
|
||||
// Read additional bytes for length.
|
||||
JE repeatLen1
|
||||
|
||||
// Rare, so the extra branch shouldn't hurt too much.
|
||||
CMPQ R_LEN, $10
|
||||
JE repeatLen2
|
||||
JMP repeatLen3
|
||||
|
||||
// Read repeat lengths.
|
||||
repeatLen1:
|
||||
// s ++
|
||||
ADDQ $1, R_SRC
|
||||
|
||||
// if uint(s) > uint(len(src)) { etc }
|
||||
CMPQ R_SRC, R_SEND
|
||||
JA errCorrupt
|
||||
|
||||
// length = src[s-1] + 8
|
||||
MOVBQZX -1(R_SRC), R_LEN
|
||||
ADDL $8, R_LEN
|
||||
JMP doCopyRepeat
|
||||
|
||||
repeatLen2:
|
||||
// s +=2
|
||||
ADDQ $2, R_SRC
|
||||
|
||||
// if uint(s) > uint(len(src)) { etc }
|
||||
CMPQ R_SRC, R_SEND
|
||||
JA errCorrupt
|
||||
|
||||
// length = uint32(src[s-2]) | (uint32(src[s-1])<<8) + (1 << 8)
|
||||
MOVWQZX -2(R_SRC), R_LEN
|
||||
ADDL $260, R_LEN
|
||||
JMP doCopyRepeat
|
||||
|
||||
repeatLen3:
|
||||
// s +=3
|
||||
ADDQ $3, R_SRC
|
||||
|
||||
// if uint(s) > uint(len(src)) { etc }
|
||||
CMPQ R_SRC, R_SEND
|
||||
JA errCorrupt
|
||||
|
||||
// length = uint32(src[s-3]) | (uint32(src[s-2])<<8) | (uint32(src[s-1])<<16) + (1 << 16)
|
||||
// Read one byte further back (just part of the tag, shifted out)
|
||||
MOVL -4(R_SRC), R_LEN
|
||||
SHRL $8, R_LEN
|
||||
ADDL $65540, R_LEN
|
||||
JMP doCopyRepeat
|
||||
|
||||
doCopy:
|
||||
// This is the end of the outer "switch", when we have a copy tag.
|
||||
//
|
||||
// We assume that:
|
||||
// - R_LEN == length && R_LEN > 0
|
||||
// - R_OFF == offset
|
||||
|
||||
// if d < offset { etc }
|
||||
MOVQ R_DST, R_TMP1
|
||||
SUBQ R_DBASE, R_TMP1
|
||||
CMPQ R_TMP1, R_OFF
|
||||
JLT errCorrupt
|
||||
|
||||
// Repeat values can skip the test above, since any offset > 0 will be in dst.
|
||||
doCopyRepeat:
|
||||
// if offset <= 0 { etc }
|
||||
CMPQ R_OFF, $0
|
||||
JLE errCorrupt
|
||||
|
||||
// if length > len(dst)-d { etc }
|
||||
MOVQ R_DEND, R_TMP1
|
||||
SUBQ R_DST, R_TMP1
|
||||
CMPQ R_LEN, R_TMP1
|
||||
JGT errCorrupt
|
||||
|
||||
// forwardCopy(dst[d:d+length], dst[d-offset:]); d += length
|
||||
//
|
||||
// Set:
|
||||
// - R_TMP2 = len(dst)-d
|
||||
// - R_TMP3 = &dst[d-offset]
|
||||
MOVQ R_DEND, R_TMP2
|
||||
SUBQ R_DST, R_TMP2
|
||||
MOVQ R_DST, R_TMP3
|
||||
SUBQ R_OFF, R_TMP3
|
||||
|
||||
// !!! Try a faster technique for short (16 or fewer bytes) forward copies.
|
||||
//
|
||||
// First, try using two 8-byte load/stores, similar to the doLit technique
|
||||
// above. Even if dst[d:d+length] and dst[d-offset:] can overlap, this is
|
||||
// still OK if offset >= 8. Note that this has to be two 8-byte load/stores
|
||||
// and not one 16-byte load/store, and the first store has to be before the
|
||||
// second load, due to the overlap if offset is in the range [8, 16).
|
||||
//
|
||||
// if length > 16 || offset < 8 || len(dst)-d < 16 {
|
||||
// goto slowForwardCopy
|
||||
// }
|
||||
// copy 16 bytes
|
||||
// d += length
|
||||
CMPQ R_LEN, $16
|
||||
JGT slowForwardCopy
|
||||
CMPQ R_OFF, $8
|
||||
JLT slowForwardCopy
|
||||
CMPQ R_TMP2, $16
|
||||
JLT slowForwardCopy
|
||||
MOVQ 0(R_TMP3), R_TMP0
|
||||
MOVQ R_TMP0, 0(R_DST)
|
||||
MOVQ 8(R_TMP3), R_TMP1
|
||||
MOVQ R_TMP1, 8(R_DST)
|
||||
ADDQ R_LEN, R_DST
|
||||
JMP loop
|
||||
|
||||
slowForwardCopy:
|
||||
// !!! If the forward copy is longer than 16 bytes, or if offset < 8, we
|
||||
// can still try 8-byte load stores, provided we can overrun up to 10 extra
|
||||
// bytes. As above, the overrun will be fixed up by subsequent iterations
|
||||
// of the outermost loop.
|
||||
//
|
||||
// The C++ snappy code calls this technique IncrementalCopyFastPath. Its
|
||||
// commentary says:
|
||||
//
|
||||
// ----
|
||||
//
|
||||
// The main part of this loop is a simple copy of eight bytes at a time
|
||||
// until we've copied (at least) the requested amount of bytes. However,
|
||||
// if d and d-offset are less than eight bytes apart (indicating a
|
||||
// repeating pattern of length < 8), we first need to expand the pattern in
|
||||
// order to get the correct results. For instance, if the buffer looks like
|
||||
// this, with the eight-byte <d-offset> and <d> patterns marked as
|
||||
// intervals:
|
||||
//
|
||||
// abxxxxxxxxxxxx
|
||||
// [------] d-offset
|
||||
// [------] d
|
||||
//
|
||||
// a single eight-byte copy from <d-offset> to <d> will repeat the pattern
|
||||
// once, after which we can move <d> two bytes without moving <d-offset>:
|
||||
//
|
||||
// ababxxxxxxxxxx
|
||||
// [------] d-offset
|
||||
// [------] d
|
||||
//
|
||||
// and repeat the exercise until the two no longer overlap.
|
||||
//
|
||||
// This allows us to do very well in the special case of one single byte
|
||||
// repeated many times, without taking a big hit for more general cases.
|
||||
//
|
||||
// The worst case of extra writing past the end of the match occurs when
|
||||
// offset == 1 and length == 1; the last copy will read from byte positions
|
||||
// [0..7] and write to [4..11], whereas it was only supposed to write to
|
||||
// position 1. Thus, ten excess bytes.
|
||||
//
|
||||
// ----
|
||||
//
|
||||
// That "10 byte overrun" worst case is confirmed by Go's
|
||||
// TestSlowForwardCopyOverrun, which also tests the fixUpSlowForwardCopy
|
||||
// and finishSlowForwardCopy algorithm.
|
||||
//
|
||||
// if length > len(dst)-d-10 {
|
||||
// goto verySlowForwardCopy
|
||||
// }
|
||||
SUBQ $10, R_TMP2
|
||||
CMPQ R_LEN, R_TMP2
|
||||
JGT verySlowForwardCopy
|
||||
|
||||
// We want to keep the offset, so we use R_TMP2 from here.
|
||||
MOVQ R_OFF, R_TMP2
|
||||
|
||||
makeOffsetAtLeast8:
|
||||
// !!! As above, expand the pattern so that offset >= 8 and we can use
|
||||
// 8-byte load/stores.
|
||||
//
|
||||
// for offset < 8 {
|
||||
// copy 8 bytes from dst[d-offset:] to dst[d:]
|
||||
// length -= offset
|
||||
// d += offset
|
||||
// offset += offset
|
||||
// // The two previous lines together means that d-offset, and therefore
|
||||
// // R_TMP3, is unchanged.
|
||||
// }
|
||||
CMPQ R_TMP2, $8
|
||||
JGE fixUpSlowForwardCopy
|
||||
MOVQ (R_TMP3), R_TMP1
|
||||
MOVQ R_TMP1, (R_DST)
|
||||
SUBQ R_TMP2, R_LEN
|
||||
ADDQ R_TMP2, R_DST
|
||||
ADDQ R_TMP2, R_TMP2
|
||||
JMP makeOffsetAtLeast8
|
||||
|
||||
fixUpSlowForwardCopy:
|
||||
// !!! Add length (which might be negative now) to d (implied by R_DST being
|
||||
// &dst[d]) so that d ends up at the right place when we jump back to the
|
||||
// top of the loop. Before we do that, though, we save R_DST to R_TMP0 so that, if
|
||||
// length is positive, copying the remaining length bytes will write to the
|
||||
// right place.
|
||||
MOVQ R_DST, R_TMP0
|
||||
ADDQ R_LEN, R_DST
|
||||
|
||||
finishSlowForwardCopy:
|
||||
// !!! Repeat 8-byte load/stores until length <= 0. Ending with a negative
|
||||
// length means that we overrun, but as above, that will be fixed up by
|
||||
// subsequent iterations of the outermost loop.
|
||||
CMPQ R_LEN, $0
|
||||
JLE loop
|
||||
MOVQ (R_TMP3), R_TMP1
|
||||
MOVQ R_TMP1, (R_TMP0)
|
||||
ADDQ $8, R_TMP3
|
||||
ADDQ $8, R_TMP0
|
||||
SUBQ $8, R_LEN
|
||||
JMP finishSlowForwardCopy
|
||||
|
||||
verySlowForwardCopy:
|
||||
// verySlowForwardCopy is a simple implementation of forward copy. In C
|
||||
// parlance, this is a do/while loop instead of a while loop, since we know
|
||||
// that length > 0. In Go syntax:
|
||||
//
|
||||
// for {
|
||||
// dst[d] = dst[d - offset]
|
||||
// d++
|
||||
// length--
|
||||
// if length == 0 {
|
||||
// break
|
||||
// }
|
||||
// }
|
||||
MOVB (R_TMP3), R_TMP1
|
||||
MOVB R_TMP1, (R_DST)
|
||||
INCQ R_TMP3
|
||||
INCQ R_DST
|
||||
DECQ R_LEN
|
||||
JNZ verySlowForwardCopy
|
||||
JMP loop
|
||||
|
||||
// The code above handles copy tags.
|
||||
// ----------------------------------------
|
||||
|
||||
end:
|
||||
// This is the end of the "for s < len(src)".
|
||||
//
|
||||
// if d != len(dst) { etc }
|
||||
CMPQ R_DST, R_DEND
|
||||
JNE errCorrupt
|
||||
|
||||
// return 0
|
||||
MOVQ $0, ret+48(FP)
|
||||
RET
|
||||
|
||||
errCorrupt:
|
||||
// return decodeErrCodeCorrupt
|
||||
MOVQ $1, ret+48(FP)
|
||||
RET
|
||||
+574
@@ -0,0 +1,574 @@
|
||||
// Copyright 2020 The Go Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
// +build !appengine
|
||||
// +build gc
|
||||
// +build !noasm
|
||||
|
||||
#include "textflag.h"
|
||||
|
||||
#define R_TMP0 R2
|
||||
#define R_TMP1 R3
|
||||
#define R_LEN R4
|
||||
#define R_OFF R5
|
||||
#define R_SRC R6
|
||||
#define R_DST R7
|
||||
#define R_DBASE R8
|
||||
#define R_DLEN R9
|
||||
#define R_DEND R10
|
||||
#define R_SBASE R11
|
||||
#define R_SLEN R12
|
||||
#define R_SEND R13
|
||||
#define R_TMP2 R14
|
||||
#define R_TMP3 R15
|
||||
|
||||
// TEST_SRC will check if R_SRC is <= SRC_END
|
||||
#define TEST_SRC() \
|
||||
CMP R_SEND, R_SRC \
|
||||
BGT errCorrupt
|
||||
|
||||
// MOVD R_SRC, R_TMP1
|
||||
// SUB R_SBASE, R_TMP1, R_TMP1
|
||||
// CMP R_SLEN, R_TMP1
|
||||
// BGT errCorrupt
|
||||
|
||||
// The asm code generally follows the pure Go code in decode_other.go, except
|
||||
// where marked with a "!!!".
|
||||
|
||||
// func decode(dst, src []byte) int
|
||||
//
|
||||
// All local variables fit into registers. The non-zero stack size is only to
|
||||
// spill registers and push args when issuing a CALL. The register allocation:
|
||||
// - R_TMP0 scratch
|
||||
// - R_TMP1 scratch
|
||||
// - R_LEN length or x
|
||||
// - R_OFF offset
|
||||
// - R_SRC &src[s]
|
||||
// - R_DST &dst[d]
|
||||
// + R_DBASE dst_base
|
||||
// + R_DLEN dst_len
|
||||
// + R_DEND dst_base + dst_len
|
||||
// + R_SBASE src_base
|
||||
// + R_SLEN src_len
|
||||
// + R_SEND src_base + src_len
|
||||
// - R_TMP2 used by doCopy
|
||||
// - R_TMP3 used by doCopy
|
||||
//
|
||||
// The registers R_DBASE-R_SEND (marked with a "+") are set at the start of the
|
||||
// function, and after a CALL returns, and are not otherwise modified.
|
||||
//
|
||||
// The d variable is implicitly R_DST - R_DBASE, and len(dst)-d is R_DEND - R_DST.
|
||||
// The s variable is implicitly R_SRC - R_SBASE, and len(src)-s is R_SEND - R_SRC.
|
||||
TEXT ·s2Decode(SB), NOSPLIT, $56-56
|
||||
// Initialize R_SRC, R_DST and R_DBASE-R_SEND.
|
||||
MOVD dst_base+0(FP), R_DBASE
|
||||
MOVD dst_len+8(FP), R_DLEN
|
||||
MOVD R_DBASE, R_DST
|
||||
MOVD R_DBASE, R_DEND
|
||||
ADD R_DLEN, R_DEND, R_DEND
|
||||
MOVD src_base+24(FP), R_SBASE
|
||||
MOVD src_len+32(FP), R_SLEN
|
||||
MOVD R_SBASE, R_SRC
|
||||
MOVD R_SBASE, R_SEND
|
||||
ADD R_SLEN, R_SEND, R_SEND
|
||||
MOVD $0, R_OFF
|
||||
|
||||
loop:
|
||||
// for s < len(src)
|
||||
CMP R_SEND, R_SRC
|
||||
BEQ end
|
||||
|
||||
// R_LEN = uint32(src[s])
|
||||
//
|
||||
// switch src[s] & 0x03
|
||||
MOVBU (R_SRC), R_LEN
|
||||
MOVW R_LEN, R_TMP1
|
||||
ANDW $3, R_TMP1
|
||||
MOVW $1, R1
|
||||
CMPW R1, R_TMP1
|
||||
BGE tagCopy
|
||||
|
||||
// ----------------------------------------
|
||||
// The code below handles literal tags.
|
||||
|
||||
// case tagLiteral:
|
||||
// x := uint32(src[s] >> 2)
|
||||
// switch
|
||||
MOVW $60, R1
|
||||
LSRW $2, R_LEN, R_LEN
|
||||
CMPW R_LEN, R1
|
||||
BLS tagLit60Plus
|
||||
|
||||
// case x < 60:
|
||||
// s++
|
||||
ADD $1, R_SRC, R_SRC
|
||||
|
||||
doLit:
|
||||
// This is the end of the inner "switch", when we have a literal tag.
|
||||
//
|
||||
// We assume that R_LEN == x and x fits in a uint32, where x is the variable
|
||||
// used in the pure Go decode_other.go code.
|
||||
|
||||
// length = int(x) + 1
|
||||
//
|
||||
// Unlike the pure Go code, we don't need to check if length <= 0 because
|
||||
// R_LEN can hold 64 bits, so the increment cannot overflow.
|
||||
ADD $1, R_LEN, R_LEN
|
||||
|
||||
// Prepare to check if copying length bytes will run past the end of dst or
|
||||
// src.
|
||||
//
|
||||
// R_TMP0 = len(dst) - d
|
||||
// R_TMP1 = len(src) - s
|
||||
MOVD R_DEND, R_TMP0
|
||||
SUB R_DST, R_TMP0, R_TMP0
|
||||
MOVD R_SEND, R_TMP1
|
||||
SUB R_SRC, R_TMP1, R_TMP1
|
||||
|
||||
// !!! Try a faster technique for short (16 or fewer bytes) copies.
|
||||
//
|
||||
// if length > 16 || len(dst)-d < 16 || len(src)-s < 16 {
|
||||
// goto callMemmove // Fall back on calling runtime·memmove.
|
||||
// }
|
||||
//
|
||||
// The C++ snappy code calls this TryFastAppend. It also checks len(src)-s
|
||||
// against 21 instead of 16, because it cannot assume that all of its input
|
||||
// is contiguous in memory and so it needs to leave enough source bytes to
|
||||
// read the next tag without refilling buffers, but Go's Decode assumes
|
||||
// contiguousness (the src argument is a []byte).
|
||||
CMP $16, R_LEN
|
||||
BGT callMemmove
|
||||
CMP $16, R_TMP0
|
||||
BLT callMemmove
|
||||
CMP $16, R_TMP1
|
||||
BLT callMemmove
|
||||
|
||||
// !!! Implement the copy from src to dst as a 16-byte load and store.
|
||||
// (Decode's documentation says that dst and src must not overlap.)
|
||||
//
|
||||
// This always copies 16 bytes, instead of only length bytes, but that's
|
||||
// OK. If the input is a valid Snappy encoding then subsequent iterations
|
||||
// will fix up the overrun. Otherwise, Decode returns a nil []byte (and a
|
||||
// non-nil error), so the overrun will be ignored.
|
||||
//
|
||||
// Note that on arm64, it is legal and cheap to issue unaligned 8-byte or
|
||||
// 16-byte loads and stores. This technique probably wouldn't be as
|
||||
// effective on architectures that are fussier about alignment.
|
||||
LDP 0(R_SRC), (R_TMP2, R_TMP3)
|
||||
STP (R_TMP2, R_TMP3), 0(R_DST)
|
||||
|
||||
// d += length
|
||||
// s += length
|
||||
ADD R_LEN, R_DST, R_DST
|
||||
ADD R_LEN, R_SRC, R_SRC
|
||||
B loop
|
||||
|
||||
callMemmove:
|
||||
// if length > len(dst)-d || length > len(src)-s { etc }
|
||||
CMP R_TMP0, R_LEN
|
||||
BGT errCorrupt
|
||||
CMP R_TMP1, R_LEN
|
||||
BGT errCorrupt
|
||||
|
||||
// copy(dst[d:], src[s:s+length])
|
||||
//
|
||||
// This means calling runtime·memmove(&dst[d], &src[s], length), so we push
|
||||
// R_DST, R_SRC and R_LEN as arguments. Coincidentally, we also need to spill those
|
||||
// three registers to the stack, to save local variables across the CALL.
|
||||
MOVD R_DST, 8(RSP)
|
||||
MOVD R_SRC, 16(RSP)
|
||||
MOVD R_LEN, 24(RSP)
|
||||
MOVD R_DST, 32(RSP)
|
||||
MOVD R_SRC, 40(RSP)
|
||||
MOVD R_LEN, 48(RSP)
|
||||
MOVD R_OFF, 56(RSP)
|
||||
CALL runtime·memmove(SB)
|
||||
|
||||
// Restore local variables: unspill registers from the stack and
|
||||
// re-calculate R_DBASE-R_SEND.
|
||||
MOVD 32(RSP), R_DST
|
||||
MOVD 40(RSP), R_SRC
|
||||
MOVD 48(RSP), R_LEN
|
||||
MOVD 56(RSP), R_OFF
|
||||
MOVD dst_base+0(FP), R_DBASE
|
||||
MOVD dst_len+8(FP), R_DLEN
|
||||
MOVD R_DBASE, R_DEND
|
||||
ADD R_DLEN, R_DEND, R_DEND
|
||||
MOVD src_base+24(FP), R_SBASE
|
||||
MOVD src_len+32(FP), R_SLEN
|
||||
MOVD R_SBASE, R_SEND
|
||||
ADD R_SLEN, R_SEND, R_SEND
|
||||
|
||||
// d += length
|
||||
// s += length
|
||||
ADD R_LEN, R_DST, R_DST
|
||||
ADD R_LEN, R_SRC, R_SRC
|
||||
B loop
|
||||
|
||||
tagLit60Plus:
|
||||
// !!! This fragment does the
|
||||
//
|
||||
// s += x - 58; if uint(s) > uint(len(src)) { etc }
|
||||
//
|
||||
// checks. In the asm version, we code it once instead of once per switch case.
|
||||
ADD R_LEN, R_SRC, R_SRC
|
||||
SUB $58, R_SRC, R_SRC
|
||||
TEST_SRC()
|
||||
|
||||
// case x == 60:
|
||||
MOVW $61, R1
|
||||
CMPW R1, R_LEN
|
||||
BEQ tagLit61
|
||||
BGT tagLit62Plus
|
||||
|
||||
// x = uint32(src[s-1])
|
||||
MOVBU -1(R_SRC), R_LEN
|
||||
B doLit
|
||||
|
||||
tagLit61:
|
||||
// case x == 61:
|
||||
// x = uint32(src[s-2]) | uint32(src[s-1])<<8
|
||||
MOVHU -2(R_SRC), R_LEN
|
||||
B doLit
|
||||
|
||||
tagLit62Plus:
|
||||
CMPW $62, R_LEN
|
||||
BHI tagLit63
|
||||
|
||||
// case x == 62:
|
||||
// x = uint32(src[s-3]) | uint32(src[s-2])<<8 | uint32(src[s-1])<<16
|
||||
MOVHU -3(R_SRC), R_LEN
|
||||
MOVBU -1(R_SRC), R_TMP1
|
||||
ORR R_TMP1<<16, R_LEN
|
||||
B doLit
|
||||
|
||||
tagLit63:
|
||||
// case x == 63:
|
||||
// x = uint32(src[s-4]) | uint32(src[s-3])<<8 | uint32(src[s-2])<<16 | uint32(src[s-1])<<24
|
||||
MOVWU -4(R_SRC), R_LEN
|
||||
B doLit
|
||||
|
||||
// The code above handles literal tags.
|
||||
// ----------------------------------------
|
||||
// The code below handles copy tags.
|
||||
|
||||
tagCopy4:
|
||||
// case tagCopy4:
|
||||
// s += 5
|
||||
ADD $5, R_SRC, R_SRC
|
||||
|
||||
// if uint(s) > uint(len(src)) { etc }
|
||||
MOVD R_SRC, R_TMP1
|
||||
SUB R_SBASE, R_TMP1, R_TMP1
|
||||
CMP R_SLEN, R_TMP1
|
||||
BGT errCorrupt
|
||||
|
||||
// length = 1 + int(src[s-5])>>2
|
||||
MOVD $1, R1
|
||||
ADD R_LEN>>2, R1, R_LEN
|
||||
|
||||
// offset = int(uint32(src[s-4]) | uint32(src[s-3])<<8 | uint32(src[s-2])<<16 | uint32(src[s-1])<<24)
|
||||
MOVWU -4(R_SRC), R_OFF
|
||||
B doCopy
|
||||
|
||||
tagCopy2:
|
||||
// case tagCopy2:
|
||||
// s += 3
|
||||
ADD $3, R_SRC, R_SRC
|
||||
|
||||
// if uint(s) > uint(len(src)) { etc }
|
||||
TEST_SRC()
|
||||
|
||||
// length = 1 + int(src[s-3])>>2
|
||||
MOVD $1, R1
|
||||
ADD R_LEN>>2, R1, R_LEN
|
||||
|
||||
// offset = int(uint32(src[s-2]) | uint32(src[s-1])<<8)
|
||||
MOVHU -2(R_SRC), R_OFF
|
||||
B doCopy
|
||||
|
||||
tagCopy:
|
||||
// We have a copy tag. We assume that:
|
||||
// - R_TMP1 == src[s] & 0x03
|
||||
// - R_LEN == src[s]
|
||||
CMP $2, R_TMP1
|
||||
BEQ tagCopy2
|
||||
BGT tagCopy4
|
||||
|
||||
// case tagCopy1:
|
||||
// s += 2
|
||||
ADD $2, R_SRC, R_SRC
|
||||
|
||||
// if uint(s) > uint(len(src)) { etc }
|
||||
TEST_SRC()
|
||||
|
||||
// offset = int(uint32(src[s-2])&0xe0<<3 | uint32(src[s-1]))
|
||||
// Calculate offset in R_TMP0 in case it is a repeat.
|
||||
MOVD R_LEN, R_TMP0
|
||||
AND $0xe0, R_TMP0
|
||||
MOVBU -1(R_SRC), R_TMP1
|
||||
ORR R_TMP0<<3, R_TMP1, R_TMP0
|
||||
|
||||
// length = 4 + int(src[s-2])>>2&0x7
|
||||
MOVD $7, R1
|
||||
AND R_LEN>>2, R1, R_LEN
|
||||
ADD $4, R_LEN, R_LEN
|
||||
|
||||
// check if repeat code with offset 0.
|
||||
CMP $0, R_TMP0
|
||||
BEQ repeatCode
|
||||
|
||||
// This is a regular copy, transfer our temporary value to R_OFF (offset)
|
||||
MOVD R_TMP0, R_OFF
|
||||
B doCopy
|
||||
|
||||
// This is a repeat code.
|
||||
repeatCode:
|
||||
// If length < 9, reuse last offset, with the length already calculated.
|
||||
CMP $9, R_LEN
|
||||
BLT doCopyRepeat
|
||||
BEQ repeatLen1
|
||||
CMP $10, R_LEN
|
||||
BEQ repeatLen2
|
||||
|
||||
repeatLen3:
|
||||
// s +=3
|
||||
ADD $3, R_SRC, R_SRC
|
||||
|
||||
// if uint(s) > uint(len(src)) { etc }
|
||||
TEST_SRC()
|
||||
|
||||
// length = uint32(src[s-3]) | (uint32(src[s-2])<<8) | (uint32(src[s-1])<<16) + 65540
|
||||
MOVBU -1(R_SRC), R_TMP0
|
||||
MOVHU -3(R_SRC), R_LEN
|
||||
ORR R_TMP0<<16, R_LEN, R_LEN
|
||||
ADD $65540, R_LEN, R_LEN
|
||||
B doCopyRepeat
|
||||
|
||||
repeatLen2:
|
||||
// s +=2
|
||||
ADD $2, R_SRC, R_SRC
|
||||
|
||||
// if uint(s) > uint(len(src)) { etc }
|
||||
TEST_SRC()
|
||||
|
||||
// length = uint32(src[s-2]) | (uint32(src[s-1])<<8) + 260
|
||||
MOVHU -2(R_SRC), R_LEN
|
||||
ADD $260, R_LEN, R_LEN
|
||||
B doCopyRepeat
|
||||
|
||||
repeatLen1:
|
||||
// s +=1
|
||||
ADD $1, R_SRC, R_SRC
|
||||
|
||||
// if uint(s) > uint(len(src)) { etc }
|
||||
TEST_SRC()
|
||||
|
||||
// length = src[s-1] + 8
|
||||
MOVBU -1(R_SRC), R_LEN
|
||||
ADD $8, R_LEN, R_LEN
|
||||
B doCopyRepeat
|
||||
|
||||
doCopy:
|
||||
// This is the end of the outer "switch", when we have a copy tag.
|
||||
//
|
||||
// We assume that:
|
||||
// - R_LEN == length && R_LEN > 0
|
||||
// - R_OFF == offset
|
||||
|
||||
// if d < offset { etc }
|
||||
MOVD R_DST, R_TMP1
|
||||
SUB R_DBASE, R_TMP1, R_TMP1
|
||||
CMP R_OFF, R_TMP1
|
||||
BLT errCorrupt
|
||||
|
||||
// Repeat values can skip the test above, since any offset > 0 will be in dst.
|
||||
doCopyRepeat:
|
||||
|
||||
// if offset <= 0 { etc }
|
||||
CMP $0, R_OFF
|
||||
BLE errCorrupt
|
||||
|
||||
// if length > len(dst)-d { etc }
|
||||
MOVD R_DEND, R_TMP1
|
||||
SUB R_DST, R_TMP1, R_TMP1
|
||||
CMP R_TMP1, R_LEN
|
||||
BGT errCorrupt
|
||||
|
||||
// forwardCopy(dst[d:d+length], dst[d-offset:]); d += length
|
||||
//
|
||||
// Set:
|
||||
// - R_TMP2 = len(dst)-d
|
||||
// - R_TMP3 = &dst[d-offset]
|
||||
MOVD R_DEND, R_TMP2
|
||||
SUB R_DST, R_TMP2, R_TMP2
|
||||
MOVD R_DST, R_TMP3
|
||||
SUB R_OFF, R_TMP3, R_TMP3
|
||||
|
||||
// !!! Try a faster technique for short (16 or fewer bytes) forward copies.
|
||||
//
|
||||
// First, try using two 8-byte load/stores, similar to the doLit technique
|
||||
// above. Even if dst[d:d+length] and dst[d-offset:] can overlap, this is
|
||||
// still OK if offset >= 8. Note that this has to be two 8-byte load/stores
|
||||
// and not one 16-byte load/store, and the first store has to be before the
|
||||
// second load, due to the overlap if offset is in the range [8, 16).
|
||||
//
|
||||
// if length > 16 || offset < 8 || len(dst)-d < 16 {
|
||||
// goto slowForwardCopy
|
||||
// }
|
||||
// copy 16 bytes
|
||||
// d += length
|
||||
CMP $16, R_LEN
|
||||
BGT slowForwardCopy
|
||||
CMP $8, R_OFF
|
||||
BLT slowForwardCopy
|
||||
CMP $16, R_TMP2
|
||||
BLT slowForwardCopy
|
||||
MOVD 0(R_TMP3), R_TMP0
|
||||
MOVD R_TMP0, 0(R_DST)
|
||||
MOVD 8(R_TMP3), R_TMP1
|
||||
MOVD R_TMP1, 8(R_DST)
|
||||
ADD R_LEN, R_DST, R_DST
|
||||
B loop
|
||||
|
||||
slowForwardCopy:
|
||||
// !!! If the forward copy is longer than 16 bytes, or if offset < 8, we
|
||||
// can still try 8-byte load stores, provided we can overrun up to 10 extra
|
||||
// bytes. As above, the overrun will be fixed up by subsequent iterations
|
||||
// of the outermost loop.
|
||||
//
|
||||
// The C++ snappy code calls this technique IncrementalCopyFastPath. Its
|
||||
// commentary says:
|
||||
//
|
||||
// ----
|
||||
//
|
||||
// The main part of this loop is a simple copy of eight bytes at a time
|
||||
// until we've copied (at least) the requested amount of bytes. However,
|
||||
// if d and d-offset are less than eight bytes apart (indicating a
|
||||
// repeating pattern of length < 8), we first need to expand the pattern in
|
||||
// order to get the correct results. For instance, if the buffer looks like
|
||||
// this, with the eight-byte <d-offset> and <d> patterns marked as
|
||||
// intervals:
|
||||
//
|
||||
// abxxxxxxxxxxxx
|
||||
// [------] d-offset
|
||||
// [------] d
|
||||
//
|
||||
// a single eight-byte copy from <d-offset> to <d> will repeat the pattern
|
||||
// once, after which we can move <d> two bytes without moving <d-offset>:
|
||||
//
|
||||
// ababxxxxxxxxxx
|
||||
// [------] d-offset
|
||||
// [------] d
|
||||
//
|
||||
// and repeat the exercise until the two no longer overlap.
|
||||
//
|
||||
// This allows us to do very well in the special case of one single byte
|
||||
// repeated many times, without taking a big hit for more general cases.
|
||||
//
|
||||
// The worst case of extra writing past the end of the match occurs when
|
||||
// offset == 1 and length == 1; the last copy will read from byte positions
|
||||
// [0..7] and write to [4..11], whereas it was only supposed to write to
|
||||
// position 1. Thus, ten excess bytes.
|
||||
//
|
||||
// ----
|
||||
//
|
||||
// That "10 byte overrun" worst case is confirmed by Go's
|
||||
// TestSlowForwardCopyOverrun, which also tests the fixUpSlowForwardCopy
|
||||
// and finishSlowForwardCopy algorithm.
|
||||
//
|
||||
// if length > len(dst)-d-10 {
|
||||
// goto verySlowForwardCopy
|
||||
// }
|
||||
SUB $10, R_TMP2, R_TMP2
|
||||
CMP R_TMP2, R_LEN
|
||||
BGT verySlowForwardCopy
|
||||
|
||||
// We want to keep the offset, so we use R_TMP2 from here.
|
||||
MOVD R_OFF, R_TMP2
|
||||
|
||||
makeOffsetAtLeast8:
|
||||
// !!! As above, expand the pattern so that offset >= 8 and we can use
|
||||
// 8-byte load/stores.
|
||||
//
|
||||
// for offset < 8 {
|
||||
// copy 8 bytes from dst[d-offset:] to dst[d:]
|
||||
// length -= offset
|
||||
// d += offset
|
||||
// offset += offset
|
||||
// // The two previous lines together means that d-offset, and therefore
|
||||
// // R_TMP3, is unchanged.
|
||||
// }
|
||||
CMP $8, R_TMP2
|
||||
BGE fixUpSlowForwardCopy
|
||||
MOVD (R_TMP3), R_TMP1
|
||||
MOVD R_TMP1, (R_DST)
|
||||
SUB R_TMP2, R_LEN, R_LEN
|
||||
ADD R_TMP2, R_DST, R_DST
|
||||
ADD R_TMP2, R_TMP2, R_TMP2
|
||||
B makeOffsetAtLeast8
|
||||
|
||||
fixUpSlowForwardCopy:
|
||||
// !!! Add length (which might be negative now) to d (implied by R_DST being
|
||||
// &dst[d]) so that d ends up at the right place when we jump back to the
|
||||
// top of the loop. Before we do that, though, we save R_DST to R_TMP0 so that, if
|
||||
// length is positive, copying the remaining length bytes will write to the
|
||||
// right place.
|
||||
MOVD R_DST, R_TMP0
|
||||
ADD R_LEN, R_DST, R_DST
|
||||
|
||||
finishSlowForwardCopy:
|
||||
// !!! Repeat 8-byte load/stores until length <= 0. Ending with a negative
|
||||
// length means that we overrun, but as above, that will be fixed up by
|
||||
// subsequent iterations of the outermost loop.
|
||||
MOVD $0, R1
|
||||
CMP R1, R_LEN
|
||||
BLE loop
|
||||
MOVD (R_TMP3), R_TMP1
|
||||
MOVD R_TMP1, (R_TMP0)
|
||||
ADD $8, R_TMP3, R_TMP3
|
||||
ADD $8, R_TMP0, R_TMP0
|
||||
SUB $8, R_LEN, R_LEN
|
||||
B finishSlowForwardCopy
|
||||
|
||||
verySlowForwardCopy:
|
||||
// verySlowForwardCopy is a simple implementation of forward copy. In C
|
||||
// parlance, this is a do/while loop instead of a while loop, since we know
|
||||
// that length > 0. In Go syntax:
|
||||
//
|
||||
// for {
|
||||
// dst[d] = dst[d - offset]
|
||||
// d++
|
||||
// length--
|
||||
// if length == 0 {
|
||||
// break
|
||||
// }
|
||||
// }
|
||||
MOVB (R_TMP3), R_TMP1
|
||||
MOVB R_TMP1, (R_DST)
|
||||
ADD $1, R_TMP3, R_TMP3
|
||||
ADD $1, R_DST, R_DST
|
||||
SUB $1, R_LEN, R_LEN
|
||||
CBNZ R_LEN, verySlowForwardCopy
|
||||
B loop
|
||||
|
||||
// The code above handles copy tags.
|
||||
// ----------------------------------------
|
||||
|
||||
end:
|
||||
// This is the end of the "for s < len(src)".
|
||||
//
|
||||
// if d != len(dst) { etc }
|
||||
CMP R_DEND, R_DST
|
||||
BNE errCorrupt
|
||||
|
||||
// return 0
|
||||
MOVD $0, ret+48(FP)
|
||||
RET
|
||||
|
||||
errCorrupt:
|
||||
// return decodeErrCodeCorrupt
|
||||
MOVD $1, R_TMP0
|
||||
MOVD R_TMP0, ret+48(FP)
|
||||
RET
|
||||
+17
@@ -0,0 +1,17 @@
|
||||
// Copyright 2016 The Snappy-Go Authors. All rights reserved.
|
||||
// Copyright (c) 2019 Klaus Post. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
//go:build (amd64 || arm64) && !appengine && gc && !noasm
|
||||
// +build amd64 arm64
|
||||
// +build !appengine
|
||||
// +build gc
|
||||
// +build !noasm
|
||||
|
||||
package s2
|
||||
|
||||
// decode has the same semantics as in decode_other.go.
|
||||
//
|
||||
//go:noescape
|
||||
func s2Decode(dst, src []byte) int
|
||||
+287
@@ -0,0 +1,287 @@
|
||||
// Copyright 2016 The Snappy-Go Authors. All rights reserved.
|
||||
// Copyright (c) 2019 Klaus Post. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
//go:build (!amd64 && !arm64) || appengine || !gc || noasm
|
||||
|
||||
package s2
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strconv"
|
||||
|
||||
"github.com/klauspost/compress/internal/le"
|
||||
)
|
||||
|
||||
// decode writes the decoding of src to dst. It assumes that the varint-encoded
|
||||
// length of the decompressed bytes has already been read, and that len(dst)
|
||||
// equals that length.
|
||||
//
|
||||
// It returns 0 on success or a decodeErrCodeXxx error code on failure.
|
||||
func s2Decode(dst, src []byte) int {
|
||||
const debug = false
|
||||
if debug {
|
||||
fmt.Println("Starting decode, dst len:", len(dst))
|
||||
}
|
||||
var d, s, length int
|
||||
offset := 0
|
||||
|
||||
// As long as we can read at least 5 bytes...
|
||||
for s < len(src)-5 {
|
||||
// Removing bounds checks is SLOWER, when if doing
|
||||
// in := src[s:s+5]
|
||||
// Checked on Go 1.18
|
||||
switch src[s] & 0x03 {
|
||||
case tagLiteral:
|
||||
x := uint32(src[s] >> 2)
|
||||
switch {
|
||||
case x < 60:
|
||||
s++
|
||||
case x == 60:
|
||||
x = uint32(src[s+1])
|
||||
s += 2
|
||||
case x == 61:
|
||||
x = uint32(le.Load16(src, s+1))
|
||||
s += 3
|
||||
case x == 62:
|
||||
// Load as 32 bit and shift down.
|
||||
x = le.Load32(src, s)
|
||||
x >>= 8
|
||||
s += 4
|
||||
case x == 63:
|
||||
x = le.Load32(src, s+1)
|
||||
s += 5
|
||||
}
|
||||
length = int(x) + 1
|
||||
if length > len(dst)-d || length > len(src)-s || (strconv.IntSize == 32 && length <= 0) {
|
||||
if debug {
|
||||
fmt.Println("corrupt: lit size", length)
|
||||
}
|
||||
return decodeErrCodeCorrupt
|
||||
}
|
||||
if debug {
|
||||
fmt.Println("literals, length:", length, "d-after:", d+length)
|
||||
}
|
||||
|
||||
copy(dst[d:], src[s:s+length])
|
||||
d += length
|
||||
s += length
|
||||
continue
|
||||
|
||||
case tagCopy1:
|
||||
s += 2
|
||||
toffset := int(uint32(src[s-2])&0xe0<<3 | uint32(src[s-1]))
|
||||
length = int(src[s-2]) >> 2 & 0x7
|
||||
if toffset == 0 {
|
||||
if debug {
|
||||
fmt.Print("(repeat) ")
|
||||
}
|
||||
// keep last offset
|
||||
switch length {
|
||||
case 5:
|
||||
length = int(src[s]) + 4
|
||||
s += 1
|
||||
case 6:
|
||||
length = int(le.Load16(src, s)) + 1<<8
|
||||
s += 2
|
||||
case 7:
|
||||
in := src[s : s+3]
|
||||
length = int((uint32(in[2])<<16)|(uint32(in[1])<<8)|uint32(in[0])) + (1 << 16)
|
||||
s += 3
|
||||
default: // 0-> 4
|
||||
}
|
||||
} else {
|
||||
offset = toffset
|
||||
}
|
||||
length += 4
|
||||
case tagCopy2:
|
||||
offset = int(le.Load16(src, s+1))
|
||||
length = 1 + int(src[s])>>2
|
||||
s += 3
|
||||
|
||||
case tagCopy4:
|
||||
offset = int(le.Load32(src, s+1))
|
||||
length = 1 + int(src[s])>>2
|
||||
s += 5
|
||||
}
|
||||
|
||||
if offset <= 0 || d < offset || length > len(dst)-d {
|
||||
if debug {
|
||||
fmt.Println("corrupt: match, length", length, "offset:", offset, "dst avail:", len(dst)-d, "dst pos:", d)
|
||||
}
|
||||
|
||||
return decodeErrCodeCorrupt
|
||||
}
|
||||
|
||||
if debug {
|
||||
fmt.Println("copy, length:", length, "offset:", offset, "d-after:", d+length)
|
||||
}
|
||||
|
||||
// Copy from an earlier sub-slice of dst to a later sub-slice.
|
||||
// If no overlap, use the built-in copy:
|
||||
if offset > length {
|
||||
copy(dst[d:d+length], dst[d-offset:])
|
||||
d += length
|
||||
continue
|
||||
}
|
||||
|
||||
// Unlike the built-in copy function, this byte-by-byte copy always runs
|
||||
// forwards, even if the slices overlap. Conceptually, this is:
|
||||
//
|
||||
// d += forwardCopy(dst[d:d+length], dst[d-offset:])
|
||||
//
|
||||
// We align the slices into a and b and show the compiler they are the same size.
|
||||
// This allows the loop to run without bounds checks.
|
||||
a := dst[d : d+length]
|
||||
b := dst[d-offset:]
|
||||
b = b[:len(a)]
|
||||
for i := range a {
|
||||
a[i] = b[i]
|
||||
}
|
||||
d += length
|
||||
}
|
||||
|
||||
// Remaining with extra checks...
|
||||
for s < len(src) {
|
||||
switch src[s] & 0x03 {
|
||||
case tagLiteral:
|
||||
x := uint32(src[s] >> 2)
|
||||
switch {
|
||||
case x < 60:
|
||||
s++
|
||||
case x == 60:
|
||||
s += 2
|
||||
if uint(s) > uint(len(src)) { // The uint conversions catch overflow from the previous line.
|
||||
return decodeErrCodeCorrupt
|
||||
}
|
||||
x = uint32(src[s-1])
|
||||
case x == 61:
|
||||
s += 3
|
||||
if uint(s) > uint(len(src)) { // The uint conversions catch overflow from the previous line.
|
||||
return decodeErrCodeCorrupt
|
||||
}
|
||||
x = uint32(src[s-2]) | uint32(src[s-1])<<8
|
||||
case x == 62:
|
||||
s += 4
|
||||
if uint(s) > uint(len(src)) { // The uint conversions catch overflow from the previous line.
|
||||
return decodeErrCodeCorrupt
|
||||
}
|
||||
x = uint32(src[s-3]) | uint32(src[s-2])<<8 | uint32(src[s-1])<<16
|
||||
case x == 63:
|
||||
s += 5
|
||||
if uint(s) > uint(len(src)) { // The uint conversions catch overflow from the previous line.
|
||||
return decodeErrCodeCorrupt
|
||||
}
|
||||
x = uint32(src[s-4]) | uint32(src[s-3])<<8 | uint32(src[s-2])<<16 | uint32(src[s-1])<<24
|
||||
}
|
||||
length = int(x) + 1
|
||||
if length > len(dst)-d || length > len(src)-s || (strconv.IntSize == 32 && length <= 0) {
|
||||
if debug {
|
||||
fmt.Println("corrupt: lit size", length)
|
||||
}
|
||||
return decodeErrCodeCorrupt
|
||||
}
|
||||
if debug {
|
||||
fmt.Println("literals, length:", length, "d-after:", d+length)
|
||||
}
|
||||
|
||||
copy(dst[d:], src[s:s+length])
|
||||
d += length
|
||||
s += length
|
||||
continue
|
||||
|
||||
case tagCopy1:
|
||||
s += 2
|
||||
if uint(s) > uint(len(src)) { // The uint conversions catch overflow from the previous line.
|
||||
return decodeErrCodeCorrupt
|
||||
}
|
||||
length = int(src[s-2]) >> 2 & 0x7
|
||||
toffset := int(uint32(src[s-2])&0xe0<<3 | uint32(src[s-1]))
|
||||
if toffset == 0 {
|
||||
if debug {
|
||||
fmt.Print("(repeat) ")
|
||||
}
|
||||
// keep last offset
|
||||
switch length {
|
||||
case 5:
|
||||
s += 1
|
||||
if uint(s) > uint(len(src)) { // The uint conversions catch overflow from the previous line.
|
||||
return decodeErrCodeCorrupt
|
||||
}
|
||||
length = int(uint32(src[s-1])) + 4
|
||||
case 6:
|
||||
s += 2
|
||||
if uint(s) > uint(len(src)) { // The uint conversions catch overflow from the previous line.
|
||||
return decodeErrCodeCorrupt
|
||||
}
|
||||
length = int(uint32(src[s-2])|(uint32(src[s-1])<<8)) + (1 << 8)
|
||||
case 7:
|
||||
s += 3
|
||||
if uint(s) > uint(len(src)) { // The uint conversions catch overflow from the previous line.
|
||||
return decodeErrCodeCorrupt
|
||||
}
|
||||
length = int(uint32(src[s-3])|(uint32(src[s-2])<<8)|(uint32(src[s-1])<<16)) + (1 << 16)
|
||||
default: // 0-> 4
|
||||
}
|
||||
} else {
|
||||
offset = toffset
|
||||
}
|
||||
length += 4
|
||||
case tagCopy2:
|
||||
s += 3
|
||||
if uint(s) > uint(len(src)) { // The uint conversions catch overflow from the previous line.
|
||||
return decodeErrCodeCorrupt
|
||||
}
|
||||
length = 1 + int(src[s-3])>>2
|
||||
offset = int(uint32(src[s-2]) | uint32(src[s-1])<<8)
|
||||
|
||||
case tagCopy4:
|
||||
s += 5
|
||||
if uint(s) > uint(len(src)) { // The uint conversions catch overflow from the previous line.
|
||||
return decodeErrCodeCorrupt
|
||||
}
|
||||
length = 1 + int(src[s-5])>>2
|
||||
offset = int(uint32(src[s-4]) | uint32(src[s-3])<<8 | uint32(src[s-2])<<16 | uint32(src[s-1])<<24)
|
||||
}
|
||||
|
||||
if offset <= 0 || d < offset || length > len(dst)-d {
|
||||
if debug {
|
||||
fmt.Println("corrupt: match, length", length, "offset:", offset, "dst avail:", len(dst)-d, "dst pos:", d)
|
||||
}
|
||||
return decodeErrCodeCorrupt
|
||||
}
|
||||
|
||||
if debug {
|
||||
fmt.Println("copy, length:", length, "offset:", offset, "d-after:", d+length)
|
||||
}
|
||||
|
||||
// Copy from an earlier sub-slice of dst to a later sub-slice.
|
||||
// If no overlap, use the built-in copy:
|
||||
if offset > length {
|
||||
copy(dst[d:d+length], dst[d-offset:])
|
||||
d += length
|
||||
continue
|
||||
}
|
||||
|
||||
// Unlike the built-in copy function, this byte-by-byte copy always runs
|
||||
// forwards, even if the slices overlap. Conceptually, this is:
|
||||
//
|
||||
// d += forwardCopy(dst[d:d+length], dst[d-offset:])
|
||||
//
|
||||
// We align the slices into a and b and show the compiler they are the same size.
|
||||
// This allows the loop to run without bounds checks.
|
||||
a := dst[d : d+length]
|
||||
b := dst[d-offset:]
|
||||
b = b[:len(a)]
|
||||
for i := range a {
|
||||
a[i] = b[i]
|
||||
}
|
||||
d += length
|
||||
}
|
||||
|
||||
if d != len(dst) {
|
||||
return decodeErrCodeCorrupt
|
||||
}
|
||||
return 0
|
||||
}
|
||||
+352
@@ -0,0 +1,352 @@
|
||||
// Copyright (c) 2022+ Klaus Post. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package s2
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/binary"
|
||||
"sync"
|
||||
)
|
||||
|
||||
const (
|
||||
// MinDictSize is the minimum dictionary size when repeat has been read.
|
||||
MinDictSize = 16
|
||||
|
||||
// MaxDictSize is the maximum dictionary size when repeat has been read.
|
||||
MaxDictSize = 65536
|
||||
|
||||
// MaxDictSrcOffset is the maximum offset where a dictionary entry can start.
|
||||
MaxDictSrcOffset = 65535
|
||||
)
|
||||
|
||||
// Dict contains a dictionary that can be used for encoding and decoding s2
|
||||
type Dict struct {
|
||||
dict []byte
|
||||
repeat int // Repeat as index of dict
|
||||
|
||||
fast, better, best sync.Once
|
||||
fastTable *[1 << 14]uint16
|
||||
|
||||
betterTableShort *[1 << 14]uint16
|
||||
betterTableLong *[1 << 17]uint16
|
||||
|
||||
bestTableShort *[1 << 16]uint32
|
||||
bestTableLong *[1 << 19]uint32
|
||||
}
|
||||
|
||||
// NewDict will read a dictionary.
|
||||
// It will return nil if the dictionary is invalid.
|
||||
func NewDict(dict []byte) *Dict {
|
||||
if len(dict) == 0 {
|
||||
return nil
|
||||
}
|
||||
var d Dict
|
||||
// Repeat is the first value of the dict
|
||||
r, n := binary.Uvarint(dict)
|
||||
if n <= 0 {
|
||||
return nil
|
||||
}
|
||||
dict = dict[n:]
|
||||
d.dict = dict
|
||||
if cap(d.dict) < len(d.dict)+16 {
|
||||
d.dict = append(make([]byte, 0, len(d.dict)+16), d.dict...)
|
||||
}
|
||||
if len(dict) < MinDictSize || len(dict) > MaxDictSize {
|
||||
return nil
|
||||
}
|
||||
// Compare as uint64: int(r) would wrap negative for r > MaxInt64,
|
||||
// slipping past the bounds check and causing an OOB read in encode.
|
||||
if r > uint64(len(dict)) {
|
||||
return nil
|
||||
}
|
||||
d.repeat = int(r)
|
||||
return &d
|
||||
}
|
||||
|
||||
// Bytes will return a serialized version of the dictionary.
|
||||
// The output can be sent to NewDict.
|
||||
func (d *Dict) Bytes() []byte {
|
||||
dst := make([]byte, binary.MaxVarintLen16+len(d.dict))
|
||||
return append(dst[:binary.PutUvarint(dst, uint64(d.repeat))], d.dict...)
|
||||
}
|
||||
|
||||
// MakeDict will create a dictionary.
|
||||
// 'data' must be at least MinDictSize.
|
||||
// If data is longer than MaxDictSize only the last MaxDictSize bytes will be used.
|
||||
// If searchStart is set the start repeat value will be set to the last
|
||||
// match of this content.
|
||||
// If no matches are found, it will attempt to find shorter matches.
|
||||
// This content should match the typical start of a block.
|
||||
// If at least 4 bytes cannot be matched, repeat is set to start of block.
|
||||
func MakeDict(data []byte, searchStart []byte) *Dict {
|
||||
if len(data) == 0 {
|
||||
return nil
|
||||
}
|
||||
if len(data) > MaxDictSize {
|
||||
data = data[len(data)-MaxDictSize:]
|
||||
}
|
||||
var d Dict
|
||||
dict := data
|
||||
d.dict = dict
|
||||
if cap(d.dict) < len(d.dict)+16 {
|
||||
d.dict = append(make([]byte, 0, len(d.dict)+16), d.dict...)
|
||||
}
|
||||
if len(dict) < MinDictSize {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Find the longest match possible, last entry if multiple.
|
||||
for s := len(searchStart); s > 4; s-- {
|
||||
if idx := bytes.LastIndex(data, searchStart[:s]); idx >= 0 && idx <= len(data)-8 {
|
||||
d.repeat = idx
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
return &d
|
||||
}
|
||||
|
||||
// MakeDictManual will create a dictionary.
|
||||
// 'data' must be at least MinDictSize and less than or equal to MaxDictSize.
|
||||
// A manual first repeat index into data must be provided.
|
||||
// It must be less than len(data)-8.
|
||||
func MakeDictManual(data []byte, firstIdx uint16) *Dict {
|
||||
if len(data) < MinDictSize || int(firstIdx) >= len(data)-8 || len(data) > MaxDictSize {
|
||||
return nil
|
||||
}
|
||||
var d Dict
|
||||
dict := data
|
||||
d.dict = dict
|
||||
if cap(d.dict) < len(d.dict)+16 {
|
||||
d.dict = append(make([]byte, 0, len(d.dict)+16), d.dict...)
|
||||
}
|
||||
|
||||
d.repeat = int(firstIdx)
|
||||
return &d
|
||||
}
|
||||
|
||||
// Encode returns the encoded form of src. The returned slice may be a sub-
|
||||
// slice of dst if dst was large enough to hold the entire encoded block.
|
||||
// Otherwise, a newly allocated slice will be returned.
|
||||
//
|
||||
// The dst and src must not overlap. It is valid to pass a nil dst.
|
||||
//
|
||||
// The blocks will require the same amount of memory to decode as encoding,
|
||||
// and does not make for concurrent decoding.
|
||||
// Also note that blocks do not contain CRC information, so corruption may be undetected.
|
||||
//
|
||||
// If you need to encode larger amounts of data, consider using
|
||||
// the streaming interface which gives all of these features.
|
||||
func (d *Dict) Encode(dst, src []byte) []byte {
|
||||
if n := MaxEncodedLen(len(src)); n < 0 {
|
||||
panic(ErrTooLarge)
|
||||
} else if cap(dst) < n {
|
||||
dst = make([]byte, n)
|
||||
} else {
|
||||
dst = dst[:n]
|
||||
}
|
||||
|
||||
// The block starts with the varint-encoded length of the decompressed bytes.
|
||||
dstP := binary.PutUvarint(dst, uint64(len(src)))
|
||||
|
||||
if len(src) == 0 {
|
||||
return dst[:dstP]
|
||||
}
|
||||
if len(src) < minNonLiteralBlockSize {
|
||||
dstP += emitLiteral(dst[dstP:], src)
|
||||
return dst[:dstP]
|
||||
}
|
||||
n := encodeBlockDictGo(dst[dstP:], src, d)
|
||||
if n > 0 {
|
||||
dstP += n
|
||||
return dst[:dstP]
|
||||
}
|
||||
// Not compressible
|
||||
dstP += emitLiteral(dst[dstP:], src)
|
||||
return dst[:dstP]
|
||||
}
|
||||
|
||||
// EncodeBetter returns the encoded form of src. The returned slice may be a sub-
|
||||
// slice of dst if dst was large enough to hold the entire encoded block.
|
||||
// Otherwise, a newly allocated slice will be returned.
|
||||
//
|
||||
// EncodeBetter compresses better than Encode but typically with a
|
||||
// 10-40% speed decrease on both compression and decompression.
|
||||
//
|
||||
// The dst and src must not overlap. It is valid to pass a nil dst.
|
||||
//
|
||||
// The blocks will require the same amount of memory to decode as encoding,
|
||||
// and does not make for concurrent decoding.
|
||||
// Also note that blocks do not contain CRC information, so corruption may be undetected.
|
||||
//
|
||||
// If you need to encode larger amounts of data, consider using
|
||||
// the streaming interface which gives all of these features.
|
||||
func (d *Dict) EncodeBetter(dst, src []byte) []byte {
|
||||
if n := MaxEncodedLen(len(src)); n < 0 {
|
||||
panic(ErrTooLarge)
|
||||
} else if len(dst) < n {
|
||||
dst = make([]byte, n)
|
||||
}
|
||||
|
||||
// The block starts with the varint-encoded length of the decompressed bytes.
|
||||
dstP := binary.PutUvarint(dst, uint64(len(src)))
|
||||
|
||||
if len(src) == 0 {
|
||||
return dst[:dstP]
|
||||
}
|
||||
if len(src) < minNonLiteralBlockSize {
|
||||
dstP += emitLiteral(dst[dstP:], src)
|
||||
return dst[:dstP]
|
||||
}
|
||||
n := encodeBlockBetterDict(dst[dstP:], src, d)
|
||||
if n > 0 {
|
||||
dstP += n
|
||||
return dst[:dstP]
|
||||
}
|
||||
// Not compressible
|
||||
dstP += emitLiteral(dst[dstP:], src)
|
||||
return dst[:dstP]
|
||||
}
|
||||
|
||||
// EncodeBest returns the encoded form of src. The returned slice may be a sub-
|
||||
// slice of dst if dst was large enough to hold the entire encoded block.
|
||||
// Otherwise, a newly allocated slice will be returned.
|
||||
//
|
||||
// EncodeBest compresses as good as reasonably possible but with a
|
||||
// big speed decrease.
|
||||
//
|
||||
// The dst and src must not overlap. It is valid to pass a nil dst.
|
||||
//
|
||||
// The blocks will require the same amount of memory to decode as encoding,
|
||||
// and does not make for concurrent decoding.
|
||||
// Also note that blocks do not contain CRC information, so corruption may be undetected.
|
||||
//
|
||||
// If you need to encode larger amounts of data, consider using
|
||||
// the streaming interface which gives all of these features.
|
||||
func (d *Dict) EncodeBest(dst, src []byte) []byte {
|
||||
if n := MaxEncodedLen(len(src)); n < 0 {
|
||||
panic(ErrTooLarge)
|
||||
} else if len(dst) < n {
|
||||
dst = make([]byte, n)
|
||||
}
|
||||
|
||||
// The block starts with the varint-encoded length of the decompressed bytes.
|
||||
dstP := binary.PutUvarint(dst, uint64(len(src)))
|
||||
|
||||
if len(src) == 0 {
|
||||
return dst[:dstP]
|
||||
}
|
||||
if len(src) < minNonLiteralBlockSize {
|
||||
dstP += emitLiteral(dst[dstP:], src)
|
||||
return dst[:dstP]
|
||||
}
|
||||
n := encodeBlockBest(dst[dstP:], src, d)
|
||||
if n > 0 {
|
||||
dstP += n
|
||||
return dst[:dstP]
|
||||
}
|
||||
// Not compressible
|
||||
dstP += emitLiteral(dst[dstP:], src)
|
||||
return dst[:dstP]
|
||||
}
|
||||
|
||||
// Decode returns the decoded form of src. The returned slice may be a sub-
|
||||
// slice of dst if dst was large enough to hold the entire decoded block.
|
||||
// Otherwise, a newly allocated slice will be returned.
|
||||
//
|
||||
// The dst and src must not overlap. It is valid to pass a nil dst.
|
||||
func (d *Dict) Decode(dst, src []byte) ([]byte, error) {
|
||||
dLen, s, err := decodedLen(src)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if dLen <= cap(dst) {
|
||||
dst = dst[:dLen]
|
||||
} else {
|
||||
dst = make([]byte, dLen)
|
||||
}
|
||||
if s2DecodeDict(dst, src[s:], d) != 0 {
|
||||
return nil, ErrCorrupt
|
||||
}
|
||||
return dst, nil
|
||||
}
|
||||
|
||||
func (d *Dict) initFast() {
|
||||
d.fast.Do(func() {
|
||||
const (
|
||||
tableBits = 14
|
||||
maxTableSize = 1 << tableBits
|
||||
)
|
||||
|
||||
var table [maxTableSize]uint16
|
||||
// We stop so any entry of length 8 can always be read.
|
||||
for i := 0; i < len(d.dict)-8-2; i += 3 {
|
||||
x0 := load64(d.dict, i)
|
||||
h0 := hash6(x0, tableBits)
|
||||
h1 := hash6(x0>>8, tableBits)
|
||||
h2 := hash6(x0>>16, tableBits)
|
||||
table[h0] = uint16(i)
|
||||
table[h1] = uint16(i + 1)
|
||||
table[h2] = uint16(i + 2)
|
||||
}
|
||||
d.fastTable = &table
|
||||
})
|
||||
}
|
||||
|
||||
func (d *Dict) initBetter() {
|
||||
d.better.Do(func() {
|
||||
const (
|
||||
// Long hash matches.
|
||||
lTableBits = 17
|
||||
maxLTableSize = 1 << lTableBits
|
||||
|
||||
// Short hash matches.
|
||||
sTableBits = 14
|
||||
maxSTableSize = 1 << sTableBits
|
||||
)
|
||||
|
||||
var lTable [maxLTableSize]uint16
|
||||
var sTable [maxSTableSize]uint16
|
||||
|
||||
// We stop so any entry of length 8 can always be read.
|
||||
for i := 0; i < len(d.dict)-8; i++ {
|
||||
cv := load64(d.dict, i)
|
||||
lTable[hash7(cv, lTableBits)] = uint16(i)
|
||||
sTable[hash4(cv, sTableBits)] = uint16(i)
|
||||
}
|
||||
d.betterTableShort = &sTable
|
||||
d.betterTableLong = &lTable
|
||||
})
|
||||
}
|
||||
|
||||
func (d *Dict) initBest() {
|
||||
d.best.Do(func() {
|
||||
const (
|
||||
// Long hash matches.
|
||||
lTableBits = 19
|
||||
maxLTableSize = 1 << lTableBits
|
||||
|
||||
// Short hash matches.
|
||||
sTableBits = 16
|
||||
maxSTableSize = 1 << sTableBits
|
||||
)
|
||||
|
||||
var lTable [maxLTableSize]uint32
|
||||
var sTable [maxSTableSize]uint32
|
||||
|
||||
// We stop so any entry of length 8 can always be read.
|
||||
for i := 0; i < len(d.dict)-8; i++ {
|
||||
cv := load64(d.dict, i)
|
||||
hashL := hash8(cv, lTableBits)
|
||||
hashS := hash4(cv, sTableBits)
|
||||
candidateL := lTable[hashL]
|
||||
candidateS := sTable[hashS]
|
||||
lTable[hashL] = uint32(i) | candidateL<<16
|
||||
sTable[hashS] = uint32(i) | candidateS<<16
|
||||
}
|
||||
d.bestTableShort = &sTable
|
||||
d.bestTableLong = &lTable
|
||||
})
|
||||
}
|
||||
+418
@@ -0,0 +1,418 @@
|
||||
// Copyright 2011 The Snappy-Go Authors. All rights reserved.
|
||||
// Copyright (c) 2019 Klaus Post. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package s2
|
||||
|
||||
import (
|
||||
"encoding/binary"
|
||||
"math"
|
||||
"math/bits"
|
||||
"sync"
|
||||
|
||||
"github.com/klauspost/compress/internal/race"
|
||||
)
|
||||
|
||||
// Encode returns the encoded form of src. The returned slice may be a sub-
|
||||
// slice of dst if dst was large enough to hold the entire encoded block.
|
||||
// Otherwise, a newly allocated slice will be returned.
|
||||
//
|
||||
// The dst and src must not overlap. It is valid to pass a nil dst.
|
||||
//
|
||||
// The blocks will require the same amount of memory to decode as encoding,
|
||||
// and does not make for concurrent decoding.
|
||||
// Also note that blocks do not contain CRC information, so corruption may be undetected.
|
||||
//
|
||||
// If you need to encode larger amounts of data, consider using
|
||||
// the streaming interface which gives all of these features.
|
||||
func Encode(dst, src []byte) []byte {
|
||||
if n := MaxEncodedLen(len(src)); n < 0 {
|
||||
panic(ErrTooLarge)
|
||||
} else if cap(dst) < n {
|
||||
dst = make([]byte, n)
|
||||
} else {
|
||||
dst = dst[:n]
|
||||
}
|
||||
|
||||
// The block starts with the varint-encoded length of the decompressed bytes.
|
||||
d := binary.PutUvarint(dst, uint64(len(src)))
|
||||
|
||||
if len(src) == 0 {
|
||||
return dst[:d]
|
||||
}
|
||||
if len(src) < minNonLiteralBlockSize {
|
||||
d += emitLiteral(dst[d:], src)
|
||||
return dst[:d]
|
||||
}
|
||||
n := encodeBlock(dst[d:], src)
|
||||
if n > 0 {
|
||||
d += n
|
||||
return dst[:d]
|
||||
}
|
||||
// Not compressible
|
||||
d += emitLiteral(dst[d:], src)
|
||||
return dst[:d]
|
||||
}
|
||||
|
||||
var estblockPool [2]sync.Pool
|
||||
|
||||
// EstimateBlockSize will perform a very fast compression
|
||||
// without outputting the result and return the compressed output size.
|
||||
// The function returns -1 if no improvement could be achieved.
|
||||
// Using actual compression will most often produce better compression than the estimate.
|
||||
func EstimateBlockSize(src []byte) (d int) {
|
||||
if len(src) <= inputMargin || int64(len(src)) > 0xffffffff {
|
||||
return -1
|
||||
}
|
||||
if len(src) <= 1024 {
|
||||
const sz, pool = 2048, 0
|
||||
tmp, ok := estblockPool[pool].Get().(*[sz]byte)
|
||||
if !ok {
|
||||
tmp = &[sz]byte{}
|
||||
}
|
||||
race.WriteSlice(tmp[:])
|
||||
defer estblockPool[pool].Put(tmp)
|
||||
|
||||
d = calcBlockSizeSmall(src, tmp)
|
||||
} else {
|
||||
const sz, pool = 32768, 1
|
||||
tmp, ok := estblockPool[pool].Get().(*[sz]byte)
|
||||
if !ok {
|
||||
tmp = &[sz]byte{}
|
||||
}
|
||||
race.WriteSlice(tmp[:])
|
||||
defer estblockPool[pool].Put(tmp)
|
||||
|
||||
d = calcBlockSize(src, tmp)
|
||||
}
|
||||
|
||||
if d == 0 {
|
||||
return -1
|
||||
}
|
||||
// Size of the varint encoded block size.
|
||||
d += (bits.Len64(uint64(len(src))) + 7) / 7
|
||||
|
||||
if d >= len(src) {
|
||||
return -1
|
||||
}
|
||||
return d
|
||||
}
|
||||
|
||||
// EncodeBetter returns the encoded form of src. The returned slice may be a sub-
|
||||
// slice of dst if dst was large enough to hold the entire encoded block.
|
||||
// Otherwise, a newly allocated slice will be returned.
|
||||
//
|
||||
// EncodeBetter compresses better than Encode but typically with a
|
||||
// 10-40% speed decrease on both compression and decompression.
|
||||
//
|
||||
// The dst and src must not overlap. It is valid to pass a nil dst.
|
||||
//
|
||||
// The blocks will require the same amount of memory to decode as encoding,
|
||||
// and does not make for concurrent decoding.
|
||||
// Also note that blocks do not contain CRC information, so corruption may be undetected.
|
||||
//
|
||||
// If you need to encode larger amounts of data, consider using
|
||||
// the streaming interface which gives all of these features.
|
||||
func EncodeBetter(dst, src []byte) []byte {
|
||||
if n := MaxEncodedLen(len(src)); n < 0 {
|
||||
panic(ErrTooLarge)
|
||||
} else if cap(dst) < n {
|
||||
dst = make([]byte, n)
|
||||
} else {
|
||||
dst = dst[:n]
|
||||
}
|
||||
|
||||
// The block starts with the varint-encoded length of the decompressed bytes.
|
||||
d := binary.PutUvarint(dst, uint64(len(src)))
|
||||
|
||||
if len(src) == 0 {
|
||||
return dst[:d]
|
||||
}
|
||||
if len(src) < minNonLiteralBlockSize {
|
||||
d += emitLiteral(dst[d:], src)
|
||||
return dst[:d]
|
||||
}
|
||||
n := encodeBlockBetter(dst[d:], src)
|
||||
if n > 0 {
|
||||
d += n
|
||||
return dst[:d]
|
||||
}
|
||||
// Not compressible
|
||||
d += emitLiteral(dst[d:], src)
|
||||
return dst[:d]
|
||||
}
|
||||
|
||||
// EncodeBest returns the encoded form of src. The returned slice may be a sub-
|
||||
// slice of dst if dst was large enough to hold the entire encoded block.
|
||||
// Otherwise, a newly allocated slice will be returned.
|
||||
//
|
||||
// EncodeBest compresses as good as reasonably possible but with a
|
||||
// big speed decrease.
|
||||
//
|
||||
// The dst and src must not overlap. It is valid to pass a nil dst.
|
||||
//
|
||||
// The blocks will require the same amount of memory to decode as encoding,
|
||||
// and does not make for concurrent decoding.
|
||||
// Also note that blocks do not contain CRC information, so corruption may be undetected.
|
||||
//
|
||||
// If you need to encode larger amounts of data, consider using
|
||||
// the streaming interface which gives all of these features.
|
||||
func EncodeBest(dst, src []byte) []byte {
|
||||
if n := MaxEncodedLen(len(src)); n < 0 {
|
||||
panic(ErrTooLarge)
|
||||
} else if cap(dst) < n {
|
||||
dst = make([]byte, n)
|
||||
} else {
|
||||
dst = dst[:n]
|
||||
}
|
||||
|
||||
// The block starts with the varint-encoded length of the decompressed bytes.
|
||||
d := binary.PutUvarint(dst, uint64(len(src)))
|
||||
|
||||
if len(src) == 0 {
|
||||
return dst[:d]
|
||||
}
|
||||
if len(src) < minNonLiteralBlockSize {
|
||||
d += emitLiteral(dst[d:], src)
|
||||
return dst[:d]
|
||||
}
|
||||
n := encodeBlockBest(dst[d:], src, nil)
|
||||
if n > 0 {
|
||||
d += n
|
||||
return dst[:d]
|
||||
}
|
||||
// Not compressible
|
||||
d += emitLiteral(dst[d:], src)
|
||||
return dst[:d]
|
||||
}
|
||||
|
||||
// EncodeSnappy returns the encoded form of src. The returned slice may be a sub-
|
||||
// slice of dst if dst was large enough to hold the entire encoded block.
|
||||
// Otherwise, a newly allocated slice will be returned.
|
||||
//
|
||||
// The output is Snappy compatible and will likely decompress faster.
|
||||
//
|
||||
// The dst and src must not overlap. It is valid to pass a nil dst.
|
||||
//
|
||||
// The blocks will require the same amount of memory to decode as encoding,
|
||||
// and does not make for concurrent decoding.
|
||||
// Also note that blocks do not contain CRC information, so corruption may be undetected.
|
||||
//
|
||||
// If you need to encode larger amounts of data, consider using
|
||||
// the streaming interface which gives all of these features.
|
||||
func EncodeSnappy(dst, src []byte) []byte {
|
||||
if n := MaxEncodedLen(len(src)); n < 0 {
|
||||
panic(ErrTooLarge)
|
||||
} else if cap(dst) < n {
|
||||
dst = make([]byte, n)
|
||||
} else {
|
||||
dst = dst[:n]
|
||||
}
|
||||
|
||||
// The block starts with the varint-encoded length of the decompressed bytes.
|
||||
d := binary.PutUvarint(dst, uint64(len(src)))
|
||||
|
||||
if len(src) == 0 {
|
||||
return dst[:d]
|
||||
}
|
||||
if len(src) < minNonLiteralBlockSize {
|
||||
d += emitLiteral(dst[d:], src)
|
||||
return dst[:d]
|
||||
}
|
||||
|
||||
n := encodeBlockSnappy(dst[d:], src)
|
||||
if n > 0 {
|
||||
d += n
|
||||
return dst[:d]
|
||||
}
|
||||
// Not compressible
|
||||
d += emitLiteral(dst[d:], src)
|
||||
return dst[:d]
|
||||
}
|
||||
|
||||
// EncodeSnappyBetter returns the encoded form of src. The returned slice may be a sub-
|
||||
// slice of dst if dst was large enough to hold the entire encoded block.
|
||||
// Otherwise, a newly allocated slice will be returned.
|
||||
//
|
||||
// The output is Snappy compatible and will likely decompress faster.
|
||||
//
|
||||
// The dst and src must not overlap. It is valid to pass a nil dst.
|
||||
//
|
||||
// The blocks will require the same amount of memory to decode as encoding,
|
||||
// and does not make for concurrent decoding.
|
||||
// Also note that blocks do not contain CRC information, so corruption may be undetected.
|
||||
//
|
||||
// If you need to encode larger amounts of data, consider using
|
||||
// the streaming interface which gives all of these features.
|
||||
func EncodeSnappyBetter(dst, src []byte) []byte {
|
||||
if n := MaxEncodedLen(len(src)); n < 0 {
|
||||
panic(ErrTooLarge)
|
||||
} else if cap(dst) < n {
|
||||
dst = make([]byte, n)
|
||||
} else {
|
||||
dst = dst[:n]
|
||||
}
|
||||
|
||||
// The block starts with the varint-encoded length of the decompressed bytes.
|
||||
d := binary.PutUvarint(dst, uint64(len(src)))
|
||||
|
||||
if len(src) == 0 {
|
||||
return dst[:d]
|
||||
}
|
||||
if len(src) < minNonLiteralBlockSize {
|
||||
d += emitLiteral(dst[d:], src)
|
||||
return dst[:d]
|
||||
}
|
||||
|
||||
n := encodeBlockBetterSnappy(dst[d:], src)
|
||||
if n > 0 {
|
||||
d += n
|
||||
return dst[:d]
|
||||
}
|
||||
// Not compressible
|
||||
d += emitLiteral(dst[d:], src)
|
||||
return dst[:d]
|
||||
}
|
||||
|
||||
// EncodeSnappyBest returns the encoded form of src. The returned slice may be a sub-
|
||||
// slice of dst if dst was large enough to hold the entire encoded block.
|
||||
// Otherwise, a newly allocated slice will be returned.
|
||||
//
|
||||
// The output is Snappy compatible and will likely decompress faster.
|
||||
//
|
||||
// The dst and src must not overlap. It is valid to pass a nil dst.
|
||||
//
|
||||
// The blocks will require the same amount of memory to decode as encoding,
|
||||
// and does not make for concurrent decoding.
|
||||
// Also note that blocks do not contain CRC information, so corruption may be undetected.
|
||||
//
|
||||
// If you need to encode larger amounts of data, consider using
|
||||
// the streaming interface which gives all of these features.
|
||||
func EncodeSnappyBest(dst, src []byte) []byte {
|
||||
if n := MaxEncodedLen(len(src)); n < 0 {
|
||||
panic(ErrTooLarge)
|
||||
} else if cap(dst) < n {
|
||||
dst = make([]byte, n)
|
||||
} else {
|
||||
dst = dst[:n]
|
||||
}
|
||||
|
||||
// The block starts with the varint-encoded length of the decompressed bytes.
|
||||
d := binary.PutUvarint(dst, uint64(len(src)))
|
||||
|
||||
if len(src) == 0 {
|
||||
return dst[:d]
|
||||
}
|
||||
if len(src) < minNonLiteralBlockSize {
|
||||
d += emitLiteral(dst[d:], src)
|
||||
return dst[:d]
|
||||
}
|
||||
|
||||
n := encodeBlockBestSnappy(dst[d:], src)
|
||||
if n > 0 {
|
||||
d += n
|
||||
return dst[:d]
|
||||
}
|
||||
// Not compressible
|
||||
d += emitLiteral(dst[d:], src)
|
||||
return dst[:d]
|
||||
}
|
||||
|
||||
// ConcatBlocks will concatenate the supplied blocks and append them to the supplied destination.
|
||||
// If the destination is nil or too small, a new will be allocated.
|
||||
// The blocks are not validated, so garbage in = garbage out.
|
||||
// dst may not overlap block data.
|
||||
// Any data in dst is preserved as is, so it will not be considered a block.
|
||||
func ConcatBlocks(dst []byte, blocks ...[]byte) ([]byte, error) {
|
||||
totalSize := uint64(0)
|
||||
compSize := 0
|
||||
for _, b := range blocks {
|
||||
l, hdr, err := decodedLen(b)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
totalSize += uint64(l)
|
||||
compSize += len(b) - hdr
|
||||
}
|
||||
if totalSize == 0 {
|
||||
dst = append(dst, 0)
|
||||
return dst, nil
|
||||
}
|
||||
if totalSize > math.MaxUint32 {
|
||||
return nil, ErrTooLarge
|
||||
}
|
||||
var tmp [binary.MaxVarintLen32]byte
|
||||
hdrSize := binary.PutUvarint(tmp[:], totalSize)
|
||||
wantSize := hdrSize + compSize
|
||||
|
||||
if cap(dst)-len(dst) < wantSize {
|
||||
dst = append(make([]byte, 0, wantSize+len(dst)), dst...)
|
||||
}
|
||||
dst = append(dst, tmp[:hdrSize]...)
|
||||
for _, b := range blocks {
|
||||
_, hdr, err := decodedLen(b)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
dst = append(dst, b[hdr:]...)
|
||||
}
|
||||
return dst, nil
|
||||
}
|
||||
|
||||
// inputMargin is the minimum number of extra input bytes to keep, inside
|
||||
// encodeBlock's inner loop. On some architectures, this margin lets us
|
||||
// implement a fast path for emitLiteral, where the copy of short (<= 16 byte)
|
||||
// literals can be implemented as a single load to and store from a 16-byte
|
||||
// register. That literal's actual length can be as short as 1 byte, so this
|
||||
// can copy up to 15 bytes too much, but that's OK as subsequent iterations of
|
||||
// the encoding loop will fix up the copy overrun, and this inputMargin ensures
|
||||
// that we don't overrun the dst and src buffers.
|
||||
const inputMargin = 8
|
||||
|
||||
// minNonLiteralBlockSize is the minimum size of the input to encodeBlock that
|
||||
// will be accepted by the encoder.
|
||||
const minNonLiteralBlockSize = 32
|
||||
|
||||
const intReduction = 2 - (1 << (^uint(0) >> 63)) // 1 (32 bits) or 0 (64 bits)
|
||||
|
||||
// MaxBlockSize is the maximum value where MaxEncodedLen will return a valid block size.
|
||||
// Blocks this big are highly discouraged, though.
|
||||
// Half the size on 32 bit systems.
|
||||
const MaxBlockSize = (1<<(32-intReduction) - 1) - binary.MaxVarintLen32 - 5
|
||||
|
||||
// MaxEncodedLen returns the maximum length of a snappy block, given its
|
||||
// uncompressed length.
|
||||
//
|
||||
// It will return a negative value if srcLen is too large to encode.
|
||||
// 32 bit platforms will have lower thresholds for rejecting big content.
|
||||
func MaxEncodedLen(srcLen int) int {
|
||||
n := uint64(srcLen)
|
||||
if intReduction == 1 {
|
||||
// 32 bits
|
||||
if n > math.MaxInt32 {
|
||||
// Also includes negative.
|
||||
return -1
|
||||
}
|
||||
} else if n > 0xffffffff {
|
||||
// 64 bits
|
||||
// Also includes negative.
|
||||
return -1
|
||||
}
|
||||
// Size of the varint encoded block size.
|
||||
n = n + uint64((bits.Len64(n)+7)/7)
|
||||
|
||||
// Add maximum size of encoding block as literals.
|
||||
n += uint64(literalExtraSize(int64(srcLen)))
|
||||
if intReduction == 1 {
|
||||
// 32 bits
|
||||
if n > math.MaxInt32 {
|
||||
return -1
|
||||
}
|
||||
} else if n > 0xffffffff {
|
||||
// 64 bits
|
||||
// Also includes negative.
|
||||
return -1
|
||||
}
|
||||
return int(n)
|
||||
}
|
||||
+1477
File diff suppressed because it is too large
Load Diff
+316
@@ -0,0 +1,316 @@
|
||||
//go:build (amd64 || arm64) && !appengine && !noasm && gc
|
||||
|
||||
package s2
|
||||
|
||||
import (
|
||||
"sync"
|
||||
|
||||
"github.com/klauspost/compress/internal/race"
|
||||
)
|
||||
|
||||
const hasAsm = true
|
||||
|
||||
var encPools [4]sync.Pool
|
||||
|
||||
// encodeBlock encodes a non-empty src to a guaranteed-large-enough dst. It
|
||||
// assumes that the varint-encoded length of the decompressed bytes has already
|
||||
// been written.
|
||||
//
|
||||
// It also assumes that:
|
||||
//
|
||||
// len(dst) >= MaxEncodedLen(len(src)) &&
|
||||
// minNonLiteralBlockSize <= len(src) && len(src) <= maxBlockSize
|
||||
func encodeBlock(dst, src []byte) (d int) {
|
||||
race.ReadSlice(src)
|
||||
race.WriteSlice(dst)
|
||||
|
||||
const (
|
||||
// Use 12 bit table when less than...
|
||||
limit12B = 16 << 10
|
||||
// Use 10 bit table when less than...
|
||||
limit10B = 4 << 10
|
||||
// Use 8 bit table when less than...
|
||||
limit8B = 512
|
||||
)
|
||||
|
||||
if len(src) >= 4<<20 {
|
||||
const sz, pool = 65536, 0
|
||||
tmp, ok := encPools[pool].Get().(*[sz]byte)
|
||||
if !ok {
|
||||
tmp = &[sz]byte{}
|
||||
}
|
||||
race.WriteSlice(tmp[:])
|
||||
defer encPools[pool].Put(tmp)
|
||||
return encodeBlockAsm(dst, src, tmp)
|
||||
}
|
||||
if len(src) >= limit12B {
|
||||
const sz, pool = 65536, 0
|
||||
tmp, ok := encPools[pool].Get().(*[sz]byte)
|
||||
if !ok {
|
||||
tmp = &[sz]byte{}
|
||||
}
|
||||
race.WriteSlice(tmp[:])
|
||||
defer encPools[pool].Put(tmp)
|
||||
return encodeBlockAsm4MB(dst, src, tmp)
|
||||
}
|
||||
if len(src) >= limit10B {
|
||||
const sz, pool = 16384, 1
|
||||
tmp, ok := encPools[pool].Get().(*[sz]byte)
|
||||
if !ok {
|
||||
tmp = &[sz]byte{}
|
||||
}
|
||||
race.WriteSlice(tmp[:])
|
||||
defer encPools[pool].Put(tmp)
|
||||
return encodeBlockAsm12B(dst, src, tmp)
|
||||
}
|
||||
if len(src) >= limit8B {
|
||||
const sz, pool = 4096, 2
|
||||
tmp, ok := encPools[pool].Get().(*[sz]byte)
|
||||
if !ok {
|
||||
tmp = &[sz]byte{}
|
||||
}
|
||||
race.WriteSlice(tmp[:])
|
||||
defer encPools[pool].Put(tmp)
|
||||
return encodeBlockAsm10B(dst, src, tmp)
|
||||
}
|
||||
if len(src) < minNonLiteralBlockSize {
|
||||
return 0
|
||||
}
|
||||
const sz, pool = 1024, 3
|
||||
tmp, ok := encPools[pool].Get().(*[sz]byte)
|
||||
if !ok {
|
||||
tmp = &[sz]byte{}
|
||||
}
|
||||
race.WriteSlice(tmp[:])
|
||||
defer encPools[pool].Put(tmp)
|
||||
return encodeBlockAsm8B(dst, src, tmp)
|
||||
}
|
||||
|
||||
var encBetterPools [5]sync.Pool
|
||||
|
||||
// encodeBlockBetter encodes a non-empty src to a guaranteed-large-enough dst. It
|
||||
// assumes that the varint-encoded length of the decompressed bytes has already
|
||||
// been written.
|
||||
//
|
||||
// It also assumes that:
|
||||
//
|
||||
// len(dst) >= MaxEncodedLen(len(src)) &&
|
||||
// minNonLiteralBlockSize <= len(src) && len(src) <= maxBlockSize
|
||||
func encodeBlockBetter(dst, src []byte) (d int) {
|
||||
race.ReadSlice(src)
|
||||
race.WriteSlice(dst)
|
||||
|
||||
const (
|
||||
// Use 12 bit table when less than...
|
||||
limit12B = 16 << 10
|
||||
// Use 10 bit table when less than...
|
||||
limit10B = 4 << 10
|
||||
// Use 8 bit table when less than...
|
||||
limit8B = 512
|
||||
)
|
||||
|
||||
if len(src) > 4<<20 {
|
||||
const sz, pool = 589824, 0
|
||||
tmp, ok := encBetterPools[pool].Get().(*[sz]byte)
|
||||
if !ok {
|
||||
tmp = &[sz]byte{}
|
||||
}
|
||||
race.WriteSlice(tmp[:])
|
||||
defer encBetterPools[pool].Put(tmp)
|
||||
return encodeBetterBlockAsm(dst, src, tmp)
|
||||
}
|
||||
if len(src) >= limit12B {
|
||||
const sz, pool = 589824, 0
|
||||
tmp, ok := encBetterPools[pool].Get().(*[sz]byte)
|
||||
if !ok {
|
||||
tmp = &[sz]byte{}
|
||||
}
|
||||
race.WriteSlice(tmp[:])
|
||||
defer encBetterPools[pool].Put(tmp)
|
||||
|
||||
return encodeBetterBlockAsm4MB(dst, src, tmp)
|
||||
}
|
||||
if len(src) >= limit10B {
|
||||
const sz, pool = 81920, 0
|
||||
tmp, ok := encBetterPools[pool].Get().(*[sz]byte)
|
||||
if !ok {
|
||||
tmp = &[sz]byte{}
|
||||
}
|
||||
race.WriteSlice(tmp[:])
|
||||
defer encBetterPools[pool].Put(tmp)
|
||||
|
||||
return encodeBetterBlockAsm12B(dst, src, tmp)
|
||||
}
|
||||
if len(src) >= limit8B {
|
||||
const sz, pool = 20480, 1
|
||||
tmp, ok := encBetterPools[pool].Get().(*[sz]byte)
|
||||
if !ok {
|
||||
tmp = &[sz]byte{}
|
||||
}
|
||||
race.WriteSlice(tmp[:])
|
||||
defer encBetterPools[pool].Put(tmp)
|
||||
return encodeBetterBlockAsm10B(dst, src, tmp)
|
||||
}
|
||||
if len(src) < minNonLiteralBlockSize {
|
||||
return 0
|
||||
}
|
||||
|
||||
const sz, pool = 5120, 2
|
||||
tmp, ok := encBetterPools[pool].Get().(*[sz]byte)
|
||||
if !ok {
|
||||
tmp = &[sz]byte{}
|
||||
}
|
||||
race.WriteSlice(tmp[:])
|
||||
defer encBetterPools[pool].Put(tmp)
|
||||
return encodeBetterBlockAsm8B(dst, src, tmp)
|
||||
}
|
||||
|
||||
// encodeBlockSnappy encodes a non-empty src to a guaranteed-large-enough dst. It
|
||||
// assumes that the varint-encoded length of the decompressed bytes has already
|
||||
// been written.
|
||||
//
|
||||
// It also assumes that:
|
||||
//
|
||||
// len(dst) >= MaxEncodedLen(len(src)) &&
|
||||
// minNonLiteralBlockSize <= len(src) && len(src) <= maxBlockSize
|
||||
func encodeBlockSnappy(dst, src []byte) (d int) {
|
||||
race.ReadSlice(src)
|
||||
race.WriteSlice(dst)
|
||||
|
||||
const (
|
||||
// Use 12 bit table when less than...
|
||||
limit12B = 16 << 10
|
||||
// Use 10 bit table when less than...
|
||||
limit10B = 4 << 10
|
||||
// Use 8 bit table when less than...
|
||||
limit8B = 512
|
||||
)
|
||||
if len(src) > 65536 {
|
||||
const sz, pool = 65536, 0
|
||||
tmp, ok := encPools[pool].Get().(*[sz]byte)
|
||||
if !ok {
|
||||
tmp = &[sz]byte{}
|
||||
}
|
||||
race.WriteSlice(tmp[:])
|
||||
defer encPools[pool].Put(tmp)
|
||||
return encodeSnappyBlockAsm(dst, src, tmp)
|
||||
}
|
||||
if len(src) >= limit12B {
|
||||
const sz, pool = 65536, 0
|
||||
tmp, ok := encPools[pool].Get().(*[sz]byte)
|
||||
if !ok {
|
||||
tmp = &[sz]byte{}
|
||||
}
|
||||
race.WriteSlice(tmp[:])
|
||||
defer encPools[pool].Put(tmp)
|
||||
return encodeSnappyBlockAsm64K(dst, src, tmp)
|
||||
}
|
||||
if len(src) >= limit10B {
|
||||
const sz, pool = 16384, 1
|
||||
tmp, ok := encPools[pool].Get().(*[sz]byte)
|
||||
if !ok {
|
||||
tmp = &[sz]byte{}
|
||||
}
|
||||
race.WriteSlice(tmp[:])
|
||||
defer encPools[pool].Put(tmp)
|
||||
return encodeSnappyBlockAsm12B(dst, src, tmp)
|
||||
}
|
||||
if len(src) >= limit8B {
|
||||
const sz, pool = 4096, 2
|
||||
tmp, ok := encPools[pool].Get().(*[sz]byte)
|
||||
if !ok {
|
||||
tmp = &[sz]byte{}
|
||||
}
|
||||
race.WriteSlice(tmp[:])
|
||||
defer encPools[pool].Put(tmp)
|
||||
return encodeSnappyBlockAsm10B(dst, src, tmp)
|
||||
}
|
||||
if len(src) < minNonLiteralBlockSize {
|
||||
return 0
|
||||
}
|
||||
const sz, pool = 1024, 3
|
||||
tmp, ok := encPools[pool].Get().(*[sz]byte)
|
||||
if !ok {
|
||||
tmp = &[sz]byte{}
|
||||
}
|
||||
race.WriteSlice(tmp[:])
|
||||
defer encPools[pool].Put(tmp)
|
||||
return encodeSnappyBlockAsm8B(dst, src, tmp)
|
||||
}
|
||||
|
||||
// encodeBlockSnappy encodes a non-empty src to a guaranteed-large-enough dst. It
|
||||
// assumes that the varint-encoded length of the decompressed bytes has already
|
||||
// been written.
|
||||
//
|
||||
// It also assumes that:
|
||||
//
|
||||
// len(dst) >= MaxEncodedLen(len(src)) &&
|
||||
// minNonLiteralBlockSize <= len(src) && len(src) <= maxBlockSize
|
||||
func encodeBlockBetterSnappy(dst, src []byte) (d int) {
|
||||
race.ReadSlice(src)
|
||||
race.WriteSlice(dst)
|
||||
|
||||
const (
|
||||
// Use 12 bit table when less than...
|
||||
limit12B = 16 << 10
|
||||
// Use 10 bit table when less than...
|
||||
limit10B = 4 << 10
|
||||
// Use 8 bit table when less than...
|
||||
limit8B = 512
|
||||
)
|
||||
if len(src) > 65536 {
|
||||
const sz, pool = 589824, 0
|
||||
tmp, ok := encBetterPools[pool].Get().(*[sz]byte)
|
||||
if !ok {
|
||||
tmp = &[sz]byte{}
|
||||
}
|
||||
race.WriteSlice(tmp[:])
|
||||
defer encBetterPools[pool].Put(tmp)
|
||||
return encodeSnappyBetterBlockAsm(dst, src, tmp)
|
||||
}
|
||||
|
||||
if len(src) >= limit12B {
|
||||
const sz, pool = 294912, 4
|
||||
tmp, ok := encBetterPools[pool].Get().(*[sz]byte)
|
||||
if !ok {
|
||||
tmp = &[sz]byte{}
|
||||
}
|
||||
race.WriteSlice(tmp[:])
|
||||
defer encBetterPools[pool].Put(tmp)
|
||||
|
||||
return encodeSnappyBetterBlockAsm64K(dst, src, tmp)
|
||||
}
|
||||
if len(src) >= limit10B {
|
||||
const sz, pool = 81920, 0
|
||||
tmp, ok := encBetterPools[pool].Get().(*[sz]byte)
|
||||
if !ok {
|
||||
tmp = &[sz]byte{}
|
||||
}
|
||||
race.WriteSlice(tmp[:])
|
||||
defer encBetterPools[pool].Put(tmp)
|
||||
|
||||
return encodeSnappyBetterBlockAsm12B(dst, src, tmp)
|
||||
}
|
||||
if len(src) >= limit8B {
|
||||
const sz, pool = 20480, 1
|
||||
tmp, ok := encBetterPools[pool].Get().(*[sz]byte)
|
||||
if !ok {
|
||||
tmp = &[sz]byte{}
|
||||
}
|
||||
race.WriteSlice(tmp[:])
|
||||
defer encBetterPools[pool].Put(tmp)
|
||||
return encodeSnappyBetterBlockAsm10B(dst, src, tmp)
|
||||
}
|
||||
if len(src) < minNonLiteralBlockSize {
|
||||
return 0
|
||||
}
|
||||
|
||||
const sz, pool = 5120, 2
|
||||
tmp, ok := encBetterPools[pool].Get().(*[sz]byte)
|
||||
if !ok {
|
||||
tmp = &[sz]byte{}
|
||||
}
|
||||
race.WriteSlice(tmp[:])
|
||||
defer encBetterPools[pool].Put(tmp)
|
||||
return encodeSnappyBetterBlockAsm8B(dst, src, tmp)
|
||||
}
|
||||
+797
@@ -0,0 +1,797 @@
|
||||
// Copyright 2016 The Snappy-Go Authors. All rights reserved.
|
||||
// Copyright (c) 2019 Klaus Post. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package s2
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"math"
|
||||
"math/bits"
|
||||
)
|
||||
|
||||
// encodeBlockBest encodes a non-empty src to a guaranteed-large-enough dst. It
|
||||
// assumes that the varint-encoded length of the decompressed bytes has already
|
||||
// been written.
|
||||
//
|
||||
// It also assumes that:
|
||||
//
|
||||
// len(dst) >= MaxEncodedLen(len(src)) &&
|
||||
// minNonLiteralBlockSize <= len(src) && len(src) <= maxBlockSize
|
||||
func encodeBlockBest(dst, src []byte, dict *Dict) (d int) {
|
||||
// Initialize the hash tables.
|
||||
const (
|
||||
// Long hash matches.
|
||||
lTableBits = bestLongTableBits
|
||||
maxLTableSize = bestLongTableSize
|
||||
|
||||
// Short hash matches.
|
||||
sTableBits = bestShortTableBits
|
||||
maxSTableSize = bestShortTableSize
|
||||
|
||||
inputMargin = 8 + 2
|
||||
|
||||
debug = false
|
||||
)
|
||||
|
||||
// sLimit is when to stop looking for offset/length copies. The inputMargin
|
||||
// lets us use a fast path for emitLiteral in the main loop, while we are
|
||||
// looking for copies.
|
||||
sLimit := len(src) - inputMargin
|
||||
if len(src) < minNonLiteralBlockSize {
|
||||
return 0
|
||||
}
|
||||
sLimitDict := min(len(src)-inputMargin, MaxDictSrcOffset-inputMargin)
|
||||
|
||||
tbl := getBestTables()
|
||||
lTable := &tbl.lTable
|
||||
sTable := &tbl.sTable
|
||||
defer bestTablePool.Put(tbl)
|
||||
|
||||
// Bail if we can't compress to at least this.
|
||||
dstLimit := len(src) - 5
|
||||
|
||||
// nextEmit is where in src the next emitLiteral should start from.
|
||||
nextEmit := 0
|
||||
|
||||
// The encoded form must start with a literal, as there are no previous
|
||||
// bytes to copy, so we start looking for hash matches at s == 1.
|
||||
s := 1
|
||||
repeat := 1
|
||||
if dict != nil {
|
||||
dict.initBest()
|
||||
s = 0
|
||||
repeat = len(dict.dict) - dict.repeat
|
||||
}
|
||||
cv := load64(src, s)
|
||||
|
||||
// We search for a repeat at -1, but don't output repeats when nextEmit == 0
|
||||
const lowbitMask = 0xffffffff
|
||||
getCur := func(x uint64) int {
|
||||
return int(x & lowbitMask)
|
||||
}
|
||||
getPrev := func(x uint64) int {
|
||||
return int(x >> 32)
|
||||
}
|
||||
const maxSkip = 64
|
||||
|
||||
for {
|
||||
type match struct {
|
||||
offset int
|
||||
s int
|
||||
length int
|
||||
score int
|
||||
rep, dict bool
|
||||
}
|
||||
var best match
|
||||
for {
|
||||
// Next src position to check
|
||||
nextS := (s-nextEmit)>>8 + 1
|
||||
if nextS > maxSkip {
|
||||
nextS = s + maxSkip
|
||||
} else {
|
||||
nextS += s
|
||||
}
|
||||
if nextS > sLimit {
|
||||
goto emitRemainder
|
||||
}
|
||||
if dict != nil && s >= MaxDictSrcOffset {
|
||||
dict = nil
|
||||
if repeat > s {
|
||||
repeat = math.MinInt32
|
||||
}
|
||||
}
|
||||
hashL := hash8(cv, lTableBits)
|
||||
hashS := hash4(cv, sTableBits)
|
||||
candidateL := lTable[hashL]
|
||||
candidateS := sTable[hashS]
|
||||
|
||||
score := func(m match) int {
|
||||
// Matches that are longer forward are penalized since we must emit it as a literal.
|
||||
score := m.length - m.s
|
||||
if nextEmit == m.s {
|
||||
// If we do not have to emit literals, we save 1 byte
|
||||
score++
|
||||
}
|
||||
offset := m.s - m.offset
|
||||
if m.rep {
|
||||
return score - emitRepeatSize(offset, m.length)
|
||||
}
|
||||
return score - emitCopySize(offset, m.length)
|
||||
}
|
||||
|
||||
matchAt := func(offset, s int, first uint32, rep bool) match {
|
||||
if best.length != 0 && best.s-best.offset == s-offset {
|
||||
// Don't retest if we have the same offset.
|
||||
return match{offset: offset, s: s}
|
||||
}
|
||||
if load32(src, offset) != first {
|
||||
return match{offset: offset, s: s}
|
||||
}
|
||||
m := match{offset: offset, s: s, length: 4 + offset, rep: rep}
|
||||
s += 4
|
||||
for s < len(src) {
|
||||
if len(src)-s < 8 {
|
||||
if src[s] == src[m.length] {
|
||||
m.length++
|
||||
s++
|
||||
continue
|
||||
}
|
||||
break
|
||||
}
|
||||
if diff := load64(src, s) ^ load64(src, m.length); diff != 0 {
|
||||
m.length += bits.TrailingZeros64(diff) >> 3
|
||||
break
|
||||
}
|
||||
s += 8
|
||||
m.length += 8
|
||||
}
|
||||
m.length -= offset
|
||||
m.score = score(m)
|
||||
if m.score <= -m.s {
|
||||
// Eliminate if no savings, we might find a better one.
|
||||
m.length = 0
|
||||
}
|
||||
return m
|
||||
}
|
||||
matchDict := func(candidate, s int, first uint32, rep bool) match {
|
||||
if s >= MaxDictSrcOffset {
|
||||
return match{offset: candidate, s: s}
|
||||
}
|
||||
// Calculate offset as if in continuous array with s
|
||||
offset := -len(dict.dict) + candidate
|
||||
if best.length != 0 && best.s-best.offset == s-offset && !rep {
|
||||
// Don't retest if we have the same offset.
|
||||
return match{offset: offset, s: s}
|
||||
}
|
||||
|
||||
if load32(dict.dict, candidate) != first {
|
||||
return match{offset: offset, s: s}
|
||||
}
|
||||
m := match{offset: offset, s: s, length: 4 + candidate, rep: rep, dict: true}
|
||||
s += 4
|
||||
if !rep {
|
||||
for s < sLimitDict && m.length < len(dict.dict) {
|
||||
if len(src)-s < 8 || len(dict.dict)-m.length < 8 {
|
||||
if src[s] == dict.dict[m.length] {
|
||||
m.length++
|
||||
s++
|
||||
continue
|
||||
}
|
||||
break
|
||||
}
|
||||
if diff := load64(src, s) ^ load64(dict.dict, m.length); diff != 0 {
|
||||
m.length += bits.TrailingZeros64(diff) >> 3
|
||||
break
|
||||
}
|
||||
s += 8
|
||||
m.length += 8
|
||||
}
|
||||
} else {
|
||||
for s < len(src) && m.length < len(dict.dict) {
|
||||
if len(src)-s < 8 || len(dict.dict)-m.length < 8 {
|
||||
if src[s] == dict.dict[m.length] {
|
||||
m.length++
|
||||
s++
|
||||
continue
|
||||
}
|
||||
break
|
||||
}
|
||||
if diff := load64(src, s) ^ load64(dict.dict, m.length); diff != 0 {
|
||||
m.length += bits.TrailingZeros64(diff) >> 3
|
||||
break
|
||||
}
|
||||
s += 8
|
||||
m.length += 8
|
||||
}
|
||||
}
|
||||
m.length -= candidate
|
||||
m.score = score(m)
|
||||
if m.score <= -m.s {
|
||||
// Eliminate if no savings, we might find a better one.
|
||||
m.length = 0
|
||||
}
|
||||
return m
|
||||
}
|
||||
|
||||
bestOf := func(a, b match) match {
|
||||
if b.length == 0 {
|
||||
return a
|
||||
}
|
||||
if a.length == 0 {
|
||||
return b
|
||||
}
|
||||
as := a.score + b.s
|
||||
bs := b.score + a.s
|
||||
if as >= bs {
|
||||
return a
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
if s > 0 {
|
||||
best = bestOf(matchAt(getCur(candidateL), s, uint32(cv), false), matchAt(getPrev(candidateL), s, uint32(cv), false))
|
||||
best = bestOf(best, matchAt(getCur(candidateS), s, uint32(cv), false))
|
||||
best = bestOf(best, matchAt(getPrev(candidateS), s, uint32(cv), false))
|
||||
}
|
||||
if dict != nil {
|
||||
candidateL := dict.bestTableLong[hashL]
|
||||
candidateS := dict.bestTableShort[hashS]
|
||||
best = bestOf(best, matchDict(int(candidateL&0xffff), s, uint32(cv), false))
|
||||
best = bestOf(best, matchDict(int(candidateL>>16), s, uint32(cv), false))
|
||||
best = bestOf(best, matchDict(int(candidateS&0xffff), s, uint32(cv), false))
|
||||
best = bestOf(best, matchDict(int(candidateS>>16), s, uint32(cv), false))
|
||||
}
|
||||
{
|
||||
if (dict == nil || repeat <= s) && repeat > 0 {
|
||||
best = bestOf(best, matchAt(s-repeat+1, s+1, uint32(cv>>8), true))
|
||||
} else if s-repeat < -4 && dict != nil {
|
||||
candidate := len(dict.dict) - (repeat - s)
|
||||
best = bestOf(best, matchDict(candidate, s, uint32(cv), true))
|
||||
candidate++
|
||||
best = bestOf(best, matchDict(candidate, s+1, uint32(cv>>8), true))
|
||||
}
|
||||
|
||||
if best.length > 0 {
|
||||
hashS := hash4(cv>>8, sTableBits)
|
||||
// s+1
|
||||
nextShort := sTable[hashS]
|
||||
s := s + 1
|
||||
cv := load64(src, s)
|
||||
hashL := hash8(cv, lTableBits)
|
||||
nextLong := lTable[hashL]
|
||||
best = bestOf(best, matchAt(getCur(nextShort), s, uint32(cv), false))
|
||||
best = bestOf(best, matchAt(getPrev(nextShort), s, uint32(cv), false))
|
||||
best = bestOf(best, matchAt(getCur(nextLong), s, uint32(cv), false))
|
||||
best = bestOf(best, matchAt(getPrev(nextLong), s, uint32(cv), false))
|
||||
|
||||
// Dict at + 1
|
||||
if dict != nil {
|
||||
candidateL := dict.bestTableLong[hashL]
|
||||
candidateS := dict.bestTableShort[hashS]
|
||||
|
||||
best = bestOf(best, matchDict(int(candidateL&0xffff), s, uint32(cv), false))
|
||||
best = bestOf(best, matchDict(int(candidateS&0xffff), s, uint32(cv), false))
|
||||
}
|
||||
|
||||
// s+2
|
||||
if true {
|
||||
hashS := hash4(cv>>8, sTableBits)
|
||||
|
||||
nextShort = sTable[hashS]
|
||||
s++
|
||||
cv = load64(src, s)
|
||||
hashL := hash8(cv, lTableBits)
|
||||
nextLong = lTable[hashL]
|
||||
|
||||
if (dict == nil || repeat <= s) && repeat > 0 {
|
||||
// Repeat at + 2
|
||||
best = bestOf(best, matchAt(s-repeat, s, uint32(cv), true))
|
||||
} else if repeat-s > 4 && dict != nil {
|
||||
candidate := len(dict.dict) - (repeat - s)
|
||||
best = bestOf(best, matchDict(candidate, s, uint32(cv), true))
|
||||
}
|
||||
best = bestOf(best, matchAt(getCur(nextShort), s, uint32(cv), false))
|
||||
best = bestOf(best, matchAt(getPrev(nextShort), s, uint32(cv), false))
|
||||
best = bestOf(best, matchAt(getCur(nextLong), s, uint32(cv), false))
|
||||
best = bestOf(best, matchAt(getPrev(nextLong), s, uint32(cv), false))
|
||||
|
||||
// Dict at +2
|
||||
// Very small gain
|
||||
if dict != nil {
|
||||
candidateL := dict.bestTableLong[hashL]
|
||||
candidateS := dict.bestTableShort[hashS]
|
||||
|
||||
best = bestOf(best, matchDict(int(candidateL&0xffff), s, uint32(cv), false))
|
||||
best = bestOf(best, matchDict(int(candidateS&0xffff), s, uint32(cv), false))
|
||||
}
|
||||
}
|
||||
// Search for a match at best match end, see if that is better.
|
||||
// Allow some bytes at the beginning to mismatch.
|
||||
// Sweet spot is around 1-2 bytes, but depends on input.
|
||||
// The skipped bytes are tested in Extend backwards,
|
||||
// and still picked up as part of the match if they do.
|
||||
const skipBeginning = 2
|
||||
const skipEnd = 1
|
||||
if sAt := best.s + best.length - skipEnd; sAt < sLimit {
|
||||
|
||||
sBack := best.s + skipBeginning - skipEnd
|
||||
backL := best.length - skipBeginning
|
||||
// Load initial values
|
||||
cv = load64(src, sBack)
|
||||
|
||||
// Grab candidates...
|
||||
next := lTable[hash8(load64(src, sAt), lTableBits)]
|
||||
|
||||
if checkAt := getCur(next) - backL; checkAt > 0 {
|
||||
best = bestOf(best, matchAt(checkAt, sBack, uint32(cv), false))
|
||||
}
|
||||
if checkAt := getPrev(next) - backL; checkAt > 0 {
|
||||
best = bestOf(best, matchAt(checkAt, sBack, uint32(cv), false))
|
||||
}
|
||||
// Disabled: Extremely small gain
|
||||
if false {
|
||||
next = sTable[hash4(load64(src, sAt), sTableBits)]
|
||||
if checkAt := getCur(next) - backL; checkAt > 0 {
|
||||
best = bestOf(best, matchAt(checkAt, sBack, uint32(cv), false))
|
||||
}
|
||||
if checkAt := getPrev(next) - backL; checkAt > 0 {
|
||||
best = bestOf(best, matchAt(checkAt, sBack, uint32(cv), false))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Update table
|
||||
lTable[hashL] = uint64(s) | candidateL<<32
|
||||
sTable[hashS] = uint64(s) | candidateS<<32
|
||||
|
||||
if best.length > 0 {
|
||||
break
|
||||
}
|
||||
|
||||
cv = load64(src, nextS)
|
||||
s = nextS
|
||||
}
|
||||
|
||||
// Extend backwards, not needed for repeats...
|
||||
s = best.s
|
||||
if !best.rep && !best.dict {
|
||||
for best.offset > 0 && s > nextEmit && src[best.offset-1] == src[s-1] {
|
||||
best.offset--
|
||||
best.length++
|
||||
s--
|
||||
}
|
||||
}
|
||||
if false && best.offset >= s {
|
||||
panic(fmt.Errorf("t %d >= s %d", best.offset, s))
|
||||
}
|
||||
// Bail if we exceed the maximum size.
|
||||
if d+(s-nextEmit) > dstLimit {
|
||||
return 0
|
||||
}
|
||||
|
||||
base := s
|
||||
offset := s - best.offset
|
||||
s += best.length
|
||||
|
||||
if offset > 65535 && s-base <= 5 && !best.rep {
|
||||
// Bail if the match is equal or worse to the encoding.
|
||||
s = best.s + 1
|
||||
if s >= sLimit {
|
||||
goto emitRemainder
|
||||
}
|
||||
cv = load64(src, s)
|
||||
continue
|
||||
}
|
||||
if debug && nextEmit != base {
|
||||
fmt.Println("EMIT", base-nextEmit, "literals. base-after:", base)
|
||||
}
|
||||
d += emitLiteral(dst[d:], src[nextEmit:base])
|
||||
if best.rep {
|
||||
if nextEmit > 0 || best.dict {
|
||||
if debug {
|
||||
fmt.Println("REPEAT, length", best.length, "offset:", offset, "s-after:", s, "dict:", best.dict, "best:", best)
|
||||
}
|
||||
// same as `add := emitCopy(dst[d:], repeat, s-base)` but skips storing offset.
|
||||
d += emitRepeat(dst[d:], offset, best.length)
|
||||
} else {
|
||||
// First match without dict cannot be a repeat.
|
||||
if debug {
|
||||
fmt.Println("COPY, length", best.length, "offset:", offset, "s-after:", s, "dict:", best.dict, "best:", best)
|
||||
}
|
||||
d += emitCopy(dst[d:], offset, best.length)
|
||||
}
|
||||
} else {
|
||||
if debug {
|
||||
fmt.Println("COPY, length", best.length, "offset:", offset, "s-after:", s, "dict:", best.dict, "best:", best)
|
||||
}
|
||||
d += emitCopy(dst[d:], offset, best.length)
|
||||
}
|
||||
repeat = offset
|
||||
|
||||
nextEmit = s
|
||||
if s >= sLimit {
|
||||
goto emitRemainder
|
||||
}
|
||||
|
||||
if d > dstLimit {
|
||||
// Do we have space for more, if not bail.
|
||||
return 0
|
||||
}
|
||||
// Fill tables...
|
||||
for i := best.s + 1; i < s; i++ {
|
||||
cv0 := load64(src, i)
|
||||
long0 := hash8(cv0, lTableBits)
|
||||
short0 := hash4(cv0, sTableBits)
|
||||
lTable[long0] = uint64(i) | lTable[long0]<<32
|
||||
sTable[short0] = uint64(i) | sTable[short0]<<32
|
||||
}
|
||||
cv = load64(src, s)
|
||||
}
|
||||
|
||||
emitRemainder:
|
||||
if nextEmit < len(src) {
|
||||
// Bail if we exceed the maximum size.
|
||||
if d+len(src)-nextEmit > dstLimit {
|
||||
return 0
|
||||
}
|
||||
if debug && nextEmit != s {
|
||||
fmt.Println("emitted ", len(src)-nextEmit, "literals")
|
||||
}
|
||||
d += emitLiteral(dst[d:], src[nextEmit:])
|
||||
}
|
||||
return d
|
||||
}
|
||||
|
||||
// encodeBlockBestSnappy encodes a non-empty src to a guaranteed-large-enough dst. It
|
||||
// assumes that the varint-encoded length of the decompressed bytes has already
|
||||
// been written.
|
||||
//
|
||||
// It also assumes that:
|
||||
//
|
||||
// len(dst) >= MaxEncodedLen(len(src)) &&
|
||||
// minNonLiteralBlockSize <= len(src) && len(src) <= maxBlockSize
|
||||
func encodeBlockBestSnappy(dst, src []byte) (d int) {
|
||||
// Initialize the hash tables.
|
||||
const (
|
||||
// Long hash matches.
|
||||
lTableBits = bestLongTableBits
|
||||
maxLTableSize = bestLongTableSize
|
||||
|
||||
// Short hash matches.
|
||||
sTableBits = bestShortTableBits
|
||||
maxSTableSize = bestShortTableSize
|
||||
|
||||
inputMargin = 8 + 2
|
||||
)
|
||||
|
||||
// sLimit is when to stop looking for offset/length copies. The inputMargin
|
||||
// lets us use a fast path for emitLiteral in the main loop, while we are
|
||||
// looking for copies.
|
||||
sLimit := len(src) - inputMargin
|
||||
if len(src) < minNonLiteralBlockSize {
|
||||
return 0
|
||||
}
|
||||
|
||||
tbl := getBestTables()
|
||||
lTable := &tbl.lTable
|
||||
sTable := &tbl.sTable
|
||||
defer bestTablePool.Put(tbl)
|
||||
|
||||
// Bail if we can't compress to at least this.
|
||||
dstLimit := len(src) - 5
|
||||
|
||||
// nextEmit is where in src the next emitLiteral should start from.
|
||||
nextEmit := 0
|
||||
|
||||
// The encoded form must start with a literal, as there are no previous
|
||||
// bytes to copy, so we start looking for hash matches at s == 1.
|
||||
s := 1
|
||||
cv := load64(src, s)
|
||||
|
||||
// We search for a repeat at -1, but don't output repeats when nextEmit == 0
|
||||
repeat := 1
|
||||
const lowbitMask = 0xffffffff
|
||||
getCur := func(x uint64) int {
|
||||
return int(x & lowbitMask)
|
||||
}
|
||||
getPrev := func(x uint64) int {
|
||||
return int(x >> 32)
|
||||
}
|
||||
const maxSkip = 64
|
||||
|
||||
for {
|
||||
type match struct {
|
||||
offset int
|
||||
s int
|
||||
length int
|
||||
score int
|
||||
}
|
||||
var best match
|
||||
for {
|
||||
// Next src position to check
|
||||
nextS := (s-nextEmit)>>8 + 1
|
||||
if nextS > maxSkip {
|
||||
nextS = s + maxSkip
|
||||
} else {
|
||||
nextS += s
|
||||
}
|
||||
if nextS > sLimit {
|
||||
goto emitRemainder
|
||||
}
|
||||
hashL := hash8(cv, lTableBits)
|
||||
hashS := hash4(cv, sTableBits)
|
||||
candidateL := lTable[hashL]
|
||||
candidateS := sTable[hashS]
|
||||
|
||||
score := func(m match) int {
|
||||
// Matches that are longer forward are penalized since we must emit it as a literal.
|
||||
score := m.length - m.s
|
||||
if nextEmit == m.s {
|
||||
// If we do not have to emit literals, we save 1 byte
|
||||
score++
|
||||
}
|
||||
offset := m.s - m.offset
|
||||
|
||||
return score - emitCopyNoRepeatSize(offset, m.length)
|
||||
}
|
||||
|
||||
matchAt := func(offset, s int, first uint32) match {
|
||||
if best.length != 0 && best.s-best.offset == s-offset {
|
||||
// Don't retest if we have the same offset.
|
||||
return match{offset: offset, s: s}
|
||||
}
|
||||
if load32(src, offset) != first {
|
||||
return match{offset: offset, s: s}
|
||||
}
|
||||
m := match{offset: offset, s: s, length: 4 + offset}
|
||||
s += 4
|
||||
for s <= sLimit {
|
||||
if diff := load64(src, s) ^ load64(src, m.length); diff != 0 {
|
||||
m.length += bits.TrailingZeros64(diff) >> 3
|
||||
break
|
||||
}
|
||||
s += 8
|
||||
m.length += 8
|
||||
}
|
||||
m.length -= offset
|
||||
m.score = score(m)
|
||||
if m.score <= -m.s {
|
||||
// Eliminate if no savings, we might find a better one.
|
||||
m.length = 0
|
||||
}
|
||||
return m
|
||||
}
|
||||
|
||||
bestOf := func(a, b match) match {
|
||||
if b.length == 0 {
|
||||
return a
|
||||
}
|
||||
if a.length == 0 {
|
||||
return b
|
||||
}
|
||||
as := a.score + b.s
|
||||
bs := b.score + a.s
|
||||
if as >= bs {
|
||||
return a
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
best = bestOf(matchAt(getCur(candidateL), s, uint32(cv)), matchAt(getPrev(candidateL), s, uint32(cv)))
|
||||
best = bestOf(best, matchAt(getCur(candidateS), s, uint32(cv)))
|
||||
best = bestOf(best, matchAt(getPrev(candidateS), s, uint32(cv)))
|
||||
|
||||
{
|
||||
best = bestOf(best, matchAt(s-repeat+1, s+1, uint32(cv>>8)))
|
||||
if best.length > 0 {
|
||||
// s+1
|
||||
nextShort := sTable[hash4(cv>>8, sTableBits)]
|
||||
s := s + 1
|
||||
cv := load64(src, s)
|
||||
nextLong := lTable[hash8(cv, lTableBits)]
|
||||
best = bestOf(best, matchAt(getCur(nextShort), s, uint32(cv)))
|
||||
best = bestOf(best, matchAt(getPrev(nextShort), s, uint32(cv)))
|
||||
best = bestOf(best, matchAt(getCur(nextLong), s, uint32(cv)))
|
||||
best = bestOf(best, matchAt(getPrev(nextLong), s, uint32(cv)))
|
||||
// Repeat at + 2
|
||||
best = bestOf(best, matchAt(s-repeat+1, s+1, uint32(cv>>8)))
|
||||
|
||||
// s+2
|
||||
if true {
|
||||
nextShort = sTable[hash4(cv>>8, sTableBits)]
|
||||
s++
|
||||
cv = load64(src, s)
|
||||
nextLong = lTable[hash8(cv, lTableBits)]
|
||||
best = bestOf(best, matchAt(getCur(nextShort), s, uint32(cv)))
|
||||
best = bestOf(best, matchAt(getPrev(nextShort), s, uint32(cv)))
|
||||
best = bestOf(best, matchAt(getCur(nextLong), s, uint32(cv)))
|
||||
best = bestOf(best, matchAt(getPrev(nextLong), s, uint32(cv)))
|
||||
}
|
||||
// Search for a match at best match end, see if that is better.
|
||||
if sAt := best.s + best.length; sAt < sLimit {
|
||||
sBack := best.s
|
||||
backL := best.length
|
||||
// Load initial values
|
||||
cv = load64(src, sBack)
|
||||
// Search for mismatch
|
||||
next := lTable[hash8(load64(src, sAt), lTableBits)]
|
||||
//next := sTable[hash4(load64(src, sAt), sTableBits)]
|
||||
|
||||
if checkAt := getCur(next) - backL; checkAt > 0 {
|
||||
best = bestOf(best, matchAt(checkAt, sBack, uint32(cv)))
|
||||
}
|
||||
if checkAt := getPrev(next) - backL; checkAt > 0 {
|
||||
best = bestOf(best, matchAt(checkAt, sBack, uint32(cv)))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Update table
|
||||
lTable[hashL] = uint64(s) | candidateL<<32
|
||||
sTable[hashS] = uint64(s) | candidateS<<32
|
||||
|
||||
if best.length > 0 {
|
||||
break
|
||||
}
|
||||
|
||||
cv = load64(src, nextS)
|
||||
s = nextS
|
||||
}
|
||||
|
||||
// Extend backwards, not needed for repeats...
|
||||
s = best.s
|
||||
if true {
|
||||
for best.offset > 0 && s > nextEmit && src[best.offset-1] == src[s-1] {
|
||||
best.offset--
|
||||
best.length++
|
||||
s--
|
||||
}
|
||||
}
|
||||
if false && best.offset >= s {
|
||||
panic(fmt.Errorf("t %d >= s %d", best.offset, s))
|
||||
}
|
||||
// Bail if we exceed the maximum size.
|
||||
if d+(s-nextEmit) > dstLimit {
|
||||
return 0
|
||||
}
|
||||
|
||||
base := s
|
||||
offset := s - best.offset
|
||||
|
||||
s += best.length
|
||||
|
||||
if offset > 65535 && s-base <= 5 {
|
||||
// Bail if the match is equal or worse to the encoding.
|
||||
s = best.s + 1
|
||||
if s >= sLimit {
|
||||
goto emitRemainder
|
||||
}
|
||||
cv = load64(src, s)
|
||||
continue
|
||||
}
|
||||
d += emitLiteral(dst[d:], src[nextEmit:base])
|
||||
d += emitCopyNoRepeat(dst[d:], offset, best.length)
|
||||
repeat = offset
|
||||
|
||||
nextEmit = s
|
||||
if s >= sLimit {
|
||||
goto emitRemainder
|
||||
}
|
||||
|
||||
if d > dstLimit {
|
||||
// Do we have space for more, if not bail.
|
||||
return 0
|
||||
}
|
||||
// Fill tables...
|
||||
for i := best.s + 1; i < s; i++ {
|
||||
cv0 := load64(src, i)
|
||||
long0 := hash8(cv0, lTableBits)
|
||||
short0 := hash4(cv0, sTableBits)
|
||||
lTable[long0] = uint64(i) | lTable[long0]<<32
|
||||
sTable[short0] = uint64(i) | sTable[short0]<<32
|
||||
}
|
||||
cv = load64(src, s)
|
||||
}
|
||||
|
||||
emitRemainder:
|
||||
if nextEmit < len(src) {
|
||||
// Bail if we exceed the maximum size.
|
||||
if d+len(src)-nextEmit > dstLimit {
|
||||
return 0
|
||||
}
|
||||
d += emitLiteral(dst[d:], src[nextEmit:])
|
||||
}
|
||||
return d
|
||||
}
|
||||
|
||||
// emitCopySize returns the size to encode the offset+length
|
||||
//
|
||||
// It assumes that:
|
||||
//
|
||||
// 1 <= offset && offset <= math.MaxUint32
|
||||
// 4 <= length && length <= 1 << 24
|
||||
func emitCopySize(offset, length int) int {
|
||||
if offset >= 65536 {
|
||||
i := 0
|
||||
if length > 64 {
|
||||
length -= 64
|
||||
if length >= 4 {
|
||||
// Emit remaining as repeats
|
||||
return 5 + emitRepeatSize(offset, length)
|
||||
}
|
||||
i = 5
|
||||
}
|
||||
if length == 0 {
|
||||
return i
|
||||
}
|
||||
return i + 5
|
||||
}
|
||||
|
||||
// Offset no more than 2 bytes.
|
||||
if length > 64 {
|
||||
if offset < 2048 {
|
||||
// Emit 8 bytes, then rest as repeats...
|
||||
return 2 + emitRepeatSize(offset, length-8)
|
||||
}
|
||||
// Emit remaining as repeats, at least 4 bytes remain.
|
||||
return 3 + emitRepeatSize(offset, length-60)
|
||||
}
|
||||
if length >= 12 || offset >= 2048 {
|
||||
return 3
|
||||
}
|
||||
// Emit the remaining copy, encoded as 2 bytes.
|
||||
return 2
|
||||
}
|
||||
|
||||
// emitCopyNoRepeatSize returns the size to encode the offset+length
|
||||
//
|
||||
// It assumes that:
|
||||
//
|
||||
// 1 <= offset && offset <= math.MaxUint32
|
||||
// 4 <= length && length <= 1 << 24
|
||||
func emitCopyNoRepeatSize(offset, length int) int {
|
||||
if offset >= 65536 {
|
||||
return 5 + 5*(length/64)
|
||||
}
|
||||
|
||||
// Offset no more than 2 bytes.
|
||||
if length > 64 {
|
||||
// Emit remaining as repeats, at least 4 bytes remain.
|
||||
return 3 + 3*(length/60)
|
||||
}
|
||||
if length >= 12 || offset >= 2048 {
|
||||
return 3
|
||||
}
|
||||
// Emit the remaining copy, encoded as 2 bytes.
|
||||
return 2
|
||||
}
|
||||
|
||||
// emitRepeatSize returns the number of bytes required to encode a repeat.
|
||||
// Length must be at least 4 and < 1<<24
|
||||
func emitRepeatSize(offset, length int) int {
|
||||
// Repeat offset, make length cheaper
|
||||
if length <= 4+4 || (length < 8+4 && offset < 2048) {
|
||||
return 2
|
||||
}
|
||||
if length < (1<<8)+4+4 {
|
||||
return 3
|
||||
}
|
||||
if length < (1<<16)+(1<<8)+4 {
|
||||
return 4
|
||||
}
|
||||
const maxRepeat = (1 << 24) - 1
|
||||
length -= (1 << 16) - 4
|
||||
left := 0
|
||||
if length > maxRepeat {
|
||||
left = length - maxRepeat + 4
|
||||
}
|
||||
if left > 0 {
|
||||
return 5 + emitRepeatSize(offset, left)
|
||||
}
|
||||
return 5
|
||||
}
|
||||
+1519
File diff suppressed because it is too large
Load Diff
+740
@@ -0,0 +1,740 @@
|
||||
//go:build (!amd64 && !arm64) || appengine || !gc || noasm
|
||||
|
||||
package s2
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"math/bits"
|
||||
)
|
||||
|
||||
const hasAsm = false
|
||||
|
||||
// encodeBlock encodes a non-empty src to a guaranteed-large-enough dst. It
|
||||
// assumes that the varint-encoded length of the decompressed bytes has already
|
||||
// been written.
|
||||
//
|
||||
// It also assumes that:
|
||||
//
|
||||
// len(dst) >= MaxEncodedLen(len(src))
|
||||
func encodeBlock(dst, src []byte) (d int) {
|
||||
if len(src) < minNonLiteralBlockSize {
|
||||
return 0
|
||||
}
|
||||
if len(src) <= 64<<10 {
|
||||
return encodeBlockGo64K(dst, src)
|
||||
}
|
||||
return encodeBlockGo(dst, src)
|
||||
}
|
||||
|
||||
// encodeBlockBetter encodes a non-empty src to a guaranteed-large-enough dst. It
|
||||
// assumes that the varint-encoded length of the decompressed bytes has already
|
||||
// been written.
|
||||
//
|
||||
// It also assumes that:
|
||||
//
|
||||
// len(dst) >= MaxEncodedLen(len(src))
|
||||
func encodeBlockBetter(dst, src []byte) (d int) {
|
||||
if len(src) <= 64<<10 {
|
||||
return encodeBlockBetterGo64K(dst, src)
|
||||
}
|
||||
return encodeBlockBetterGo(dst, src)
|
||||
}
|
||||
|
||||
// encodeBlockBetter encodes a non-empty src to a guaranteed-large-enough dst. It
|
||||
// assumes that the varint-encoded length of the decompressed bytes has already
|
||||
// been written.
|
||||
//
|
||||
// It also assumes that:
|
||||
//
|
||||
// len(dst) >= MaxEncodedLen(len(src))
|
||||
func encodeBlockBetterSnappy(dst, src []byte) (d int) {
|
||||
if len(src) <= 64<<10 {
|
||||
return encodeBlockBetterSnappyGo64K(dst, src)
|
||||
}
|
||||
return encodeBlockBetterSnappyGo(dst, src)
|
||||
}
|
||||
|
||||
// encodeBlock encodes a non-empty src to a guaranteed-large-enough dst. It
|
||||
// assumes that the varint-encoded length of the decompressed bytes has already
|
||||
// been written.
|
||||
//
|
||||
// It also assumes that:
|
||||
//
|
||||
// len(dst) >= MaxEncodedLen(len(src))
|
||||
func encodeBlockSnappy(dst, src []byte) (d int) {
|
||||
if len(src) < minNonLiteralBlockSize {
|
||||
return 0
|
||||
}
|
||||
if len(src) <= 64<<10 {
|
||||
return encodeBlockSnappyGo64K(dst, src)
|
||||
}
|
||||
return encodeBlockSnappyGo(dst, src)
|
||||
}
|
||||
|
||||
// emitLiteral writes a literal chunk and returns the number of bytes written.
|
||||
//
|
||||
// It assumes that:
|
||||
//
|
||||
// dst is long enough to hold the encoded bytes
|
||||
// 0 <= len(lit) && len(lit) <= math.MaxUint32
|
||||
func emitLiteral(dst, lit []byte) int {
|
||||
if len(lit) == 0 {
|
||||
return 0
|
||||
}
|
||||
const num = 63<<2 | tagLiteral
|
||||
i, n := 0, uint(len(lit)-1)
|
||||
switch {
|
||||
case n < 60:
|
||||
dst[0] = uint8(n)<<2 | tagLiteral
|
||||
i = 1
|
||||
case n < 1<<8:
|
||||
dst[1] = uint8(n)
|
||||
dst[0] = 60<<2 | tagLiteral
|
||||
i = 2
|
||||
case n < 1<<16:
|
||||
dst[2] = uint8(n >> 8)
|
||||
dst[1] = uint8(n)
|
||||
dst[0] = 61<<2 | tagLiteral
|
||||
i = 3
|
||||
case n < 1<<24:
|
||||
dst[3] = uint8(n >> 16)
|
||||
dst[2] = uint8(n >> 8)
|
||||
dst[1] = uint8(n)
|
||||
dst[0] = 62<<2 | tagLiteral
|
||||
i = 4
|
||||
default:
|
||||
dst[4] = uint8(n >> 24)
|
||||
dst[3] = uint8(n >> 16)
|
||||
dst[2] = uint8(n >> 8)
|
||||
dst[1] = uint8(n)
|
||||
dst[0] = 63<<2 | tagLiteral
|
||||
i = 5
|
||||
}
|
||||
return i + copy(dst[i:], lit)
|
||||
}
|
||||
|
||||
// emitRepeat writes a repeat chunk and returns the number of bytes written.
|
||||
// Length must be at least 4 and < 1<<24
|
||||
func emitRepeat(dst []byte, offset, length int) int {
|
||||
// Repeat offset, make length cheaper
|
||||
length -= 4
|
||||
if length <= 4 {
|
||||
dst[0] = uint8(length)<<2 | tagCopy1
|
||||
dst[1] = 0
|
||||
return 2
|
||||
}
|
||||
if length < 8 && offset < 2048 {
|
||||
// Encode WITH offset
|
||||
dst[1] = uint8(offset)
|
||||
dst[0] = uint8(offset>>8)<<5 | uint8(length)<<2 | tagCopy1
|
||||
return 2
|
||||
}
|
||||
if length < (1<<8)+4 {
|
||||
length -= 4
|
||||
dst[2] = uint8(length)
|
||||
dst[1] = 0
|
||||
dst[0] = 5<<2 | tagCopy1
|
||||
return 3
|
||||
}
|
||||
if length < (1<<16)+(1<<8) {
|
||||
length -= 1 << 8
|
||||
dst[3] = uint8(length >> 8)
|
||||
dst[2] = uint8(length >> 0)
|
||||
dst[1] = 0
|
||||
dst[0] = 6<<2 | tagCopy1
|
||||
return 4
|
||||
}
|
||||
const maxRepeat = (1 << 24) - 1
|
||||
length -= 1 << 16
|
||||
left := 0
|
||||
if length > maxRepeat {
|
||||
left = length - maxRepeat + 4
|
||||
length = maxRepeat - 4
|
||||
}
|
||||
dst[4] = uint8(length >> 16)
|
||||
dst[3] = uint8(length >> 8)
|
||||
dst[2] = uint8(length >> 0)
|
||||
dst[1] = 0
|
||||
dst[0] = 7<<2 | tagCopy1
|
||||
if left > 0 {
|
||||
return 5 + emitRepeat(dst[5:], offset, left)
|
||||
}
|
||||
return 5
|
||||
}
|
||||
|
||||
// emitCopy writes a copy chunk and returns the number of bytes written.
|
||||
//
|
||||
// It assumes that:
|
||||
//
|
||||
// dst is long enough to hold the encoded bytes
|
||||
// 1 <= offset && offset <= math.MaxUint32
|
||||
// 4 <= length && length <= 1 << 24
|
||||
func emitCopy(dst []byte, offset, length int) int {
|
||||
if offset >= 65536 {
|
||||
i := 0
|
||||
if length > 64 {
|
||||
// Emit a length 64 copy, encoded as 5 bytes.
|
||||
dst[4] = uint8(offset >> 24)
|
||||
dst[3] = uint8(offset >> 16)
|
||||
dst[2] = uint8(offset >> 8)
|
||||
dst[1] = uint8(offset)
|
||||
dst[0] = 63<<2 | tagCopy4
|
||||
length -= 64
|
||||
if length >= 4 {
|
||||
// Emit remaining as repeats
|
||||
return 5 + emitRepeat(dst[5:], offset, length)
|
||||
}
|
||||
i = 5
|
||||
}
|
||||
if length == 0 {
|
||||
return i
|
||||
}
|
||||
// Emit a copy, offset encoded as 4 bytes.
|
||||
dst[i+0] = uint8(length-1)<<2 | tagCopy4
|
||||
dst[i+1] = uint8(offset)
|
||||
dst[i+2] = uint8(offset >> 8)
|
||||
dst[i+3] = uint8(offset >> 16)
|
||||
dst[i+4] = uint8(offset >> 24)
|
||||
return i + 5
|
||||
}
|
||||
|
||||
// Offset no more than 2 bytes.
|
||||
if length > 64 {
|
||||
off := 3
|
||||
if offset < 2048 {
|
||||
// emit 8 bytes as tagCopy1, rest as repeats.
|
||||
dst[1] = uint8(offset)
|
||||
dst[0] = uint8(offset>>8)<<5 | uint8(8-4)<<2 | tagCopy1
|
||||
length -= 8
|
||||
off = 2
|
||||
} else {
|
||||
// Emit a length 60 copy, encoded as 3 bytes.
|
||||
// Emit remaining as repeat value (minimum 4 bytes).
|
||||
dst[2] = uint8(offset >> 8)
|
||||
dst[1] = uint8(offset)
|
||||
dst[0] = 59<<2 | tagCopy2
|
||||
length -= 60
|
||||
}
|
||||
// Emit remaining as repeats, at least 4 bytes remain.
|
||||
return off + emitRepeat(dst[off:], offset, length)
|
||||
}
|
||||
if length >= 12 || offset >= 2048 {
|
||||
// Emit the remaining copy, encoded as 3 bytes.
|
||||
dst[2] = uint8(offset >> 8)
|
||||
dst[1] = uint8(offset)
|
||||
dst[0] = uint8(length-1)<<2 | tagCopy2
|
||||
return 3
|
||||
}
|
||||
// Emit the remaining copy, encoded as 2 bytes.
|
||||
dst[1] = uint8(offset)
|
||||
dst[0] = uint8(offset>>8)<<5 | uint8(length-4)<<2 | tagCopy1
|
||||
return 2
|
||||
}
|
||||
|
||||
// emitCopyNoRepeat writes a copy chunk and returns the number of bytes written.
|
||||
//
|
||||
// It assumes that:
|
||||
//
|
||||
// dst is long enough to hold the encoded bytes
|
||||
// 1 <= offset && offset <= math.MaxUint32
|
||||
// 4 <= length && length <= 1 << 24
|
||||
func emitCopyNoRepeat(dst []byte, offset, length int) int {
|
||||
if offset >= 65536 {
|
||||
i := 0
|
||||
if length > 64 {
|
||||
// Emit a length 64 copy, encoded as 5 bytes.
|
||||
dst[4] = uint8(offset >> 24)
|
||||
dst[3] = uint8(offset >> 16)
|
||||
dst[2] = uint8(offset >> 8)
|
||||
dst[1] = uint8(offset)
|
||||
dst[0] = 63<<2 | tagCopy4
|
||||
length -= 64
|
||||
if length >= 4 {
|
||||
// Emit remaining as repeats
|
||||
return 5 + emitCopyNoRepeat(dst[5:], offset, length)
|
||||
}
|
||||
i = 5
|
||||
}
|
||||
if length == 0 {
|
||||
return i
|
||||
}
|
||||
// Emit a copy, offset encoded as 4 bytes.
|
||||
dst[i+0] = uint8(length-1)<<2 | tagCopy4
|
||||
dst[i+1] = uint8(offset)
|
||||
dst[i+2] = uint8(offset >> 8)
|
||||
dst[i+3] = uint8(offset >> 16)
|
||||
dst[i+4] = uint8(offset >> 24)
|
||||
return i + 5
|
||||
}
|
||||
|
||||
// Offset no more than 2 bytes.
|
||||
if length > 64 {
|
||||
// Emit a length 60 copy, encoded as 3 bytes.
|
||||
// Emit remaining as repeat value (minimum 4 bytes).
|
||||
dst[2] = uint8(offset >> 8)
|
||||
dst[1] = uint8(offset)
|
||||
dst[0] = 59<<2 | tagCopy2
|
||||
length -= 60
|
||||
// Emit remaining as repeats, at least 4 bytes remain.
|
||||
return 3 + emitCopyNoRepeat(dst[3:], offset, length)
|
||||
}
|
||||
if length >= 12 || offset >= 2048 {
|
||||
// Emit the remaining copy, encoded as 3 bytes.
|
||||
dst[2] = uint8(offset >> 8)
|
||||
dst[1] = uint8(offset)
|
||||
dst[0] = uint8(length-1)<<2 | tagCopy2
|
||||
return 3
|
||||
}
|
||||
// Emit the remaining copy, encoded as 2 bytes.
|
||||
dst[1] = uint8(offset)
|
||||
dst[0] = uint8(offset>>8)<<5 | uint8(length-4)<<2 | tagCopy1
|
||||
return 2
|
||||
}
|
||||
|
||||
// matchLen returns how many bytes match in a and b
|
||||
//
|
||||
// It assumes that:
|
||||
//
|
||||
// len(a) <= len(b)
|
||||
func matchLen(a []byte, b []byte) int {
|
||||
b = b[:len(a)]
|
||||
var checked int
|
||||
if len(a) > 4 {
|
||||
// Try 4 bytes first
|
||||
if diff := load32(a, 0) ^ load32(b, 0); diff != 0 {
|
||||
return bits.TrailingZeros32(diff) >> 3
|
||||
}
|
||||
// Switch to 8 byte matching.
|
||||
checked = 4
|
||||
a = a[4:]
|
||||
b = b[4:]
|
||||
for len(a) >= 8 {
|
||||
b = b[:len(a)]
|
||||
if diff := load64(a, 0) ^ load64(b, 0); diff != 0 {
|
||||
return checked + (bits.TrailingZeros64(diff) >> 3)
|
||||
}
|
||||
checked += 8
|
||||
a = a[8:]
|
||||
b = b[8:]
|
||||
}
|
||||
}
|
||||
b = b[:len(a)]
|
||||
for i := range a {
|
||||
if a[i] != b[i] {
|
||||
return int(i) + checked
|
||||
}
|
||||
}
|
||||
return len(a) + checked
|
||||
}
|
||||
|
||||
// input must be > inputMargin
|
||||
func calcBlockSize(src []byte, _ *[32768]byte) (d int) {
|
||||
// Initialize the hash table.
|
||||
const (
|
||||
tableBits = 13
|
||||
maxTableSize = 1 << tableBits
|
||||
)
|
||||
|
||||
var table [maxTableSize]uint32
|
||||
|
||||
// sLimit is when to stop looking for offset/length copies. The inputMargin
|
||||
// lets us use a fast path for emitLiteral in the main loop, while we are
|
||||
// looking for copies.
|
||||
sLimit := len(src) - inputMargin
|
||||
|
||||
// Bail if we can't compress to at least this.
|
||||
dstLimit := len(src) - len(src)>>5 - 5
|
||||
|
||||
// nextEmit is where in src the next emitLiteral should start from.
|
||||
nextEmit := 0
|
||||
|
||||
// The encoded form must start with a literal, as there are no previous
|
||||
// bytes to copy, so we start looking for hash matches at s == 1.
|
||||
s := 1
|
||||
cv := load64(src, s)
|
||||
|
||||
// We search for a repeat at -1, but don't output repeats when nextEmit == 0
|
||||
repeat := 1
|
||||
|
||||
for {
|
||||
candidate := 0
|
||||
for {
|
||||
// Next src position to check
|
||||
nextS := s + (s-nextEmit)>>6 + 4
|
||||
if nextS > sLimit {
|
||||
goto emitRemainder
|
||||
}
|
||||
hash0 := hash6(cv, tableBits)
|
||||
hash1 := hash6(cv>>8, tableBits)
|
||||
candidate = int(table[hash0])
|
||||
candidate2 := int(table[hash1])
|
||||
table[hash0] = uint32(s)
|
||||
table[hash1] = uint32(s + 1)
|
||||
hash2 := hash6(cv>>16, tableBits)
|
||||
|
||||
// Check repeat at offset checkRep.
|
||||
const checkRep = 1
|
||||
if uint32(cv>>(checkRep*8)) == load32(src, s-repeat+checkRep) {
|
||||
base := s + checkRep
|
||||
// Extend back
|
||||
for i := base - repeat; base > nextEmit && i > 0 && src[i-1] == src[base-1]; {
|
||||
i--
|
||||
base--
|
||||
}
|
||||
d += emitLiteralSize(src[nextEmit:base])
|
||||
|
||||
// Extend forward
|
||||
candidate := s - repeat + 4 + checkRep
|
||||
s += 4 + checkRep
|
||||
for s <= sLimit {
|
||||
if diff := load64(src, s) ^ load64(src, candidate); diff != 0 {
|
||||
s += bits.TrailingZeros64(diff) >> 3
|
||||
break
|
||||
}
|
||||
s += 8
|
||||
candidate += 8
|
||||
}
|
||||
|
||||
d += emitCopyNoRepeatSize(repeat, s-base)
|
||||
nextEmit = s
|
||||
if s >= sLimit {
|
||||
goto emitRemainder
|
||||
}
|
||||
|
||||
cv = load64(src, s)
|
||||
continue
|
||||
}
|
||||
|
||||
if uint32(cv) == load32(src, candidate) {
|
||||
break
|
||||
}
|
||||
candidate = int(table[hash2])
|
||||
if uint32(cv>>8) == load32(src, candidate2) {
|
||||
table[hash2] = uint32(s + 2)
|
||||
candidate = candidate2
|
||||
s++
|
||||
break
|
||||
}
|
||||
table[hash2] = uint32(s + 2)
|
||||
if uint32(cv>>16) == load32(src, candidate) {
|
||||
s += 2
|
||||
break
|
||||
}
|
||||
|
||||
cv = load64(src, nextS)
|
||||
s = nextS
|
||||
}
|
||||
|
||||
// Extend backwards
|
||||
for candidate > 0 && s > nextEmit && src[candidate-1] == src[s-1] {
|
||||
candidate--
|
||||
s--
|
||||
}
|
||||
|
||||
// Bail if we exceed the maximum size.
|
||||
if d+(s-nextEmit) > dstLimit {
|
||||
return 0
|
||||
}
|
||||
|
||||
// A 4-byte match has been found. We'll later see if more than 4 bytes
|
||||
// match. But, prior to the match, src[nextEmit:s] are unmatched. Emit
|
||||
// them as literal bytes.
|
||||
|
||||
d += emitLiteralSize(src[nextEmit:s])
|
||||
|
||||
// Call emitCopy, and then see if another emitCopy could be our next
|
||||
// move. Repeat until we find no match for the input immediately after
|
||||
// what was consumed by the last emitCopy call.
|
||||
//
|
||||
// If we exit this loop normally then we need to call emitLiteral next,
|
||||
// though we don't yet know how big the literal will be. We handle that
|
||||
// by proceeding to the next iteration of the main loop. We also can
|
||||
// exit this loop via goto if we get close to exhausting the input.
|
||||
for {
|
||||
// Invariant: we have a 4-byte match at s, and no need to emit any
|
||||
// literal bytes prior to s.
|
||||
base := s
|
||||
repeat = base - candidate
|
||||
|
||||
// Extend the 4-byte match as long as possible.
|
||||
s += 4
|
||||
candidate += 4
|
||||
for s <= len(src)-8 {
|
||||
if diff := load64(src, s) ^ load64(src, candidate); diff != 0 {
|
||||
s += bits.TrailingZeros64(diff) >> 3
|
||||
break
|
||||
}
|
||||
s += 8
|
||||
candidate += 8
|
||||
}
|
||||
|
||||
d += emitCopyNoRepeatSize(repeat, s-base)
|
||||
if false {
|
||||
// Validate match.
|
||||
a := src[base:s]
|
||||
b := src[base-repeat : base-repeat+(s-base)]
|
||||
if !bytes.Equal(a, b) {
|
||||
panic("mismatch")
|
||||
}
|
||||
}
|
||||
|
||||
nextEmit = s
|
||||
if s >= sLimit {
|
||||
goto emitRemainder
|
||||
}
|
||||
|
||||
if d > dstLimit {
|
||||
// Do we have space for more, if not bail.
|
||||
return 0
|
||||
}
|
||||
// Check for an immediate match, otherwise start search at s+1
|
||||
x := load64(src, s-2)
|
||||
m2Hash := hash6(x, tableBits)
|
||||
currHash := hash6(x>>16, tableBits)
|
||||
candidate = int(table[currHash])
|
||||
table[m2Hash] = uint32(s - 2)
|
||||
table[currHash] = uint32(s)
|
||||
if uint32(x>>16) != load32(src, candidate) {
|
||||
cv = load64(src, s+1)
|
||||
s++
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
emitRemainder:
|
||||
if nextEmit < len(src) {
|
||||
// Bail if we exceed the maximum size.
|
||||
if d+len(src)-nextEmit > dstLimit {
|
||||
return 0
|
||||
}
|
||||
d += emitLiteralSize(src[nextEmit:])
|
||||
}
|
||||
return d
|
||||
}
|
||||
|
||||
// length must be > inputMargin.
|
||||
func calcBlockSizeSmall(src []byte, _ *[2048]byte) (d int) {
|
||||
// Initialize the hash table.
|
||||
const (
|
||||
tableBits = 9
|
||||
maxTableSize = 1 << tableBits
|
||||
)
|
||||
|
||||
var table [maxTableSize]uint32
|
||||
|
||||
// sLimit is when to stop looking for offset/length copies. The inputMargin
|
||||
// lets us use a fast path for emitLiteral in the main loop, while we are
|
||||
// looking for copies.
|
||||
sLimit := len(src) - inputMargin
|
||||
|
||||
// Bail if we can't compress to at least this.
|
||||
dstLimit := len(src) - len(src)>>5 - 5
|
||||
|
||||
// nextEmit is where in src the next emitLiteral should start from.
|
||||
nextEmit := 0
|
||||
|
||||
// The encoded form must start with a literal, as there are no previous
|
||||
// bytes to copy, so we start looking for hash matches at s == 1.
|
||||
s := 1
|
||||
cv := load64(src, s)
|
||||
|
||||
// We search for a repeat at -1, but don't output repeats when nextEmit == 0
|
||||
repeat := 1
|
||||
|
||||
for {
|
||||
candidate := 0
|
||||
for {
|
||||
// Next src position to check
|
||||
nextS := s + (s-nextEmit)>>6 + 4
|
||||
if nextS > sLimit {
|
||||
goto emitRemainder
|
||||
}
|
||||
hash0 := hash6(cv, tableBits)
|
||||
hash1 := hash6(cv>>8, tableBits)
|
||||
candidate = int(table[hash0])
|
||||
candidate2 := int(table[hash1])
|
||||
table[hash0] = uint32(s)
|
||||
table[hash1] = uint32(s + 1)
|
||||
hash2 := hash6(cv>>16, tableBits)
|
||||
|
||||
// Check repeat at offset checkRep.
|
||||
const checkRep = 1
|
||||
if uint32(cv>>(checkRep*8)) == load32(src, s-repeat+checkRep) {
|
||||
base := s + checkRep
|
||||
// Extend back
|
||||
for i := base - repeat; base > nextEmit && i > 0 && src[i-1] == src[base-1]; {
|
||||
i--
|
||||
base--
|
||||
}
|
||||
d += emitLiteralSize(src[nextEmit:base])
|
||||
|
||||
// Extend forward
|
||||
candidate := s - repeat + 4 + checkRep
|
||||
s += 4 + checkRep
|
||||
for s <= sLimit {
|
||||
if diff := load64(src, s) ^ load64(src, candidate); diff != 0 {
|
||||
s += bits.TrailingZeros64(diff) >> 3
|
||||
break
|
||||
}
|
||||
s += 8
|
||||
candidate += 8
|
||||
}
|
||||
|
||||
d += emitCopyNoRepeatSize(repeat, s-base)
|
||||
nextEmit = s
|
||||
if s >= sLimit {
|
||||
goto emitRemainder
|
||||
}
|
||||
|
||||
cv = load64(src, s)
|
||||
continue
|
||||
}
|
||||
|
||||
if uint32(cv) == load32(src, candidate) {
|
||||
break
|
||||
}
|
||||
candidate = int(table[hash2])
|
||||
if uint32(cv>>8) == load32(src, candidate2) {
|
||||
table[hash2] = uint32(s + 2)
|
||||
candidate = candidate2
|
||||
s++
|
||||
break
|
||||
}
|
||||
table[hash2] = uint32(s + 2)
|
||||
if uint32(cv>>16) == load32(src, candidate) {
|
||||
s += 2
|
||||
break
|
||||
}
|
||||
|
||||
cv = load64(src, nextS)
|
||||
s = nextS
|
||||
}
|
||||
|
||||
// Extend backwards
|
||||
for candidate > 0 && s > nextEmit && src[candidate-1] == src[s-1] {
|
||||
candidate--
|
||||
s--
|
||||
}
|
||||
|
||||
// Bail if we exceed the maximum size.
|
||||
if d+(s-nextEmit) > dstLimit {
|
||||
return 0
|
||||
}
|
||||
|
||||
// A 4-byte match has been found. We'll later see if more than 4 bytes
|
||||
// match. But, prior to the match, src[nextEmit:s] are unmatched. Emit
|
||||
// them as literal bytes.
|
||||
|
||||
d += emitLiteralSize(src[nextEmit:s])
|
||||
|
||||
// Call emitCopy, and then see if another emitCopy could be our next
|
||||
// move. Repeat until we find no match for the input immediately after
|
||||
// what was consumed by the last emitCopy call.
|
||||
//
|
||||
// If we exit this loop normally then we need to call emitLiteral next,
|
||||
// though we don't yet know how big the literal will be. We handle that
|
||||
// by proceeding to the next iteration of the main loop. We also can
|
||||
// exit this loop via goto if we get close to exhausting the input.
|
||||
for {
|
||||
// Invariant: we have a 4-byte match at s, and no need to emit any
|
||||
// literal bytes prior to s.
|
||||
base := s
|
||||
repeat = base - candidate
|
||||
|
||||
// Extend the 4-byte match as long as possible.
|
||||
s += 4
|
||||
candidate += 4
|
||||
for s <= len(src)-8 {
|
||||
if diff := load64(src, s) ^ load64(src, candidate); diff != 0 {
|
||||
s += bits.TrailingZeros64(diff) >> 3
|
||||
break
|
||||
}
|
||||
s += 8
|
||||
candidate += 8
|
||||
}
|
||||
|
||||
d += emitCopyNoRepeatSize(repeat, s-base)
|
||||
if false {
|
||||
// Validate match.
|
||||
a := src[base:s]
|
||||
b := src[base-repeat : base-repeat+(s-base)]
|
||||
if !bytes.Equal(a, b) {
|
||||
panic("mismatch")
|
||||
}
|
||||
}
|
||||
|
||||
nextEmit = s
|
||||
if s >= sLimit {
|
||||
goto emitRemainder
|
||||
}
|
||||
|
||||
if d > dstLimit {
|
||||
// Do we have space for more, if not bail.
|
||||
return 0
|
||||
}
|
||||
// Check for an immediate match, otherwise start search at s+1
|
||||
x := load64(src, s-2)
|
||||
m2Hash := hash6(x, tableBits)
|
||||
currHash := hash6(x>>16, tableBits)
|
||||
candidate = int(table[currHash])
|
||||
table[m2Hash] = uint32(s - 2)
|
||||
table[currHash] = uint32(s)
|
||||
if uint32(x>>16) != load32(src, candidate) {
|
||||
cv = load64(src, s+1)
|
||||
s++
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
emitRemainder:
|
||||
if nextEmit < len(src) {
|
||||
// Bail if we exceed the maximum size.
|
||||
if d+len(src)-nextEmit > dstLimit {
|
||||
return 0
|
||||
}
|
||||
d += emitLiteralSize(src[nextEmit:])
|
||||
}
|
||||
return d
|
||||
}
|
||||
|
||||
// emitLiteral writes a literal chunk and returns the number of bytes written.
|
||||
//
|
||||
// It assumes that:
|
||||
//
|
||||
// dst is long enough to hold the encoded bytes
|
||||
// 0 <= len(lit) && len(lit) <= math.MaxUint32
|
||||
func emitLiteralSize(lit []byte) int {
|
||||
if len(lit) == 0 {
|
||||
return 0
|
||||
}
|
||||
switch {
|
||||
case len(lit) <= 60:
|
||||
return len(lit) + 1
|
||||
case len(lit) <= 1<<8:
|
||||
return len(lit) + 2
|
||||
case len(lit) <= 1<<16:
|
||||
return len(lit) + 3
|
||||
case len(lit) <= 1<<24:
|
||||
return len(lit) + 4
|
||||
default:
|
||||
return len(lit) + 5
|
||||
}
|
||||
}
|
||||
|
||||
func cvtLZ4BlockAsm(dst []byte, src []byte) (uncompressed int, dstUsed int) {
|
||||
panic("cvtLZ4BlockAsm should be unreachable")
|
||||
}
|
||||
|
||||
func cvtLZ4BlockSnappyAsm(dst []byte, src []byte) (uncompressed int, dstUsed int) {
|
||||
panic("cvtLZ4BlockSnappyAsm should be unreachable")
|
||||
}
|
||||
|
||||
func cvtLZ4sBlockAsm(dst []byte, src []byte) (uncompressed int, dstUsed int) {
|
||||
panic("cvtLZ4sBlockAsm should be unreachable")
|
||||
}
|
||||
|
||||
func cvtLZ4sBlockSnappyAsm(dst []byte, src []byte) (uncompressed int, dstUsed int) {
|
||||
panic("cvtLZ4sBlockSnappyAsm should be unreachable")
|
||||
}
|
||||
+20883
File diff suppressed because it is too large
Load Diff
+23562
File diff suppressed because it is too large
Load Diff
+228
@@ -0,0 +1,228 @@
|
||||
// Code generated by command: go run gen.go -out ../encodeblock_amd64.s -stubs ../encodeblock_asm.go -pkg=s2. DO NOT EDIT.
|
||||
|
||||
//go:build (amd64 || arm64) && !appengine && !noasm && gc && !noasm
|
||||
|
||||
package s2
|
||||
|
||||
func _dummy_()
|
||||
|
||||
// encodeBlockAsm encodes a non-empty src to a guaranteed-large-enough dst.
|
||||
// Maximum input 4294967295 bytes.
|
||||
// It assumes that the varint-encoded length of the decompressed bytes has already been written.
|
||||
//
|
||||
//go:noescape
|
||||
func encodeBlockAsm(dst []byte, src []byte, tmp *[65536]byte) int
|
||||
|
||||
// encodeBlockAsm4MB encodes a non-empty src to a guaranteed-large-enough dst.
|
||||
// Maximum input 4194304 bytes.
|
||||
// It assumes that the varint-encoded length of the decompressed bytes has already been written.
|
||||
//
|
||||
//go:noescape
|
||||
func encodeBlockAsm4MB(dst []byte, src []byte, tmp *[65536]byte) int
|
||||
|
||||
// encodeBlockAsm12B encodes a non-empty src to a guaranteed-large-enough dst.
|
||||
// Maximum input 16383 bytes.
|
||||
// It assumes that the varint-encoded length of the decompressed bytes has already been written.
|
||||
//
|
||||
//go:noescape
|
||||
func encodeBlockAsm12B(dst []byte, src []byte, tmp *[16384]byte) int
|
||||
|
||||
// encodeBlockAsm10B encodes a non-empty src to a guaranteed-large-enough dst.
|
||||
// Maximum input 4095 bytes.
|
||||
// It assumes that the varint-encoded length of the decompressed bytes has already been written.
|
||||
//
|
||||
//go:noescape
|
||||
func encodeBlockAsm10B(dst []byte, src []byte, tmp *[4096]byte) int
|
||||
|
||||
// encodeBlockAsm8B encodes a non-empty src to a guaranteed-large-enough dst.
|
||||
// Maximum input 511 bytes.
|
||||
// It assumes that the varint-encoded length of the decompressed bytes has already been written.
|
||||
//
|
||||
//go:noescape
|
||||
func encodeBlockAsm8B(dst []byte, src []byte, tmp *[1024]byte) int
|
||||
|
||||
// encodeBetterBlockAsm encodes a non-empty src to a guaranteed-large-enough dst.
|
||||
// Maximum input 4294967295 bytes.
|
||||
// It assumes that the varint-encoded length of the decompressed bytes has already been written.
|
||||
//
|
||||
//go:noescape
|
||||
func encodeBetterBlockAsm(dst []byte, src []byte, tmp *[589824]byte) int
|
||||
|
||||
// encodeBetterBlockAsm4MB encodes a non-empty src to a guaranteed-large-enough dst.
|
||||
// Maximum input 4194304 bytes.
|
||||
// It assumes that the varint-encoded length of the decompressed bytes has already been written.
|
||||
//
|
||||
//go:noescape
|
||||
func encodeBetterBlockAsm4MB(dst []byte, src []byte, tmp *[589824]byte) int
|
||||
|
||||
// encodeBetterBlockAsm12B encodes a non-empty src to a guaranteed-large-enough dst.
|
||||
// Maximum input 16383 bytes.
|
||||
// It assumes that the varint-encoded length of the decompressed bytes has already been written.
|
||||
//
|
||||
//go:noescape
|
||||
func encodeBetterBlockAsm12B(dst []byte, src []byte, tmp *[81920]byte) int
|
||||
|
||||
// encodeBetterBlockAsm10B encodes a non-empty src to a guaranteed-large-enough dst.
|
||||
// Maximum input 4095 bytes.
|
||||
// It assumes that the varint-encoded length of the decompressed bytes has already been written.
|
||||
//
|
||||
//go:noescape
|
||||
func encodeBetterBlockAsm10B(dst []byte, src []byte, tmp *[20480]byte) int
|
||||
|
||||
// encodeBetterBlockAsm8B encodes a non-empty src to a guaranteed-large-enough dst.
|
||||
// Maximum input 511 bytes.
|
||||
// It assumes that the varint-encoded length of the decompressed bytes has already been written.
|
||||
//
|
||||
//go:noescape
|
||||
func encodeBetterBlockAsm8B(dst []byte, src []byte, tmp *[5120]byte) int
|
||||
|
||||
// encodeSnappyBlockAsm encodes a non-empty src to a guaranteed-large-enough dst.
|
||||
// Maximum input 4294967295 bytes.
|
||||
// It assumes that the varint-encoded length of the decompressed bytes has already been written.
|
||||
//
|
||||
//go:noescape
|
||||
func encodeSnappyBlockAsm(dst []byte, src []byte, tmp *[65536]byte) int
|
||||
|
||||
// encodeSnappyBlockAsm64K encodes a non-empty src to a guaranteed-large-enough dst.
|
||||
// Maximum input 65535 bytes.
|
||||
// It assumes that the varint-encoded length of the decompressed bytes has already been written.
|
||||
//
|
||||
//go:noescape
|
||||
func encodeSnappyBlockAsm64K(dst []byte, src []byte, tmp *[65536]byte) int
|
||||
|
||||
// encodeSnappyBlockAsm12B encodes a non-empty src to a guaranteed-large-enough dst.
|
||||
// Maximum input 16383 bytes.
|
||||
// It assumes that the varint-encoded length of the decompressed bytes has already been written.
|
||||
//
|
||||
//go:noescape
|
||||
func encodeSnappyBlockAsm12B(dst []byte, src []byte, tmp *[16384]byte) int
|
||||
|
||||
// encodeSnappyBlockAsm10B encodes a non-empty src to a guaranteed-large-enough dst.
|
||||
// Maximum input 4095 bytes.
|
||||
// It assumes that the varint-encoded length of the decompressed bytes has already been written.
|
||||
//
|
||||
//go:noescape
|
||||
func encodeSnappyBlockAsm10B(dst []byte, src []byte, tmp *[4096]byte) int
|
||||
|
||||
// encodeSnappyBlockAsm8B encodes a non-empty src to a guaranteed-large-enough dst.
|
||||
// Maximum input 511 bytes.
|
||||
// It assumes that the varint-encoded length of the decompressed bytes has already been written.
|
||||
//
|
||||
//go:noescape
|
||||
func encodeSnappyBlockAsm8B(dst []byte, src []byte, tmp *[1024]byte) int
|
||||
|
||||
// encodeSnappyBetterBlockAsm encodes a non-empty src to a guaranteed-large-enough dst.
|
||||
// Maximum input 4294967295 bytes.
|
||||
// It assumes that the varint-encoded length of the decompressed bytes has already been written.
|
||||
//
|
||||
//go:noescape
|
||||
func encodeSnappyBetterBlockAsm(dst []byte, src []byte, tmp *[589824]byte) int
|
||||
|
||||
// encodeSnappyBetterBlockAsm64K encodes a non-empty src to a guaranteed-large-enough dst.
|
||||
// Maximum input 65535 bytes.
|
||||
// It assumes that the varint-encoded length of the decompressed bytes has already been written.
|
||||
//
|
||||
//go:noescape
|
||||
func encodeSnappyBetterBlockAsm64K(dst []byte, src []byte, tmp *[294912]byte) int
|
||||
|
||||
// encodeSnappyBetterBlockAsm12B encodes a non-empty src to a guaranteed-large-enough dst.
|
||||
// Maximum input 16383 bytes.
|
||||
// It assumes that the varint-encoded length of the decompressed bytes has already been written.
|
||||
//
|
||||
//go:noescape
|
||||
func encodeSnappyBetterBlockAsm12B(dst []byte, src []byte, tmp *[81920]byte) int
|
||||
|
||||
// encodeSnappyBetterBlockAsm10B encodes a non-empty src to a guaranteed-large-enough dst.
|
||||
// Maximum input 4095 bytes.
|
||||
// It assumes that the varint-encoded length of the decompressed bytes has already been written.
|
||||
//
|
||||
//go:noescape
|
||||
func encodeSnappyBetterBlockAsm10B(dst []byte, src []byte, tmp *[20480]byte) int
|
||||
|
||||
// encodeSnappyBetterBlockAsm8B encodes a non-empty src to a guaranteed-large-enough dst.
|
||||
// Maximum input 511 bytes.
|
||||
// It assumes that the varint-encoded length of the decompressed bytes has already been written.
|
||||
//
|
||||
//go:noescape
|
||||
func encodeSnappyBetterBlockAsm8B(dst []byte, src []byte, tmp *[5120]byte) int
|
||||
|
||||
// calcBlockSize encodes a non-empty src to a guaranteed-large-enough dst.
|
||||
// Maximum input 4294967295 bytes.
|
||||
// It assumes that the varint-encoded length of the decompressed bytes has already been written.
|
||||
//
|
||||
//go:noescape
|
||||
func calcBlockSize(src []byte, tmp *[32768]byte) int
|
||||
|
||||
// calcBlockSizeSmall encodes a non-empty src to a guaranteed-large-enough dst.
|
||||
// Maximum input 1024 bytes.
|
||||
// It assumes that the varint-encoded length of the decompressed bytes has already been written.
|
||||
//
|
||||
//go:noescape
|
||||
func calcBlockSizeSmall(src []byte, tmp *[2048]byte) int
|
||||
|
||||
// emitLiteral writes a literal chunk and returns the number of bytes written.
|
||||
//
|
||||
// It assumes that:
|
||||
//
|
||||
// dst is long enough to hold the encoded bytes with margin of 0 bytes
|
||||
// 0 <= len(lit) && len(lit) <= math.MaxUint32
|
||||
//
|
||||
//go:noescape
|
||||
func emitLiteral(dst []byte, lit []byte) int
|
||||
|
||||
// emitRepeat writes a repeat chunk and returns the number of bytes written.
|
||||
// Length must be at least 4 and < 1<<32
|
||||
//
|
||||
//go:noescape
|
||||
func emitRepeat(dst []byte, offset int, length int) int
|
||||
|
||||
// emitCopy writes a copy chunk and returns the number of bytes written.
|
||||
//
|
||||
// It assumes that:
|
||||
//
|
||||
// dst is long enough to hold the encoded bytes
|
||||
// 1 <= offset && offset <= math.MaxUint32
|
||||
// 4 <= length && length <= 1 << 24
|
||||
//
|
||||
//go:noescape
|
||||
func emitCopy(dst []byte, offset int, length int) int
|
||||
|
||||
// emitCopyNoRepeat writes a copy chunk and returns the number of bytes written.
|
||||
//
|
||||
// It assumes that:
|
||||
//
|
||||
// dst is long enough to hold the encoded bytes
|
||||
// 1 <= offset && offset <= math.MaxUint32
|
||||
// 4 <= length && length <= 1 << 24
|
||||
//
|
||||
//go:noescape
|
||||
func emitCopyNoRepeat(dst []byte, offset int, length int) int
|
||||
|
||||
// matchLen returns how many bytes match in a and b
|
||||
//
|
||||
// It assumes that:
|
||||
//
|
||||
// len(a) <= len(b)
|
||||
//
|
||||
//go:noescape
|
||||
func matchLen(a []byte, b []byte) int
|
||||
|
||||
// cvtLZ4Block converts an LZ4 block to S2
|
||||
//
|
||||
//go:noescape
|
||||
func cvtLZ4BlockAsm(dst []byte, src []byte) (uncompressed int, dstUsed int)
|
||||
|
||||
// cvtLZ4sBlock converts an LZ4s block to S2
|
||||
//
|
||||
//go:noescape
|
||||
func cvtLZ4sBlockAsm(dst []byte, src []byte) (uncompressed int, dstUsed int)
|
||||
|
||||
// cvtLZ4Block converts an LZ4 block to Snappy
|
||||
//
|
||||
//go:noescape
|
||||
func cvtLZ4BlockSnappyAsm(dst []byte, src []byte) (uncompressed int, dstUsed int)
|
||||
|
||||
// cvtLZ4sBlock converts an LZ4s block to Snappy
|
||||
//
|
||||
//go:noescape
|
||||
func cvtLZ4sBlockSnappyAsm(dst []byte, src []byte) (uncompressed int, dstUsed int)
|
||||
+65
@@ -0,0 +1,65 @@
|
||||
package s2
|
||||
|
||||
import "sync"
|
||||
|
||||
// Table size constants
|
||||
const (
|
||||
betterLongTableBits = 17
|
||||
betterLongTableSize = 1 << betterLongTableBits // 131072
|
||||
|
||||
betterShortTableBits = 14
|
||||
betterShortTableSize = 1 << betterShortTableBits // 16384
|
||||
|
||||
betterSnappyLongTableBits = 16
|
||||
betterSnappyLongTableSize = 1 << betterSnappyLongTableBits // 65536
|
||||
|
||||
bestLongTableBits = 19
|
||||
bestLongTableSize = 1 << bestLongTableBits // 524288
|
||||
|
||||
bestShortTableBits = 16
|
||||
bestShortTableSize = 1 << bestShortTableBits // 65536
|
||||
)
|
||||
|
||||
type betterTables struct {
|
||||
lTable [betterLongTableSize]uint32
|
||||
sTable [betterShortTableSize]uint32
|
||||
}
|
||||
|
||||
var betterTablePool = sync.Pool{New: func() any { return &betterTables{} }}
|
||||
|
||||
// betterSnappyTables holds better-snappy compression hash tables.
|
||||
type betterSnappyTables struct {
|
||||
lTable [betterSnappyLongTableSize]uint32
|
||||
sTable [betterShortTableSize]uint32
|
||||
}
|
||||
|
||||
var betterSnappyTablePool = sync.Pool{New: func() any { return &betterSnappyTables{} }}
|
||||
|
||||
// bestTables holds best compression hash tables.
|
||||
type bestTables struct {
|
||||
lTable [bestLongTableSize]uint64
|
||||
sTable [bestShortTableSize]uint64
|
||||
}
|
||||
|
||||
var bestTablePool = sync.Pool{New: func() any { return &bestTables{} }}
|
||||
|
||||
// getBetterTables gets a zeroed betterTables from the pool.
|
||||
func getBetterTables() *betterTables {
|
||||
t := betterTablePool.Get().(*betterTables)
|
||||
*t = betterTables{}
|
||||
return t
|
||||
}
|
||||
|
||||
// getBetterSnappyTables gets a zeroed betterSnappyTables from the pool.
|
||||
func getBetterSnappyTables() *betterSnappyTables {
|
||||
t := betterSnappyTablePool.Get().(*betterSnappyTables)
|
||||
*t = betterSnappyTables{}
|
||||
return t
|
||||
}
|
||||
|
||||
// getBestTables gets a zeroed bestTables from the pool.
|
||||
func getBestTables() *bestTables {
|
||||
t := bestTablePool.Get().(*bestTables)
|
||||
*t = bestTables{}
|
||||
return t
|
||||
}
|
||||
+602
@@ -0,0 +1,602 @@
|
||||
// Copyright (c) 2022+ Klaus Post. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package s2
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/binary"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"sort"
|
||||
)
|
||||
|
||||
const (
|
||||
S2IndexHeader = "s2idx\x00"
|
||||
S2IndexTrailer = "\x00xdi2s"
|
||||
maxIndexEntries = 1 << 16
|
||||
// If distance is less than this, we do not add the entry.
|
||||
minIndexDist = 1 << 20
|
||||
)
|
||||
|
||||
// Index represents an S2/Snappy index.
|
||||
type Index struct {
|
||||
TotalUncompressed int64 // Total Uncompressed size if known. Will be -1 if unknown.
|
||||
TotalCompressed int64 // Total Compressed size if known. Will be -1 if unknown.
|
||||
info []struct {
|
||||
compressedOffset int64
|
||||
uncompressedOffset int64
|
||||
}
|
||||
estBlockUncomp int64
|
||||
}
|
||||
|
||||
func (i *Index) reset(maxBlock int) {
|
||||
i.estBlockUncomp = int64(maxBlock)
|
||||
i.TotalCompressed = -1
|
||||
i.TotalUncompressed = -1
|
||||
if len(i.info) > 0 {
|
||||
i.info = i.info[:0]
|
||||
}
|
||||
}
|
||||
|
||||
// allocInfos will allocate an empty slice of infos.
|
||||
func (i *Index) allocInfos(n int) {
|
||||
if n > maxIndexEntries {
|
||||
panic("n > maxIndexEntries")
|
||||
}
|
||||
i.info = make([]struct {
|
||||
compressedOffset int64
|
||||
uncompressedOffset int64
|
||||
}, 0, n)
|
||||
}
|
||||
|
||||
// add an uncompressed and compressed pair.
|
||||
// Entries must be sent in order.
|
||||
func (i *Index) add(compressedOffset, uncompressedOffset int64) error {
|
||||
if i == nil {
|
||||
return nil
|
||||
}
|
||||
lastIdx := len(i.info) - 1
|
||||
if lastIdx >= 0 {
|
||||
latest := i.info[lastIdx]
|
||||
if latest.uncompressedOffset == uncompressedOffset {
|
||||
// Uncompressed didn't change, don't add entry,
|
||||
// but update start index.
|
||||
latest.compressedOffset = compressedOffset
|
||||
i.info[lastIdx] = latest
|
||||
return nil
|
||||
}
|
||||
if latest.uncompressedOffset > uncompressedOffset {
|
||||
return fmt.Errorf("internal error: Earlier uncompressed received (%d > %d)", latest.uncompressedOffset, uncompressedOffset)
|
||||
}
|
||||
if latest.compressedOffset > compressedOffset {
|
||||
return fmt.Errorf("internal error: Earlier compressed received (%d > %d)", latest.compressedOffset, compressedOffset)
|
||||
}
|
||||
if latest.uncompressedOffset+minIndexDist > uncompressedOffset {
|
||||
// Only add entry if distance is large enough.
|
||||
return nil
|
||||
}
|
||||
}
|
||||
i.info = append(i.info, struct {
|
||||
compressedOffset int64
|
||||
uncompressedOffset int64
|
||||
}{compressedOffset: compressedOffset, uncompressedOffset: uncompressedOffset})
|
||||
return nil
|
||||
}
|
||||
|
||||
// Find the offset at or before the wanted (uncompressed) offset.
|
||||
// If offset is 0 or positive it is the offset from the beginning of the file.
|
||||
// If the uncompressed size is known, the offset must be within the file.
|
||||
// If an offset outside the file is requested io.ErrUnexpectedEOF is returned.
|
||||
// If the offset is negative, it is interpreted as the distance from the end of the file,
|
||||
// where -1 represents the last byte.
|
||||
// If offset from the end of the file is requested, but size is unknown,
|
||||
// ErrUnsupported will be returned.
|
||||
func (i *Index) Find(offset int64) (compressedOff, uncompressedOff int64, err error) {
|
||||
if i.TotalUncompressed < 0 {
|
||||
return 0, 0, ErrCorrupt
|
||||
}
|
||||
if offset < 0 {
|
||||
offset = i.TotalUncompressed + offset
|
||||
if offset < 0 {
|
||||
return 0, 0, io.ErrUnexpectedEOF
|
||||
}
|
||||
}
|
||||
if offset > i.TotalUncompressed {
|
||||
return 0, 0, io.ErrUnexpectedEOF
|
||||
}
|
||||
if len(i.info) > 200 {
|
||||
n := sort.Search(len(i.info), func(n int) bool {
|
||||
return i.info[n].uncompressedOffset > offset
|
||||
})
|
||||
if n == 0 {
|
||||
n = 1
|
||||
}
|
||||
return i.info[n-1].compressedOffset, i.info[n-1].uncompressedOffset, nil
|
||||
}
|
||||
for _, info := range i.info {
|
||||
if info.uncompressedOffset > offset {
|
||||
break
|
||||
}
|
||||
compressedOff = info.compressedOffset
|
||||
uncompressedOff = info.uncompressedOffset
|
||||
}
|
||||
return compressedOff, uncompressedOff, nil
|
||||
}
|
||||
|
||||
// reduce to stay below maxIndexEntries
|
||||
func (i *Index) reduce() {
|
||||
if len(i.info) < maxIndexEntries && i.estBlockUncomp >= minIndexDist {
|
||||
return
|
||||
}
|
||||
|
||||
// Algorithm, keep 1, remove removeN entries...
|
||||
removeN := (len(i.info) + 1) / maxIndexEntries
|
||||
src := i.info
|
||||
j := 0
|
||||
|
||||
// Each block should be at least 1MB, but don't reduce below 1000 entries.
|
||||
for i.estBlockUncomp*(int64(removeN)+1) < minIndexDist && len(i.info)/(removeN+1) > 1000 {
|
||||
removeN++
|
||||
}
|
||||
for idx := 0; idx < len(src); idx++ {
|
||||
i.info[j] = src[idx]
|
||||
j++
|
||||
idx += removeN
|
||||
}
|
||||
i.info = i.info[:j]
|
||||
// Update maxblock estimate.
|
||||
i.estBlockUncomp += i.estBlockUncomp * int64(removeN)
|
||||
}
|
||||
|
||||
func (i *Index) appendTo(b []byte, uncompTotal, compTotal int64) []byte {
|
||||
i.reduce()
|
||||
var tmp [binary.MaxVarintLen64]byte
|
||||
|
||||
initSize := len(b)
|
||||
// We make the start a skippable header+size.
|
||||
b = append(b, ChunkTypeIndex, 0, 0, 0)
|
||||
b = append(b, []byte(S2IndexHeader)...)
|
||||
// Total Uncompressed size
|
||||
n := binary.PutVarint(tmp[:], uncompTotal)
|
||||
b = append(b, tmp[:n]...)
|
||||
// Total Compressed size
|
||||
n = binary.PutVarint(tmp[:], compTotal)
|
||||
b = append(b, tmp[:n]...)
|
||||
// Put EstBlockUncomp size
|
||||
n = binary.PutVarint(tmp[:], i.estBlockUncomp)
|
||||
b = append(b, tmp[:n]...)
|
||||
// Put length
|
||||
n = binary.PutVarint(tmp[:], int64(len(i.info)))
|
||||
b = append(b, tmp[:n]...)
|
||||
|
||||
// Check if we should add uncompressed offsets
|
||||
var hasUncompressed byte
|
||||
for idx, info := range i.info {
|
||||
if idx == 0 {
|
||||
if info.uncompressedOffset != 0 {
|
||||
hasUncompressed = 1
|
||||
break
|
||||
}
|
||||
continue
|
||||
}
|
||||
if info.uncompressedOffset != i.info[idx-1].uncompressedOffset+i.estBlockUncomp {
|
||||
hasUncompressed = 1
|
||||
break
|
||||
}
|
||||
}
|
||||
b = append(b, hasUncompressed)
|
||||
|
||||
// Add each entry
|
||||
if hasUncompressed == 1 {
|
||||
for idx, info := range i.info {
|
||||
uOff := info.uncompressedOffset
|
||||
if idx > 0 {
|
||||
prev := i.info[idx-1]
|
||||
uOff -= prev.uncompressedOffset + (i.estBlockUncomp)
|
||||
}
|
||||
n = binary.PutVarint(tmp[:], uOff)
|
||||
b = append(b, tmp[:n]...)
|
||||
}
|
||||
}
|
||||
|
||||
// Initial compressed size estimate.
|
||||
cPredict := i.estBlockUncomp / 2
|
||||
|
||||
for idx, info := range i.info {
|
||||
cOff := info.compressedOffset
|
||||
if idx > 0 {
|
||||
prev := i.info[idx-1]
|
||||
cOff -= prev.compressedOffset + cPredict
|
||||
// Update compressed size prediction, with half the error.
|
||||
cPredict += cOff / 2
|
||||
}
|
||||
n = binary.PutVarint(tmp[:], cOff)
|
||||
b = append(b, tmp[:n]...)
|
||||
}
|
||||
|
||||
// Add Total Size.
|
||||
// Stored as fixed size for easier reading.
|
||||
binary.LittleEndian.PutUint32(tmp[:], uint32(len(b)-initSize+4+len(S2IndexTrailer)))
|
||||
b = append(b, tmp[:4]...)
|
||||
// Trailer
|
||||
b = append(b, []byte(S2IndexTrailer)...)
|
||||
|
||||
// Update size
|
||||
chunkLen := len(b) - initSize - skippableFrameHeader
|
||||
b[initSize+1] = uint8(chunkLen >> 0)
|
||||
b[initSize+2] = uint8(chunkLen >> 8)
|
||||
b[initSize+3] = uint8(chunkLen >> 16)
|
||||
//fmt.Printf("chunklen: 0x%x Uncomp:%d, Comp:%d\n", chunkLen, uncompTotal, compTotal)
|
||||
return b
|
||||
}
|
||||
|
||||
// Load a binary index.
|
||||
// A zero value Index can be used or a previous one can be reused.
|
||||
func (i *Index) Load(b []byte) ([]byte, error) {
|
||||
if len(b) <= 4+len(S2IndexHeader)+len(S2IndexTrailer) {
|
||||
return b, io.ErrUnexpectedEOF
|
||||
}
|
||||
if b[0] != ChunkTypeIndex {
|
||||
return b, ErrCorrupt
|
||||
}
|
||||
chunkLen := int(b[1]) | int(b[2])<<8 | int(b[3])<<16
|
||||
b = b[4:]
|
||||
|
||||
// Validate we have enough...
|
||||
if len(b) < chunkLen {
|
||||
return b, io.ErrUnexpectedEOF
|
||||
}
|
||||
if !bytes.Equal(b[:len(S2IndexHeader)], []byte(S2IndexHeader)) {
|
||||
return b, ErrUnsupported
|
||||
}
|
||||
b = b[len(S2IndexHeader):]
|
||||
|
||||
// Total Uncompressed
|
||||
if v, n := binary.Varint(b); n <= 0 || v < 0 {
|
||||
return b, ErrCorrupt
|
||||
} else {
|
||||
i.TotalUncompressed = v
|
||||
b = b[n:]
|
||||
}
|
||||
|
||||
// Total Compressed
|
||||
if v, n := binary.Varint(b); n <= 0 {
|
||||
return b, ErrCorrupt
|
||||
} else {
|
||||
i.TotalCompressed = v
|
||||
b = b[n:]
|
||||
}
|
||||
|
||||
// Read EstBlockUncomp
|
||||
if v, n := binary.Varint(b); n <= 0 {
|
||||
return b, ErrCorrupt
|
||||
} else {
|
||||
if v < 0 {
|
||||
return b, ErrCorrupt
|
||||
}
|
||||
i.estBlockUncomp = v
|
||||
b = b[n:]
|
||||
}
|
||||
|
||||
var entries int
|
||||
if v, n := binary.Varint(b); n <= 0 {
|
||||
return b, ErrCorrupt
|
||||
} else {
|
||||
if v < 0 || v > maxIndexEntries {
|
||||
return b, ErrCorrupt
|
||||
}
|
||||
entries = int(v)
|
||||
b = b[n:]
|
||||
}
|
||||
if cap(i.info) < entries {
|
||||
i.allocInfos(entries)
|
||||
}
|
||||
i.info = i.info[:entries]
|
||||
|
||||
if len(b) < 1 {
|
||||
return b, io.ErrUnexpectedEOF
|
||||
}
|
||||
hasUncompressed := b[0]
|
||||
b = b[1:]
|
||||
if hasUncompressed&1 != hasUncompressed {
|
||||
return b, ErrCorrupt
|
||||
}
|
||||
|
||||
// Add each uncompressed entry
|
||||
for idx := range i.info {
|
||||
var uOff int64
|
||||
if hasUncompressed != 0 {
|
||||
// Load delta
|
||||
if v, n := binary.Varint(b); n <= 0 {
|
||||
return b, ErrCorrupt
|
||||
} else {
|
||||
uOff = v
|
||||
b = b[n:]
|
||||
}
|
||||
}
|
||||
|
||||
if idx > 0 {
|
||||
prev := i.info[idx-1].uncompressedOffset
|
||||
uOff += prev + (i.estBlockUncomp)
|
||||
if uOff <= prev {
|
||||
return b, ErrCorrupt
|
||||
}
|
||||
}
|
||||
if uOff < 0 {
|
||||
return b, ErrCorrupt
|
||||
}
|
||||
i.info[idx].uncompressedOffset = uOff
|
||||
}
|
||||
|
||||
// Initial compressed size estimate.
|
||||
cPredict := i.estBlockUncomp / 2
|
||||
|
||||
// Add each compressed entry
|
||||
for idx := range i.info {
|
||||
var cOff int64
|
||||
if v, n := binary.Varint(b); n <= 0 {
|
||||
return b, ErrCorrupt
|
||||
} else {
|
||||
cOff = v
|
||||
b = b[n:]
|
||||
}
|
||||
|
||||
if idx > 0 {
|
||||
// Update compressed size prediction, with half the error.
|
||||
cPredictNew := cPredict + cOff/2
|
||||
|
||||
prev := i.info[idx-1].compressedOffset
|
||||
cOff += prev + cPredict
|
||||
if cOff <= prev {
|
||||
return b, ErrCorrupt
|
||||
}
|
||||
cPredict = cPredictNew
|
||||
}
|
||||
if cOff < 0 {
|
||||
return b, ErrCorrupt
|
||||
}
|
||||
i.info[idx].compressedOffset = cOff
|
||||
}
|
||||
if len(b) < 4+len(S2IndexTrailer) {
|
||||
return b, io.ErrUnexpectedEOF
|
||||
}
|
||||
// Skip size...
|
||||
b = b[4:]
|
||||
|
||||
// Check trailer...
|
||||
if !bytes.Equal(b[:len(S2IndexTrailer)], []byte(S2IndexTrailer)) {
|
||||
return b, ErrCorrupt
|
||||
}
|
||||
return b[len(S2IndexTrailer):], nil
|
||||
}
|
||||
|
||||
// LoadStream will load an index from the end of the supplied stream.
|
||||
// ErrUnsupported will be returned if the signature cannot be found.
|
||||
// ErrCorrupt will be returned if unexpected values are found.
|
||||
// io.ErrUnexpectedEOF is returned if there are too few bytes.
|
||||
// IO errors are returned as-is.
|
||||
func (i *Index) LoadStream(rs io.ReadSeeker) error {
|
||||
// Go to end.
|
||||
_, err := rs.Seek(-10, io.SeekEnd)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var tmp [10]byte
|
||||
_, err = io.ReadFull(rs, tmp[:])
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// Check trailer...
|
||||
if !bytes.Equal(tmp[4:4+len(S2IndexTrailer)], []byte(S2IndexTrailer)) {
|
||||
return ErrUnsupported
|
||||
}
|
||||
sz := binary.LittleEndian.Uint32(tmp[:4])
|
||||
if sz > maxChunkSize+skippableFrameHeader {
|
||||
return ErrCorrupt
|
||||
}
|
||||
_, err = rs.Seek(-int64(sz), io.SeekEnd)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Read index.
|
||||
buf := make([]byte, sz)
|
||||
_, err = io.ReadFull(rs, buf)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = i.Load(buf)
|
||||
return err
|
||||
}
|
||||
|
||||
// IndexStream will return an index for a stream.
|
||||
// The stream structure will be checked, but
|
||||
// data within blocks is not verified.
|
||||
// The returned index can either be appended to the end of the stream
|
||||
// or stored separately.
|
||||
func IndexStream(r io.Reader) ([]byte, error) {
|
||||
var i Index
|
||||
var buf [maxChunkSize]byte
|
||||
var readHeader bool
|
||||
for {
|
||||
_, err := io.ReadFull(r, buf[:4])
|
||||
if err != nil {
|
||||
if err == io.EOF {
|
||||
return i.appendTo(nil, i.TotalUncompressed, i.TotalCompressed), nil
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
// Start of this chunk.
|
||||
startChunk := i.TotalCompressed
|
||||
i.TotalCompressed += 4
|
||||
|
||||
chunkType := buf[0]
|
||||
if !readHeader {
|
||||
if chunkType != chunkTypeStreamIdentifier {
|
||||
return nil, ErrCorrupt
|
||||
}
|
||||
readHeader = true
|
||||
}
|
||||
chunkLen := int(buf[1]) | int(buf[2])<<8 | int(buf[3])<<16
|
||||
if chunkLen < checksumSize {
|
||||
return nil, ErrCorrupt
|
||||
}
|
||||
|
||||
i.TotalCompressed += int64(chunkLen)
|
||||
_, err = io.ReadFull(r, buf[:chunkLen])
|
||||
if err != nil {
|
||||
return nil, io.ErrUnexpectedEOF
|
||||
}
|
||||
// The chunk types are specified at
|
||||
// https://github.com/google/snappy/blob/master/framing_format.txt
|
||||
switch chunkType {
|
||||
case chunkTypeCompressedData:
|
||||
// Section 4.2. Compressed data (chunk type 0x00).
|
||||
// Skip checksum.
|
||||
dLen, err := DecodedLen(buf[checksumSize:])
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if dLen > maxBlockSize {
|
||||
return nil, ErrCorrupt
|
||||
}
|
||||
if i.estBlockUncomp == 0 {
|
||||
// Use first block for estimate...
|
||||
i.estBlockUncomp = int64(dLen)
|
||||
}
|
||||
err = i.add(startChunk, i.TotalUncompressed)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
i.TotalUncompressed += int64(dLen)
|
||||
continue
|
||||
case chunkTypeUncompressedData:
|
||||
n2 := chunkLen - checksumSize
|
||||
if n2 > maxBlockSize {
|
||||
return nil, ErrCorrupt
|
||||
}
|
||||
if i.estBlockUncomp == 0 {
|
||||
// Use first block for estimate...
|
||||
i.estBlockUncomp = int64(n2)
|
||||
}
|
||||
err = i.add(startChunk, i.TotalUncompressed)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
i.TotalUncompressed += int64(n2)
|
||||
continue
|
||||
case chunkTypeStreamIdentifier:
|
||||
// Section 4.1. Stream identifier (chunk type 0xff).
|
||||
if chunkLen != len(magicBody) {
|
||||
return nil, ErrCorrupt
|
||||
}
|
||||
|
||||
if string(buf[:len(magicBody)]) != magicBody {
|
||||
if string(buf[:len(magicBody)]) != magicBodySnappy {
|
||||
return nil, ErrCorrupt
|
||||
}
|
||||
}
|
||||
|
||||
continue
|
||||
}
|
||||
|
||||
if chunkType <= 0x7f {
|
||||
// Section 4.5. Reserved unskippable chunks (chunk types 0x02-0x7f).
|
||||
return nil, ErrUnsupported
|
||||
}
|
||||
if chunkLen > maxChunkSize {
|
||||
return nil, ErrUnsupported
|
||||
}
|
||||
// Section 4.4 Padding (chunk type 0xfe).
|
||||
// Section 4.6. Reserved skippable chunks (chunk types 0x80-0xfd).
|
||||
}
|
||||
}
|
||||
|
||||
// JSON returns the index as JSON text.
|
||||
func (i *Index) JSON() []byte {
|
||||
type offset struct {
|
||||
CompressedOffset int64 `json:"compressed"`
|
||||
UncompressedOffset int64 `json:"uncompressed"`
|
||||
}
|
||||
x := struct {
|
||||
TotalUncompressed int64 `json:"total_uncompressed"` // Total Uncompressed size if known. Will be -1 if unknown.
|
||||
TotalCompressed int64 `json:"total_compressed"` // Total Compressed size if known. Will be -1 if unknown.
|
||||
Offsets []offset `json:"offsets"`
|
||||
EstBlockUncomp int64 `json:"est_block_uncompressed"`
|
||||
}{
|
||||
TotalUncompressed: i.TotalUncompressed,
|
||||
TotalCompressed: i.TotalCompressed,
|
||||
EstBlockUncomp: i.estBlockUncomp,
|
||||
}
|
||||
for _, v := range i.info {
|
||||
x.Offsets = append(x.Offsets, offset{CompressedOffset: v.compressedOffset, UncompressedOffset: v.uncompressedOffset})
|
||||
}
|
||||
b, _ := json.MarshalIndent(x, "", " ")
|
||||
return b
|
||||
}
|
||||
|
||||
// RemoveIndexHeaders will trim all headers and trailers from a given index.
|
||||
// This is expected to save 20 bytes.
|
||||
// These can be restored using RestoreIndexHeaders.
|
||||
// This removes a layer of security, but is the most compact representation.
|
||||
// Returns nil if headers contains errors.
|
||||
// The returned slice references the provided slice.
|
||||
func RemoveIndexHeaders(b []byte) []byte {
|
||||
const save = 4 + len(S2IndexHeader) + len(S2IndexTrailer) + 4
|
||||
if len(b) <= save {
|
||||
return nil
|
||||
}
|
||||
if b[0] != ChunkTypeIndex {
|
||||
return nil
|
||||
}
|
||||
chunkLen := int(b[1]) | int(b[2])<<8 | int(b[3])<<16
|
||||
b = b[4:]
|
||||
|
||||
// Validate we have enough...
|
||||
if len(b) < chunkLen {
|
||||
return nil
|
||||
}
|
||||
b = b[:chunkLen]
|
||||
|
||||
if !bytes.Equal(b[:len(S2IndexHeader)], []byte(S2IndexHeader)) {
|
||||
return nil
|
||||
}
|
||||
b = b[len(S2IndexHeader):]
|
||||
if !bytes.HasSuffix(b, []byte(S2IndexTrailer)) {
|
||||
return nil
|
||||
}
|
||||
b = bytes.TrimSuffix(b, []byte(S2IndexTrailer))
|
||||
|
||||
if len(b) < 4 {
|
||||
return nil
|
||||
}
|
||||
return b[:len(b)-4]
|
||||
}
|
||||
|
||||
// RestoreIndexHeaders will index restore headers removed by RemoveIndexHeaders.
|
||||
// No error checking is performed on the input.
|
||||
// If a 0 length slice is sent, it is returned without modification.
|
||||
func RestoreIndexHeaders(in []byte) []byte {
|
||||
if len(in) == 0 {
|
||||
return in
|
||||
}
|
||||
b := make([]byte, 0, 4+len(S2IndexHeader)+len(in)+len(S2IndexTrailer)+4)
|
||||
b = append(b, ChunkTypeIndex, 0, 0, 0)
|
||||
b = append(b, []byte(S2IndexHeader)...)
|
||||
b = append(b, in...)
|
||||
|
||||
var tmp [4]byte
|
||||
binary.LittleEndian.PutUint32(tmp[:], uint32(len(b)+4+len(S2IndexTrailer)))
|
||||
b = append(b, tmp[:4]...)
|
||||
// Trailer
|
||||
b = append(b, []byte(S2IndexTrailer)...)
|
||||
|
||||
chunkLen := len(b) - skippableFrameHeader
|
||||
b[1] = uint8(chunkLen >> 0)
|
||||
b[2] = uint8(chunkLen >> 8)
|
||||
b[3] = uint8(chunkLen >> 16)
|
||||
return b
|
||||
}
|
||||
+585
@@ -0,0 +1,585 @@
|
||||
// Copyright (c) 2022 Klaus Post. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package s2
|
||||
|
||||
import (
|
||||
"encoding/binary"
|
||||
"errors"
|
||||
"fmt"
|
||||
)
|
||||
|
||||
// LZ4Converter provides conversion from LZ4 blocks as defined here:
|
||||
// https://github.com/lz4/lz4/blob/dev/doc/lz4_Block_format.md
|
||||
type LZ4Converter struct {
|
||||
}
|
||||
|
||||
// ErrDstTooSmall is returned when provided destination is too small.
|
||||
var ErrDstTooSmall = errors.New("s2: destination too small")
|
||||
|
||||
// ConvertBlock will convert an LZ4 block and append it as an S2
|
||||
// block without block length to dst.
|
||||
// The uncompressed size is returned as well.
|
||||
// dst must have capacity to contain the entire compressed block.
|
||||
func (l *LZ4Converter) ConvertBlock(dst, src []byte) ([]byte, int, error) {
|
||||
if len(src) == 0 {
|
||||
return dst, 0, nil
|
||||
}
|
||||
const debug = false
|
||||
const inline = true
|
||||
const lz4MinMatch = 4
|
||||
|
||||
s, d := 0, len(dst)
|
||||
dst = dst[:cap(dst)]
|
||||
if !debug && hasAsm {
|
||||
res, sz := cvtLZ4BlockAsm(dst[d:], src)
|
||||
if res < 0 {
|
||||
const (
|
||||
errCorrupt = -1
|
||||
errDstTooSmall = -2
|
||||
)
|
||||
switch res {
|
||||
case errCorrupt:
|
||||
return nil, 0, ErrCorrupt
|
||||
case errDstTooSmall:
|
||||
return nil, 0, ErrDstTooSmall
|
||||
default:
|
||||
return nil, 0, fmt.Errorf("unexpected result: %d", res)
|
||||
}
|
||||
}
|
||||
if d+sz > len(dst) {
|
||||
return nil, 0, ErrDstTooSmall
|
||||
}
|
||||
return dst[:d+sz], res, nil
|
||||
}
|
||||
|
||||
dLimit := len(dst) - 10
|
||||
var lastOffset uint16
|
||||
var uncompressed int
|
||||
if debug {
|
||||
fmt.Printf("convert block start: len(src): %d, len(dst):%d \n", len(src), len(dst))
|
||||
}
|
||||
|
||||
for {
|
||||
if s >= len(src) {
|
||||
return dst[:d], 0, ErrCorrupt
|
||||
}
|
||||
// Read literal info
|
||||
token := src[s]
|
||||
ll := int(token >> 4)
|
||||
ml := int(lz4MinMatch + (token & 0xf))
|
||||
|
||||
// If upper nibble is 15, literal length is extended
|
||||
if token >= 0xf0 {
|
||||
for {
|
||||
s++
|
||||
if s >= len(src) {
|
||||
if debug {
|
||||
fmt.Printf("error reading ll: s (%d) >= len(src) (%d)\n", s, len(src))
|
||||
}
|
||||
return dst[:d], 0, ErrCorrupt
|
||||
}
|
||||
val := src[s]
|
||||
ll += int(val)
|
||||
if val != 255 {
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
// Skip past token
|
||||
if s+ll >= len(src) {
|
||||
if debug {
|
||||
fmt.Printf("error literals: s+ll (%d+%d) >= len(src) (%d)\n", s, ll, len(src))
|
||||
}
|
||||
return nil, 0, ErrCorrupt
|
||||
}
|
||||
s++
|
||||
if ll > 0 {
|
||||
if d+ll > dLimit {
|
||||
return nil, 0, ErrDstTooSmall
|
||||
}
|
||||
if debug {
|
||||
fmt.Printf("emit %d literals\n", ll)
|
||||
}
|
||||
d += emitLiteralGo(dst[d:], src[s:s+ll])
|
||||
s += ll
|
||||
uncompressed += ll
|
||||
}
|
||||
|
||||
// Check if we are done...
|
||||
if s == len(src) && ml == lz4MinMatch {
|
||||
break
|
||||
}
|
||||
// 2 byte offset
|
||||
if s >= len(src)-2 {
|
||||
if debug {
|
||||
fmt.Printf("s (%d) >= len(src)-2 (%d)", s, len(src)-2)
|
||||
}
|
||||
return nil, 0, ErrCorrupt
|
||||
}
|
||||
offset := binary.LittleEndian.Uint16(src[s:])
|
||||
s += 2
|
||||
if offset == 0 {
|
||||
if debug {
|
||||
fmt.Printf("error: offset 0, ml: %d, len(src)-s: %d\n", ml, len(src)-s)
|
||||
}
|
||||
return nil, 0, ErrCorrupt
|
||||
}
|
||||
if int(offset) > uncompressed {
|
||||
if debug {
|
||||
fmt.Printf("error: offset (%d)> uncompressed (%d)\n", offset, uncompressed)
|
||||
}
|
||||
return nil, 0, ErrCorrupt
|
||||
}
|
||||
|
||||
if ml == lz4MinMatch+15 {
|
||||
for {
|
||||
if s >= len(src) {
|
||||
if debug {
|
||||
fmt.Printf("error reading ml: s (%d) >= len(src) (%d)\n", s, len(src))
|
||||
}
|
||||
return nil, 0, ErrCorrupt
|
||||
}
|
||||
val := src[s]
|
||||
s++
|
||||
ml += int(val)
|
||||
if val != 255 {
|
||||
if s >= len(src) {
|
||||
if debug {
|
||||
fmt.Printf("error reading ml: s (%d) >= len(src) (%d)\n", s, len(src))
|
||||
}
|
||||
return nil, 0, ErrCorrupt
|
||||
}
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
if offset == lastOffset {
|
||||
if debug {
|
||||
fmt.Printf("emit repeat, length: %d, offset: %d\n", ml, offset)
|
||||
}
|
||||
if !inline {
|
||||
d += emitRepeat16(dst[d:], offset, ml)
|
||||
} else {
|
||||
length := ml
|
||||
dst := dst[d:]
|
||||
for len(dst) > 5 {
|
||||
// Repeat offset, make length cheaper
|
||||
length -= 4
|
||||
if length <= 4 {
|
||||
dst[0] = uint8(length)<<2 | tagCopy1
|
||||
dst[1] = 0
|
||||
d += 2
|
||||
break
|
||||
}
|
||||
if length < 8 && offset < 2048 {
|
||||
// Encode WITH offset
|
||||
dst[1] = uint8(offset)
|
||||
dst[0] = uint8(offset>>8)<<5 | uint8(length)<<2 | tagCopy1
|
||||
d += 2
|
||||
break
|
||||
}
|
||||
if length < (1<<8)+4 {
|
||||
length -= 4
|
||||
dst[2] = uint8(length)
|
||||
dst[1] = 0
|
||||
dst[0] = 5<<2 | tagCopy1
|
||||
d += 3
|
||||
break
|
||||
}
|
||||
if length < (1<<16)+(1<<8) {
|
||||
length -= 1 << 8
|
||||
dst[3] = uint8(length >> 8)
|
||||
dst[2] = uint8(length >> 0)
|
||||
dst[1] = 0
|
||||
dst[0] = 6<<2 | tagCopy1
|
||||
d += 4
|
||||
break
|
||||
}
|
||||
const maxRepeat = (1 << 24) - 1
|
||||
length -= 1 << 16
|
||||
left := 0
|
||||
if length > maxRepeat {
|
||||
left = length - maxRepeat + 4
|
||||
length = maxRepeat - 4
|
||||
}
|
||||
dst[4] = uint8(length >> 16)
|
||||
dst[3] = uint8(length >> 8)
|
||||
dst[2] = uint8(length >> 0)
|
||||
dst[1] = 0
|
||||
dst[0] = 7<<2 | tagCopy1
|
||||
if left > 0 {
|
||||
d += 5 + emitRepeat16(dst[5:], offset, left)
|
||||
break
|
||||
}
|
||||
d += 5
|
||||
break
|
||||
}
|
||||
}
|
||||
} else {
|
||||
if debug {
|
||||
fmt.Printf("emit copy, length: %d, offset: %d\n", ml, offset)
|
||||
}
|
||||
if !inline {
|
||||
d += emitCopy16(dst[d:], offset, ml)
|
||||
} else {
|
||||
length := ml
|
||||
dst := dst[d:]
|
||||
for len(dst) > 5 {
|
||||
// Offset no more than 2 bytes.
|
||||
if length > 64 {
|
||||
off := 3
|
||||
if offset < 2048 {
|
||||
// emit 8 bytes as tagCopy1, rest as repeats.
|
||||
dst[1] = uint8(offset)
|
||||
dst[0] = uint8(offset>>8)<<5 | uint8(8-4)<<2 | tagCopy1
|
||||
length -= 8
|
||||
off = 2
|
||||
} else {
|
||||
// Emit a length 60 copy, encoded as 3 bytes.
|
||||
// Emit remaining as repeat value (minimum 4 bytes).
|
||||
dst[2] = uint8(offset >> 8)
|
||||
dst[1] = uint8(offset)
|
||||
dst[0] = 59<<2 | tagCopy2
|
||||
length -= 60
|
||||
}
|
||||
// Emit remaining as repeats, at least 4 bytes remain.
|
||||
d += off + emitRepeat16(dst[off:], offset, length)
|
||||
break
|
||||
}
|
||||
if length >= 12 || offset >= 2048 {
|
||||
// Emit the remaining copy, encoded as 3 bytes.
|
||||
dst[2] = uint8(offset >> 8)
|
||||
dst[1] = uint8(offset)
|
||||
dst[0] = uint8(length-1)<<2 | tagCopy2
|
||||
d += 3
|
||||
break
|
||||
}
|
||||
// Emit the remaining copy, encoded as 2 bytes.
|
||||
dst[1] = uint8(offset)
|
||||
dst[0] = uint8(offset>>8)<<5 | uint8(length-4)<<2 | tagCopy1
|
||||
d += 2
|
||||
break
|
||||
}
|
||||
}
|
||||
lastOffset = offset
|
||||
}
|
||||
uncompressed += ml
|
||||
if d > dLimit {
|
||||
return nil, 0, ErrDstTooSmall
|
||||
}
|
||||
}
|
||||
|
||||
return dst[:d], uncompressed, nil
|
||||
}
|
||||
|
||||
// ConvertBlockSnappy will convert an LZ4 block and append it
|
||||
// as a Snappy block without block length to dst.
|
||||
// The uncompressed size is returned as well.
|
||||
// dst must have capacity to contain the entire compressed block.
|
||||
func (l *LZ4Converter) ConvertBlockSnappy(dst, src []byte) ([]byte, int, error) {
|
||||
if len(src) == 0 {
|
||||
return dst, 0, nil
|
||||
}
|
||||
const debug = false
|
||||
const lz4MinMatch = 4
|
||||
|
||||
s, d := 0, len(dst)
|
||||
dst = dst[:cap(dst)]
|
||||
// Use assembly when possible
|
||||
if !debug && hasAsm {
|
||||
res, sz := cvtLZ4BlockSnappyAsm(dst[d:], src)
|
||||
if res < 0 {
|
||||
const (
|
||||
errCorrupt = -1
|
||||
errDstTooSmall = -2
|
||||
)
|
||||
switch res {
|
||||
case errCorrupt:
|
||||
return nil, 0, ErrCorrupt
|
||||
case errDstTooSmall:
|
||||
return nil, 0, ErrDstTooSmall
|
||||
default:
|
||||
return nil, 0, fmt.Errorf("unexpected result: %d", res)
|
||||
}
|
||||
}
|
||||
if d+sz > len(dst) {
|
||||
return nil, 0, ErrDstTooSmall
|
||||
}
|
||||
return dst[:d+sz], res, nil
|
||||
}
|
||||
|
||||
dLimit := len(dst) - 10
|
||||
var uncompressed int
|
||||
if debug {
|
||||
fmt.Printf("convert block start: len(src): %d, len(dst):%d \n", len(src), len(dst))
|
||||
}
|
||||
|
||||
for {
|
||||
if s >= len(src) {
|
||||
return nil, 0, ErrCorrupt
|
||||
}
|
||||
// Read literal info
|
||||
token := src[s]
|
||||
ll := int(token >> 4)
|
||||
ml := int(lz4MinMatch + (token & 0xf))
|
||||
|
||||
// If upper nibble is 15, literal length is extended
|
||||
if token >= 0xf0 {
|
||||
for {
|
||||
s++
|
||||
if s >= len(src) {
|
||||
if debug {
|
||||
fmt.Printf("error reading ll: s (%d) >= len(src) (%d)\n", s, len(src))
|
||||
}
|
||||
return nil, 0, ErrCorrupt
|
||||
}
|
||||
val := src[s]
|
||||
ll += int(val)
|
||||
if val != 255 {
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
// Skip past token
|
||||
if s+ll >= len(src) {
|
||||
if debug {
|
||||
fmt.Printf("error literals: s+ll (%d+%d) >= len(src) (%d)\n", s, ll, len(src))
|
||||
}
|
||||
return nil, 0, ErrCorrupt
|
||||
}
|
||||
s++
|
||||
if ll > 0 {
|
||||
if d+ll > dLimit {
|
||||
return nil, 0, ErrDstTooSmall
|
||||
}
|
||||
if debug {
|
||||
fmt.Printf("emit %d literals\n", ll)
|
||||
}
|
||||
d += emitLiteralGo(dst[d:], src[s:s+ll])
|
||||
s += ll
|
||||
uncompressed += ll
|
||||
}
|
||||
|
||||
// Check if we are done...
|
||||
if s == len(src) && ml == lz4MinMatch {
|
||||
break
|
||||
}
|
||||
// 2 byte offset
|
||||
if s >= len(src)-2 {
|
||||
if debug {
|
||||
fmt.Printf("s (%d) >= len(src)-2 (%d)", s, len(src)-2)
|
||||
}
|
||||
return nil, 0, ErrCorrupt
|
||||
}
|
||||
offset := binary.LittleEndian.Uint16(src[s:])
|
||||
s += 2
|
||||
if offset == 0 {
|
||||
if debug {
|
||||
fmt.Printf("error: offset 0, ml: %d, len(src)-s: %d\n", ml, len(src)-s)
|
||||
}
|
||||
return nil, 0, ErrCorrupt
|
||||
}
|
||||
if int(offset) > uncompressed {
|
||||
if debug {
|
||||
fmt.Printf("error: offset (%d)> uncompressed (%d)\n", offset, uncompressed)
|
||||
}
|
||||
return nil, 0, ErrCorrupt
|
||||
}
|
||||
|
||||
if ml == lz4MinMatch+15 {
|
||||
for {
|
||||
if s >= len(src) {
|
||||
if debug {
|
||||
fmt.Printf("error reading ml: s (%d) >= len(src) (%d)\n", s, len(src))
|
||||
}
|
||||
return nil, 0, ErrCorrupt
|
||||
}
|
||||
val := src[s]
|
||||
s++
|
||||
ml += int(val)
|
||||
if val != 255 {
|
||||
if s >= len(src) {
|
||||
if debug {
|
||||
fmt.Printf("error reading ml: s (%d) >= len(src) (%d)\n", s, len(src))
|
||||
}
|
||||
return nil, 0, ErrCorrupt
|
||||
}
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
if debug {
|
||||
fmt.Printf("emit copy, length: %d, offset: %d\n", ml, offset)
|
||||
}
|
||||
length := ml
|
||||
// d += emitCopyNoRepeat(dst[d:], int(offset), ml)
|
||||
for length > 0 {
|
||||
if d >= dLimit {
|
||||
return nil, 0, ErrDstTooSmall
|
||||
}
|
||||
|
||||
// Offset no more than 2 bytes.
|
||||
if length > 64 {
|
||||
// Emit a length 64 copy, encoded as 3 bytes.
|
||||
dst[d+2] = uint8(offset >> 8)
|
||||
dst[d+1] = uint8(offset)
|
||||
dst[d+0] = 63<<2 | tagCopy2
|
||||
length -= 64
|
||||
d += 3
|
||||
continue
|
||||
}
|
||||
if length >= 12 || offset >= 2048 || length < 4 {
|
||||
// Emit the remaining copy, encoded as 3 bytes.
|
||||
dst[d+2] = uint8(offset >> 8)
|
||||
dst[d+1] = uint8(offset)
|
||||
dst[d+0] = uint8(length-1)<<2 | tagCopy2
|
||||
d += 3
|
||||
break
|
||||
}
|
||||
// Emit the remaining copy, encoded as 2 bytes.
|
||||
dst[d+1] = uint8(offset)
|
||||
dst[d+0] = uint8(offset>>8)<<5 | uint8(length-4)<<2 | tagCopy1
|
||||
d += 2
|
||||
break
|
||||
}
|
||||
uncompressed += ml
|
||||
if d > dLimit {
|
||||
return nil, 0, ErrDstTooSmall
|
||||
}
|
||||
}
|
||||
|
||||
return dst[:d], uncompressed, nil
|
||||
}
|
||||
|
||||
// emitRepeat writes a repeat chunk and returns the number of bytes written.
|
||||
// Length must be at least 4 and < 1<<24
|
||||
func emitRepeat16(dst []byte, offset uint16, length int) int {
|
||||
// Repeat offset, make length cheaper
|
||||
length -= 4
|
||||
if length <= 4 {
|
||||
dst[0] = uint8(length)<<2 | tagCopy1
|
||||
dst[1] = 0
|
||||
return 2
|
||||
}
|
||||
if length < 8 && offset < 2048 {
|
||||
// Encode WITH offset
|
||||
dst[1] = uint8(offset)
|
||||
dst[0] = uint8(offset>>8)<<5 | uint8(length)<<2 | tagCopy1
|
||||
return 2
|
||||
}
|
||||
if length < (1<<8)+4 {
|
||||
length -= 4
|
||||
dst[2] = uint8(length)
|
||||
dst[1] = 0
|
||||
dst[0] = 5<<2 | tagCopy1
|
||||
return 3
|
||||
}
|
||||
if length < (1<<16)+(1<<8) {
|
||||
length -= 1 << 8
|
||||
dst[3] = uint8(length >> 8)
|
||||
dst[2] = uint8(length >> 0)
|
||||
dst[1] = 0
|
||||
dst[0] = 6<<2 | tagCopy1
|
||||
return 4
|
||||
}
|
||||
const maxRepeat = (1 << 24) - 1
|
||||
length -= 1 << 16
|
||||
left := 0
|
||||
if length > maxRepeat {
|
||||
left = length - maxRepeat + 4
|
||||
length = maxRepeat - 4
|
||||
}
|
||||
dst[4] = uint8(length >> 16)
|
||||
dst[3] = uint8(length >> 8)
|
||||
dst[2] = uint8(length >> 0)
|
||||
dst[1] = 0
|
||||
dst[0] = 7<<2 | tagCopy1
|
||||
if left > 0 {
|
||||
return 5 + emitRepeat16(dst[5:], offset, left)
|
||||
}
|
||||
return 5
|
||||
}
|
||||
|
||||
// emitCopy writes a copy chunk and returns the number of bytes written.
|
||||
//
|
||||
// It assumes that:
|
||||
//
|
||||
// dst is long enough to hold the encoded bytes
|
||||
// 1 <= offset && offset <= math.MaxUint16
|
||||
// 4 <= length && length <= math.MaxUint32
|
||||
func emitCopy16(dst []byte, offset uint16, length int) int {
|
||||
// Offset no more than 2 bytes.
|
||||
if length > 64 {
|
||||
off := 3
|
||||
if offset < 2048 {
|
||||
// emit 8 bytes as tagCopy1, rest as repeats.
|
||||
dst[1] = uint8(offset)
|
||||
dst[0] = uint8(offset>>8)<<5 | uint8(8-4)<<2 | tagCopy1
|
||||
length -= 8
|
||||
off = 2
|
||||
} else {
|
||||
// Emit a length 60 copy, encoded as 3 bytes.
|
||||
// Emit remaining as repeat value (minimum 4 bytes).
|
||||
dst[2] = uint8(offset >> 8)
|
||||
dst[1] = uint8(offset)
|
||||
dst[0] = 59<<2 | tagCopy2
|
||||
length -= 60
|
||||
}
|
||||
// Emit remaining as repeats, at least 4 bytes remain.
|
||||
return off + emitRepeat16(dst[off:], offset, length)
|
||||
}
|
||||
if length >= 12 || offset >= 2048 {
|
||||
// Emit the remaining copy, encoded as 3 bytes.
|
||||
dst[2] = uint8(offset >> 8)
|
||||
dst[1] = uint8(offset)
|
||||
dst[0] = uint8(length-1)<<2 | tagCopy2
|
||||
return 3
|
||||
}
|
||||
// Emit the remaining copy, encoded as 2 bytes.
|
||||
dst[1] = uint8(offset)
|
||||
dst[0] = uint8(offset>>8)<<5 | uint8(length-4)<<2 | tagCopy1
|
||||
return 2
|
||||
}
|
||||
|
||||
// emitLiteral writes a literal chunk and returns the number of bytes written.
|
||||
//
|
||||
// It assumes that:
|
||||
//
|
||||
// dst is long enough to hold the encoded bytes
|
||||
// 0 <= len(lit) && len(lit) <= math.MaxUint32
|
||||
func emitLiteralGo(dst, lit []byte) int {
|
||||
if len(lit) == 0 {
|
||||
return 0
|
||||
}
|
||||
i, n := 0, uint(len(lit)-1)
|
||||
switch {
|
||||
case n < 60:
|
||||
dst[0] = uint8(n)<<2 | tagLiteral
|
||||
i = 1
|
||||
case n < 1<<8:
|
||||
dst[1] = uint8(n)
|
||||
dst[0] = 60<<2 | tagLiteral
|
||||
i = 2
|
||||
case n < 1<<16:
|
||||
dst[2] = uint8(n >> 8)
|
||||
dst[1] = uint8(n)
|
||||
dst[0] = 61<<2 | tagLiteral
|
||||
i = 3
|
||||
case n < 1<<24:
|
||||
dst[3] = uint8(n >> 16)
|
||||
dst[2] = uint8(n >> 8)
|
||||
dst[1] = uint8(n)
|
||||
dst[0] = 62<<2 | tagLiteral
|
||||
i = 4
|
||||
default:
|
||||
dst[4] = uint8(n >> 24)
|
||||
dst[3] = uint8(n >> 16)
|
||||
dst[2] = uint8(n >> 8)
|
||||
dst[1] = uint8(n)
|
||||
dst[0] = 63<<2 | tagLiteral
|
||||
i = 5
|
||||
}
|
||||
return i + copy(dst[i:], lit)
|
||||
}
|
||||
+467
@@ -0,0 +1,467 @@
|
||||
// Copyright (c) 2022 Klaus Post. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package s2
|
||||
|
||||
import (
|
||||
"encoding/binary"
|
||||
"fmt"
|
||||
)
|
||||
|
||||
// LZ4sConverter provides conversion from LZ4s.
|
||||
// (Intel modified LZ4 Blocks)
|
||||
// https://cdrdv2-public.intel.com/743912/743912-qat-programmers-guide-v2.0.pdf
|
||||
// LZ4s is a variant of LZ4 block format. LZ4s should be considered as an intermediate compressed block format.
|
||||
// The LZ4s format is selected when the application sets the compType to CPA_DC_LZ4S in CpaDcSessionSetupData.
|
||||
// The LZ4s block returned by the Intel® QAT hardware can be used by an external
|
||||
// software post-processing to generate other compressed data formats.
|
||||
// The following table lists the differences between LZ4 and LZ4s block format. LZ4s block format uses
|
||||
// the same high-level formatting as LZ4 block format with the following encoding changes:
|
||||
// For Min Match of 4 bytes, Copy length value 1-15 means length 4-18 with 18 bytes adding an extra byte.
|
||||
// ONLY "Min match of 4 bytes" is supported.
|
||||
type LZ4sConverter struct {
|
||||
}
|
||||
|
||||
// ConvertBlock will convert an LZ4s block and append it as an S2
|
||||
// block without block length to dst.
|
||||
// The uncompressed size is returned as well.
|
||||
// dst must have capacity to contain the entire compressed block.
|
||||
func (l *LZ4sConverter) ConvertBlock(dst, src []byte) ([]byte, int, error) {
|
||||
if len(src) == 0 {
|
||||
return dst, 0, nil
|
||||
}
|
||||
const debug = false
|
||||
const inline = true
|
||||
const lz4MinMatch = 3
|
||||
|
||||
s, d := 0, len(dst)
|
||||
dst = dst[:cap(dst)]
|
||||
if !debug && hasAsm {
|
||||
res, sz := cvtLZ4sBlockAsm(dst[d:], src)
|
||||
if res < 0 {
|
||||
const (
|
||||
errCorrupt = -1
|
||||
errDstTooSmall = -2
|
||||
)
|
||||
switch res {
|
||||
case errCorrupt:
|
||||
return nil, 0, ErrCorrupt
|
||||
case errDstTooSmall:
|
||||
return nil, 0, ErrDstTooSmall
|
||||
default:
|
||||
return nil, 0, fmt.Errorf("unexpected result: %d", res)
|
||||
}
|
||||
}
|
||||
if d+sz > len(dst) {
|
||||
return nil, 0, ErrDstTooSmall
|
||||
}
|
||||
return dst[:d+sz], res, nil
|
||||
}
|
||||
|
||||
dLimit := len(dst) - 10
|
||||
var lastOffset uint16
|
||||
var uncompressed int
|
||||
if debug {
|
||||
fmt.Printf("convert block start: len(src): %d, len(dst):%d \n", len(src), len(dst))
|
||||
}
|
||||
|
||||
for {
|
||||
if s >= len(src) {
|
||||
return dst[:d], 0, ErrCorrupt
|
||||
}
|
||||
// Read literal info
|
||||
token := src[s]
|
||||
ll := int(token >> 4)
|
||||
ml := int(lz4MinMatch + (token & 0xf))
|
||||
|
||||
// If upper nibble is 15, literal length is extended
|
||||
if token >= 0xf0 {
|
||||
for {
|
||||
s++
|
||||
if s >= len(src) {
|
||||
if debug {
|
||||
fmt.Printf("error reading ll: s (%d) >= len(src) (%d)\n", s, len(src))
|
||||
}
|
||||
return dst[:d], 0, ErrCorrupt
|
||||
}
|
||||
val := src[s]
|
||||
ll += int(val)
|
||||
if val != 255 {
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
// Skip past token
|
||||
if s+ll >= len(src) {
|
||||
if debug {
|
||||
fmt.Printf("error literals: s+ll (%d+%d) >= len(src) (%d)\n", s, ll, len(src))
|
||||
}
|
||||
return nil, 0, ErrCorrupt
|
||||
}
|
||||
s++
|
||||
if ll > 0 {
|
||||
if d+ll > dLimit {
|
||||
return nil, 0, ErrDstTooSmall
|
||||
}
|
||||
if debug {
|
||||
fmt.Printf("emit %d literals\n", ll)
|
||||
}
|
||||
d += emitLiteralGo(dst[d:], src[s:s+ll])
|
||||
s += ll
|
||||
uncompressed += ll
|
||||
}
|
||||
|
||||
// Check if we are done...
|
||||
if ml == lz4MinMatch {
|
||||
if s == len(src) {
|
||||
break
|
||||
}
|
||||
// 0 bytes.
|
||||
continue
|
||||
}
|
||||
// 2 byte offset
|
||||
if s >= len(src)-2 {
|
||||
if debug {
|
||||
fmt.Printf("s (%d) >= len(src)-2 (%d)", s, len(src)-2)
|
||||
}
|
||||
return nil, 0, ErrCorrupt
|
||||
}
|
||||
offset := binary.LittleEndian.Uint16(src[s:])
|
||||
s += 2
|
||||
if offset == 0 {
|
||||
if debug {
|
||||
fmt.Printf("error: offset 0, ml: %d, len(src)-s: %d\n", ml, len(src)-s)
|
||||
}
|
||||
return nil, 0, ErrCorrupt
|
||||
}
|
||||
if int(offset) > uncompressed {
|
||||
if debug {
|
||||
fmt.Printf("error: offset (%d)> uncompressed (%d)\n", offset, uncompressed)
|
||||
}
|
||||
return nil, 0, ErrCorrupt
|
||||
}
|
||||
|
||||
if ml == lz4MinMatch+15 {
|
||||
for {
|
||||
if s >= len(src) {
|
||||
if debug {
|
||||
fmt.Printf("error reading ml: s (%d) >= len(src) (%d)\n", s, len(src))
|
||||
}
|
||||
return nil, 0, ErrCorrupt
|
||||
}
|
||||
val := src[s]
|
||||
s++
|
||||
ml += int(val)
|
||||
if val != 255 {
|
||||
if s >= len(src) {
|
||||
if debug {
|
||||
fmt.Printf("error reading ml: s (%d) >= len(src) (%d)\n", s, len(src))
|
||||
}
|
||||
return nil, 0, ErrCorrupt
|
||||
}
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
if offset == lastOffset {
|
||||
if debug {
|
||||
fmt.Printf("emit repeat, length: %d, offset: %d\n", ml, offset)
|
||||
}
|
||||
if !inline {
|
||||
d += emitRepeat16(dst[d:], offset, ml)
|
||||
} else {
|
||||
length := ml
|
||||
dst := dst[d:]
|
||||
for len(dst) > 5 {
|
||||
// Repeat offset, make length cheaper
|
||||
length -= 4
|
||||
if length <= 4 {
|
||||
dst[0] = uint8(length)<<2 | tagCopy1
|
||||
dst[1] = 0
|
||||
d += 2
|
||||
break
|
||||
}
|
||||
if length < 8 && offset < 2048 {
|
||||
// Encode WITH offset
|
||||
dst[1] = uint8(offset)
|
||||
dst[0] = uint8(offset>>8)<<5 | uint8(length)<<2 | tagCopy1
|
||||
d += 2
|
||||
break
|
||||
}
|
||||
if length < (1<<8)+4 {
|
||||
length -= 4
|
||||
dst[2] = uint8(length)
|
||||
dst[1] = 0
|
||||
dst[0] = 5<<2 | tagCopy1
|
||||
d += 3
|
||||
break
|
||||
}
|
||||
if length < (1<<16)+(1<<8) {
|
||||
length -= 1 << 8
|
||||
dst[3] = uint8(length >> 8)
|
||||
dst[2] = uint8(length >> 0)
|
||||
dst[1] = 0
|
||||
dst[0] = 6<<2 | tagCopy1
|
||||
d += 4
|
||||
break
|
||||
}
|
||||
const maxRepeat = (1 << 24) - 1
|
||||
length -= 1 << 16
|
||||
left := 0
|
||||
if length > maxRepeat {
|
||||
left = length - maxRepeat + 4
|
||||
length = maxRepeat - 4
|
||||
}
|
||||
dst[4] = uint8(length >> 16)
|
||||
dst[3] = uint8(length >> 8)
|
||||
dst[2] = uint8(length >> 0)
|
||||
dst[1] = 0
|
||||
dst[0] = 7<<2 | tagCopy1
|
||||
if left > 0 {
|
||||
d += 5 + emitRepeat16(dst[5:], offset, left)
|
||||
break
|
||||
}
|
||||
d += 5
|
||||
break
|
||||
}
|
||||
}
|
||||
} else {
|
||||
if debug {
|
||||
fmt.Printf("emit copy, length: %d, offset: %d\n", ml, offset)
|
||||
}
|
||||
if !inline {
|
||||
d += emitCopy16(dst[d:], offset, ml)
|
||||
} else {
|
||||
length := ml
|
||||
dst := dst[d:]
|
||||
for len(dst) > 5 {
|
||||
// Offset no more than 2 bytes.
|
||||
if length > 64 {
|
||||
off := 3
|
||||
if offset < 2048 {
|
||||
// emit 8 bytes as tagCopy1, rest as repeats.
|
||||
dst[1] = uint8(offset)
|
||||
dst[0] = uint8(offset>>8)<<5 | uint8(8-4)<<2 | tagCopy1
|
||||
length -= 8
|
||||
off = 2
|
||||
} else {
|
||||
// Emit a length 60 copy, encoded as 3 bytes.
|
||||
// Emit remaining as repeat value (minimum 4 bytes).
|
||||
dst[2] = uint8(offset >> 8)
|
||||
dst[1] = uint8(offset)
|
||||
dst[0] = 59<<2 | tagCopy2
|
||||
length -= 60
|
||||
}
|
||||
// Emit remaining as repeats, at least 4 bytes remain.
|
||||
d += off + emitRepeat16(dst[off:], offset, length)
|
||||
break
|
||||
}
|
||||
if length >= 12 || offset >= 2048 {
|
||||
// Emit the remaining copy, encoded as 3 bytes.
|
||||
dst[2] = uint8(offset >> 8)
|
||||
dst[1] = uint8(offset)
|
||||
dst[0] = uint8(length-1)<<2 | tagCopy2
|
||||
d += 3
|
||||
break
|
||||
}
|
||||
// Emit the remaining copy, encoded as 2 bytes.
|
||||
dst[1] = uint8(offset)
|
||||
dst[0] = uint8(offset>>8)<<5 | uint8(length-4)<<2 | tagCopy1
|
||||
d += 2
|
||||
break
|
||||
}
|
||||
}
|
||||
lastOffset = offset
|
||||
}
|
||||
uncompressed += ml
|
||||
if d > dLimit {
|
||||
return nil, 0, ErrDstTooSmall
|
||||
}
|
||||
}
|
||||
|
||||
return dst[:d], uncompressed, nil
|
||||
}
|
||||
|
||||
// ConvertBlockSnappy will convert an LZ4s block and append it
|
||||
// as a Snappy block without block length to dst.
|
||||
// The uncompressed size is returned as well.
|
||||
// dst must have capacity to contain the entire compressed block.
|
||||
func (l *LZ4sConverter) ConvertBlockSnappy(dst, src []byte) ([]byte, int, error) {
|
||||
if len(src) == 0 {
|
||||
return dst, 0, nil
|
||||
}
|
||||
const debug = false
|
||||
const lz4MinMatch = 3
|
||||
|
||||
s, d := 0, len(dst)
|
||||
dst = dst[:cap(dst)]
|
||||
// Use assembly when possible
|
||||
if !debug && hasAsm {
|
||||
res, sz := cvtLZ4sBlockSnappyAsm(dst[d:], src)
|
||||
if res < 0 {
|
||||
const (
|
||||
errCorrupt = -1
|
||||
errDstTooSmall = -2
|
||||
)
|
||||
switch res {
|
||||
case errCorrupt:
|
||||
return nil, 0, ErrCorrupt
|
||||
case errDstTooSmall:
|
||||
return nil, 0, ErrDstTooSmall
|
||||
default:
|
||||
return nil, 0, fmt.Errorf("unexpected result: %d", res)
|
||||
}
|
||||
}
|
||||
if d+sz > len(dst) {
|
||||
return nil, 0, ErrDstTooSmall
|
||||
}
|
||||
return dst[:d+sz], res, nil
|
||||
}
|
||||
|
||||
dLimit := len(dst) - 10
|
||||
var uncompressed int
|
||||
if debug {
|
||||
fmt.Printf("convert block start: len(src): %d, len(dst):%d \n", len(src), len(dst))
|
||||
}
|
||||
|
||||
for {
|
||||
if s >= len(src) {
|
||||
return nil, 0, ErrCorrupt
|
||||
}
|
||||
// Read literal info
|
||||
token := src[s]
|
||||
ll := int(token >> 4)
|
||||
ml := int(lz4MinMatch + (token & 0xf))
|
||||
|
||||
// If upper nibble is 15, literal length is extended
|
||||
if token >= 0xf0 {
|
||||
for {
|
||||
s++
|
||||
if s >= len(src) {
|
||||
if debug {
|
||||
fmt.Printf("error reading ll: s (%d) >= len(src) (%d)\n", s, len(src))
|
||||
}
|
||||
return nil, 0, ErrCorrupt
|
||||
}
|
||||
val := src[s]
|
||||
ll += int(val)
|
||||
if val != 255 {
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
// Skip past token
|
||||
if s+ll >= len(src) {
|
||||
if debug {
|
||||
fmt.Printf("error literals: s+ll (%d+%d) >= len(src) (%d)\n", s, ll, len(src))
|
||||
}
|
||||
return nil, 0, ErrCorrupt
|
||||
}
|
||||
s++
|
||||
if ll > 0 {
|
||||
if d+ll > dLimit {
|
||||
return nil, 0, ErrDstTooSmall
|
||||
}
|
||||
if debug {
|
||||
fmt.Printf("emit %d literals\n", ll)
|
||||
}
|
||||
d += emitLiteralGo(dst[d:], src[s:s+ll])
|
||||
s += ll
|
||||
uncompressed += ll
|
||||
}
|
||||
|
||||
// Check if we are done...
|
||||
if ml == lz4MinMatch {
|
||||
if s == len(src) {
|
||||
break
|
||||
}
|
||||
// 0 bytes.
|
||||
continue
|
||||
}
|
||||
// 2 byte offset
|
||||
if s >= len(src)-2 {
|
||||
if debug {
|
||||
fmt.Printf("s (%d) >= len(src)-2 (%d)", s, len(src)-2)
|
||||
}
|
||||
return nil, 0, ErrCorrupt
|
||||
}
|
||||
offset := binary.LittleEndian.Uint16(src[s:])
|
||||
s += 2
|
||||
if offset == 0 {
|
||||
if debug {
|
||||
fmt.Printf("error: offset 0, ml: %d, len(src)-s: %d\n", ml, len(src)-s)
|
||||
}
|
||||
return nil, 0, ErrCorrupt
|
||||
}
|
||||
if int(offset) > uncompressed {
|
||||
if debug {
|
||||
fmt.Printf("error: offset (%d)> uncompressed (%d)\n", offset, uncompressed)
|
||||
}
|
||||
return nil, 0, ErrCorrupt
|
||||
}
|
||||
|
||||
if ml == lz4MinMatch+15 {
|
||||
for {
|
||||
if s >= len(src) {
|
||||
if debug {
|
||||
fmt.Printf("error reading ml: s (%d) >= len(src) (%d)\n", s, len(src))
|
||||
}
|
||||
return nil, 0, ErrCorrupt
|
||||
}
|
||||
val := src[s]
|
||||
s++
|
||||
ml += int(val)
|
||||
if val != 255 {
|
||||
if s >= len(src) {
|
||||
if debug {
|
||||
fmt.Printf("error reading ml: s (%d) >= len(src) (%d)\n", s, len(src))
|
||||
}
|
||||
return nil, 0, ErrCorrupt
|
||||
}
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
if debug {
|
||||
fmt.Printf("emit copy, length: %d, offset: %d\n", ml, offset)
|
||||
}
|
||||
length := ml
|
||||
// d += emitCopyNoRepeat(dst[d:], int(offset), ml)
|
||||
for length > 0 {
|
||||
if d >= dLimit {
|
||||
return nil, 0, ErrDstTooSmall
|
||||
}
|
||||
|
||||
// Offset no more than 2 bytes.
|
||||
if length > 64 {
|
||||
// Emit a length 64 copy, encoded as 3 bytes.
|
||||
dst[d+2] = uint8(offset >> 8)
|
||||
dst[d+1] = uint8(offset)
|
||||
dst[d+0] = 63<<2 | tagCopy2
|
||||
length -= 64
|
||||
d += 3
|
||||
continue
|
||||
}
|
||||
if length >= 12 || offset >= 2048 || length < 4 {
|
||||
// Emit the remaining copy, encoded as 3 bytes.
|
||||
dst[d+2] = uint8(offset >> 8)
|
||||
dst[d+1] = uint8(offset)
|
||||
dst[d+0] = uint8(length-1)<<2 | tagCopy2
|
||||
d += 3
|
||||
break
|
||||
}
|
||||
// Emit the remaining copy, encoded as 2 bytes.
|
||||
dst[d+1] = uint8(offset)
|
||||
dst[d+0] = uint8(offset>>8)<<5 | uint8(length-4)<<2 | tagCopy1
|
||||
d += 2
|
||||
break
|
||||
}
|
||||
uncompressed += ml
|
||||
if d > dLimit {
|
||||
return nil, 0, ErrDstTooSmall
|
||||
}
|
||||
}
|
||||
|
||||
return dst[:d], uncompressed, nil
|
||||
}
|
||||
+1078
File diff suppressed because it is too large
Load Diff
+151
@@ -0,0 +1,151 @@
|
||||
// Copyright 2011 The Snappy-Go Authors. All rights reserved.
|
||||
// Copyright (c) 2019 Klaus Post. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
// Package s2 implements the S2 compression format.
|
||||
//
|
||||
// S2 is an extension of Snappy. Similar to Snappy S2 is aimed for high throughput,
|
||||
// which is why it features concurrent compression for bigger payloads.
|
||||
//
|
||||
// Decoding is compatible with Snappy compressed content,
|
||||
// but content compressed with S2 cannot be decompressed by Snappy.
|
||||
//
|
||||
// For more information on Snappy/S2 differences see README in: https://github.com/klauspost/compress/tree/master/s2
|
||||
//
|
||||
// There are actually two S2 formats: block and stream. They are related,
|
||||
// but different: trying to decompress block-compressed data as a S2 stream
|
||||
// will fail, and vice versa. The block format is the Decode and Encode
|
||||
// functions and the stream format is the Reader and Writer types.
|
||||
//
|
||||
// A "better" compression option is available. This will trade some compression
|
||||
// speed
|
||||
//
|
||||
// The block format, the more common case, is used when the complete size (the
|
||||
// number of bytes) of the original data is known upfront, at the time
|
||||
// compression starts. The stream format, also known as the framing format, is
|
||||
// for when that isn't always true.
|
||||
//
|
||||
// Blocks to not offer much data protection, so it is up to you to
|
||||
// add data validation of decompressed blocks.
|
||||
//
|
||||
// Streams perform CRC validation of the decompressed data.
|
||||
// Stream compression will also be performed on multiple CPU cores concurrently
|
||||
// significantly improving throughput.
|
||||
package s2
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"hash/crc32"
|
||||
|
||||
"github.com/klauspost/compress/internal/race"
|
||||
)
|
||||
|
||||
/*
|
||||
Each encoded block begins with the varint-encoded length of the decoded data,
|
||||
followed by a sequence of chunks. Chunks begin and end on byte boundaries. The
|
||||
first byte of each chunk is broken into its 2 least and 6 most significant bits
|
||||
called l and m: l ranges in [0, 4) and m ranges in [0, 64). l is the chunk tag.
|
||||
Zero means a literal tag. All other values mean a copy tag.
|
||||
|
||||
For literal tags:
|
||||
- If m < 60, the next 1 + m bytes are literal bytes.
|
||||
- Otherwise, let n be the little-endian unsigned integer denoted by the next
|
||||
m - 59 bytes. The next 1 + n bytes after that are literal bytes.
|
||||
|
||||
For copy tags, length bytes are copied from offset bytes ago, in the style of
|
||||
Lempel-Ziv compression algorithms. In particular:
|
||||
- For l == 1, the offset ranges in [0, 1<<11) and the length in [4, 12).
|
||||
The length is 4 + the low 3 bits of m. The high 3 bits of m form bits 8-10
|
||||
of the offset. The next byte is bits 0-7 of the offset.
|
||||
- For l == 2, the offset ranges in [0, 1<<16) and the length in [1, 65).
|
||||
The length is 1 + m. The offset is the little-endian unsigned integer
|
||||
denoted by the next 2 bytes.
|
||||
- For l == 3, the offset ranges in [0, 1<<32) and the length in
|
||||
[1, 65). The length is 1 + m. The offset is the little-endian unsigned
|
||||
integer denoted by the next 4 bytes.
|
||||
*/
|
||||
const (
|
||||
tagLiteral = 0x00
|
||||
tagCopy1 = 0x01
|
||||
tagCopy2 = 0x02
|
||||
tagCopy4 = 0x03
|
||||
)
|
||||
|
||||
const (
|
||||
checksumSize = 4
|
||||
chunkHeaderSize = 4
|
||||
magicChunk = "\xff\x06\x00\x00" + magicBody
|
||||
magicChunkSnappy = "\xff\x06\x00\x00" + magicBodySnappy
|
||||
magicBodySnappy = "sNaPpY"
|
||||
magicBody = "S2sTwO"
|
||||
|
||||
// maxBlockSize is the maximum size of the input to encodeBlock.
|
||||
//
|
||||
// For the framing format (Writer type instead of Encode function),
|
||||
// this is the maximum uncompressed size of a block.
|
||||
maxBlockSize = 4 << 20
|
||||
|
||||
// minBlockSize is the minimum size of block setting when creating a writer.
|
||||
minBlockSize = 4 << 10
|
||||
|
||||
skippableFrameHeader = 4
|
||||
maxChunkSize = 1<<24 - 1 // 16777215
|
||||
|
||||
// Default block size
|
||||
defaultBlockSize = 1 << 20
|
||||
|
||||
// maxSnappyBlockSize is the maximum snappy block size.
|
||||
maxSnappyBlockSize = 1 << 16
|
||||
|
||||
obufHeaderLen = checksumSize + chunkHeaderSize
|
||||
)
|
||||
|
||||
const (
|
||||
chunkTypeCompressedData = 0x00
|
||||
chunkTypeUncompressedData = 0x01
|
||||
ChunkTypeIndex = 0x99
|
||||
chunkTypePadding = 0xfe
|
||||
chunkTypeStreamIdentifier = 0xff
|
||||
)
|
||||
|
||||
var (
|
||||
crcTable = crc32.MakeTable(crc32.Castagnoli)
|
||||
magicChunkSnappyBytes = []byte(magicChunkSnappy) // Can be passed to functions where it escapes.
|
||||
magicChunkBytes = []byte(magicChunk) // Can be passed to functions where it escapes.
|
||||
)
|
||||
|
||||
// crc implements the checksum specified in section 3 of
|
||||
// https://github.com/google/snappy/blob/master/framing_format.txt
|
||||
func crc(b []byte) uint32 {
|
||||
race.ReadSlice(b)
|
||||
|
||||
c := crc32.Update(0, crcTable, b)
|
||||
return c>>15 | c<<17 + 0xa282ead8
|
||||
}
|
||||
|
||||
// literalExtraSize returns the extra size of encoding n literals.
|
||||
// n should be >= 0 and <= math.MaxUint32.
|
||||
func literalExtraSize(n int64) int64 {
|
||||
if n == 0 {
|
||||
return 0
|
||||
}
|
||||
switch {
|
||||
case n < 60:
|
||||
return 1
|
||||
case n < 1<<8:
|
||||
return 2
|
||||
case n < 1<<16:
|
||||
return 3
|
||||
case n < 1<<24:
|
||||
return 4
|
||||
default:
|
||||
return 5
|
||||
}
|
||||
}
|
||||
|
||||
type byter interface {
|
||||
Bytes() []byte
|
||||
}
|
||||
|
||||
var _ byter = &bytes.Buffer{}
|
||||
+1062
File diff suppressed because it is too large
Load Diff
+1
@@ -0,0 +1 @@
|
||||
*.test
|
||||
+28
@@ -0,0 +1,28 @@
|
||||
linters-settings:
|
||||
golint:
|
||||
min-confidence: 0
|
||||
|
||||
misspell:
|
||||
locale: US
|
||||
|
||||
linters:
|
||||
disable-all: true
|
||||
enable:
|
||||
- typecheck
|
||||
- goimports
|
||||
- misspell
|
||||
- govet
|
||||
- revive
|
||||
- ineffassign
|
||||
- gosimple
|
||||
- unparam
|
||||
- unused
|
||||
|
||||
issues:
|
||||
exclude-use-default: false
|
||||
exclude:
|
||||
- should have a package comment
|
||||
- error strings should not be capitalized or end with punctuation or a newline
|
||||
- should have comment # TODO(aead): Remove once all exported ident. have comments!
|
||||
service:
|
||||
golangci-lint-version: 1.51.2 # use the fixed version to not introduce new linters unexpectedly
|
||||
+202
@@ -0,0 +1,202 @@
|
||||
|
||||
Apache License
|
||||
Version 2.0, January 2004
|
||||
http://www.apache.org/licenses/
|
||||
|
||||
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
||||
|
||||
1. Definitions.
|
||||
|
||||
"License" shall mean the terms and conditions for use, reproduction,
|
||||
and distribution as defined by Sections 1 through 9 of this document.
|
||||
|
||||
"Licensor" shall mean the copyright owner or entity authorized by
|
||||
the copyright owner that is granting the License.
|
||||
|
||||
"Legal Entity" shall mean the union of the acting entity and all
|
||||
other entities that control, are controlled by, or are under common
|
||||
control with that entity. For the purposes of this definition,
|
||||
"control" means (i) the power, direct or indirect, to cause the
|
||||
direction or management of such entity, whether by contract or
|
||||
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
||||
outstanding shares, or (iii) beneficial ownership of such entity.
|
||||
|
||||
"You" (or "Your") shall mean an individual or Legal Entity
|
||||
exercising permissions granted by this License.
|
||||
|
||||
"Source" form shall mean the preferred form for making modifications,
|
||||
including but not limited to software source code, documentation
|
||||
source, and configuration files.
|
||||
|
||||
"Object" form shall mean any form resulting from mechanical
|
||||
transformation or translation of a Source form, including but
|
||||
not limited to compiled object code, generated documentation,
|
||||
and conversions to other media types.
|
||||
|
||||
"Work" shall mean the work of authorship, whether in Source or
|
||||
Object form, made available under the License, as indicated by a
|
||||
copyright notice that is included in or attached to the work
|
||||
(an example is provided in the Appendix below).
|
||||
|
||||
"Derivative Works" shall mean any work, whether in Source or Object
|
||||
form, that is based on (or derived from) the Work and for which the
|
||||
editorial revisions, annotations, elaborations, or other modifications
|
||||
represent, as a whole, an original work of authorship. For the purposes
|
||||
of this License, Derivative Works shall not include works that remain
|
||||
separable from, or merely link (or bind by name) to the interfaces of,
|
||||
the Work and Derivative Works thereof.
|
||||
|
||||
"Contribution" shall mean any work of authorship, including
|
||||
the original version of the Work and any modifications or additions
|
||||
to that Work or Derivative Works thereof, that is intentionally
|
||||
submitted to Licensor for inclusion in the Work by the copyright owner
|
||||
or by an individual or Legal Entity authorized to submit on behalf of
|
||||
the copyright owner. For the purposes of this definition, "submitted"
|
||||
means any form of electronic, verbal, or written communication sent
|
||||
to the Licensor or its representatives, including but not limited to
|
||||
communication on electronic mailing lists, source code control systems,
|
||||
and issue tracking systems that are managed by, or on behalf of, the
|
||||
Licensor for the purpose of discussing and improving the Work, but
|
||||
excluding communication that is conspicuously marked or otherwise
|
||||
designated in writing by the copyright owner as "Not a Contribution."
|
||||
|
||||
"Contributor" shall mean Licensor and any individual or Legal Entity
|
||||
on behalf of whom a Contribution has been received by Licensor and
|
||||
subsequently incorporated within the Work.
|
||||
|
||||
2. Grant of Copyright License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
copyright license to reproduce, prepare Derivative Works of,
|
||||
publicly display, publicly perform, sublicense, and distribute the
|
||||
Work and such Derivative Works in Source or Object form.
|
||||
|
||||
3. Grant of Patent License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
(except as stated in this section) patent license to make, have made,
|
||||
use, offer to sell, sell, import, and otherwise transfer the Work,
|
||||
where such license applies only to those patent claims licensable
|
||||
by such Contributor that are necessarily infringed by their
|
||||
Contribution(s) alone or by combination of their Contribution(s)
|
||||
with the Work to which such Contribution(s) was submitted. If You
|
||||
institute patent litigation against any entity (including a
|
||||
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
||||
or a Contribution incorporated within the Work constitutes direct
|
||||
or contributory patent infringement, then any patent licenses
|
||||
granted to You under this License for that Work shall terminate
|
||||
as of the date such litigation is filed.
|
||||
|
||||
4. Redistribution. You may reproduce and distribute copies of the
|
||||
Work or Derivative Works thereof in any medium, with or without
|
||||
modifications, and in Source or Object form, provided that You
|
||||
meet the following conditions:
|
||||
|
||||
(a) You must give any other recipients of the Work or
|
||||
Derivative Works a copy of this License; and
|
||||
|
||||
(b) You must cause any modified files to carry prominent notices
|
||||
stating that You changed the files; and
|
||||
|
||||
(c) You must retain, in the Source form of any Derivative Works
|
||||
that You distribute, all copyright, patent, trademark, and
|
||||
attribution notices from the Source form of the Work,
|
||||
excluding those notices that do not pertain to any part of
|
||||
the Derivative Works; and
|
||||
|
||||
(d) If the Work includes a "NOTICE" text file as part of its
|
||||
distribution, then any Derivative Works that You distribute must
|
||||
include a readable copy of the attribution notices contained
|
||||
within such NOTICE file, excluding those notices that do not
|
||||
pertain to any part of the Derivative Works, in at least one
|
||||
of the following places: within a NOTICE text file distributed
|
||||
as part of the Derivative Works; within the Source form or
|
||||
documentation, if provided along with the Derivative Works; or,
|
||||
within a display generated by the Derivative Works, if and
|
||||
wherever such third-party notices normally appear. The contents
|
||||
of the NOTICE file are for informational purposes only and
|
||||
do not modify the License. You may add Your own attribution
|
||||
notices within Derivative Works that You distribute, alongside
|
||||
or as an addendum to the NOTICE text from the Work, provided
|
||||
that such additional attribution notices cannot be construed
|
||||
as modifying the License.
|
||||
|
||||
You may add Your own copyright statement to Your modifications and
|
||||
may provide additional or different license terms and conditions
|
||||
for use, reproduction, or distribution of Your modifications, or
|
||||
for any such Derivative Works as a whole, provided Your use,
|
||||
reproduction, and distribution of the Work otherwise complies with
|
||||
the conditions stated in this License.
|
||||
|
||||
5. Submission of Contributions. Unless You explicitly state otherwise,
|
||||
any Contribution intentionally submitted for inclusion in the Work
|
||||
by You to the Licensor shall be under the terms and conditions of
|
||||
this License, without any additional terms or conditions.
|
||||
Notwithstanding the above, nothing herein shall supersede or modify
|
||||
the terms of any separate license agreement you may have executed
|
||||
with Licensor regarding such Contributions.
|
||||
|
||||
6. Trademarks. This License does not grant permission to use the trade
|
||||
names, trademarks, service marks, or product names of the Licensor,
|
||||
except as required for reasonable and customary use in describing the
|
||||
origin of the Work and reproducing the content of the NOTICE file.
|
||||
|
||||
7. Disclaimer of Warranty. Unless required by applicable law or
|
||||
agreed to in writing, Licensor provides the Work (and each
|
||||
Contributor provides its Contributions) on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
implied, including, without limitation, any warranties or conditions
|
||||
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
||||
PARTICULAR PURPOSE. You are solely responsible for determining the
|
||||
appropriateness of using or redistributing the Work and assume any
|
||||
risks associated with Your exercise of permissions under this License.
|
||||
|
||||
8. Limitation of Liability. In no event and under no legal theory,
|
||||
whether in tort (including negligence), contract, or otherwise,
|
||||
unless required by applicable law (such as deliberate and grossly
|
||||
negligent acts) or agreed to in writing, shall any Contributor be
|
||||
liable to You for damages, including any direct, indirect, special,
|
||||
incidental, or consequential damages of any character arising as a
|
||||
result of this License or out of the use or inability to use the
|
||||
Work (including but not limited to damages for loss of goodwill,
|
||||
work stoppage, computer failure or malfunction, or any and all
|
||||
other commercial damages or losses), even if such Contributor
|
||||
has been advised of the possibility of such damages.
|
||||
|
||||
9. Accepting Warranty or Additional Liability. While redistributing
|
||||
the Work or Derivative Works thereof, You may choose to offer,
|
||||
and charge a fee for, acceptance of support, warranty, indemnity,
|
||||
or other liability obligations and/or rights consistent with this
|
||||
License. However, in accepting such obligations, You may act only
|
||||
on Your own behalf and on Your sole responsibility, not on behalf
|
||||
of any other Contributor, and only if You agree to indemnify,
|
||||
defend, and hold each Contributor harmless for any liability
|
||||
incurred by, or claims asserted against, such Contributor by reason
|
||||
of your accepting any such warranty or additional liability.
|
||||
|
||||
END OF TERMS AND CONDITIONS
|
||||
|
||||
APPENDIX: How to apply the Apache License to your work.
|
||||
|
||||
To apply the Apache License to your work, attach the following
|
||||
boilerplate notice, with the fields enclosed by brackets "[]"
|
||||
replaced with your own identifying information. (Don't include
|
||||
the brackets!) The text should be enclosed in the appropriate
|
||||
comment syntax for the file format. We also recommend that a
|
||||
file or class name and description of purpose be included on the
|
||||
same "printed page" as the copyright notice for easier
|
||||
identification within third-party archives.
|
||||
|
||||
Copyright [yyyy] [name of copyright owner]
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
+99
@@ -0,0 +1,99 @@
|
||||
[](https://godoc.org/github.com/minio/highwayhash)
|
||||
[](https://travis-ci.org/minio/highwayhash)
|
||||
|
||||
## HighwayHash
|
||||
|
||||
[HighwayHash](https://github.com/google/highwayhash) is a pseudo-random-function (PRF) developed by Jyrki Alakuijala, Bill Cox and Jan Wassenberg (Google research). HighwayHash takes a 256 bit key and computes 64, 128 or 256 bit hash values of given messages.
|
||||
|
||||
It can be used to prevent hash-flooding attacks or authenticate short-lived messages. Additionally it can be used as a fingerprinting function. HighwayHash is not a general purpose cryptographic hash function (such as Blake2b, SHA-3 or SHA-2) and should not be used if strong collision resistance is required.
|
||||
|
||||
This repository contains a native Go version and optimized assembly implementations for Intel, ARM and ppc64le architectures.
|
||||
|
||||
### High performance
|
||||
|
||||
HighwayHash is an approximately 5x faster SIMD hash function as compared to [SipHash](https://www.131002.net/siphash/siphash.pdf) which in itself is a fast and 'cryptographically strong' pseudo-random function designed by Aumasson and Bernstein.
|
||||
|
||||
HighwayHash uses a new way of mixing inputs with AVX2 multiply and permute instructions. The multiplications are 32x32 bit giving 64 bits-wide results and are therefore infeasible to reverse. Additionally permuting equalizes the distribution of the resulting bytes. The algorithm outputs digests ranging from 64 bits up to 256 bits at no extra cost.
|
||||
|
||||
### Stable
|
||||
|
||||
All three output sizes of HighwayHash have been declared [stable](https://github.com/google/highwayhash/#versioning-and-stability) as of January 2018. This means that the hash results for any given input message are guaranteed not to change.
|
||||
|
||||
### Installation
|
||||
|
||||
Install: `go get -u github.com/minio/highwayhash`
|
||||
|
||||
### Intel Performance
|
||||
|
||||
Below are the single core results on an Intel Core i7 (3.1 GHz) for 256 bit outputs:
|
||||
|
||||
```
|
||||
BenchmarkSum256_16 204.17 MB/s
|
||||
BenchmarkSum256_64 1040.63 MB/s
|
||||
BenchmarkSum256_1K 8653.30 MB/s
|
||||
BenchmarkSum256_8K 13476.07 MB/s
|
||||
BenchmarkSum256_1M 14928.71 MB/s
|
||||
BenchmarkSum256_5M 14180.04 MB/s
|
||||
BenchmarkSum256_10M 12458.65 MB/s
|
||||
BenchmarkSum256_25M 11927.25 MB/s
|
||||
```
|
||||
|
||||
So for moderately sized messages it tops out at about 15 GB/sec. Also for small messages (1K) the performance is already at approximately 60% of the maximum throughput.
|
||||
|
||||
### ARM Performance
|
||||
|
||||
Below are the single core results on an EC2 c7g.4xlarge (Graviton3) instance for 256 bit outputs:
|
||||
|
||||
```
|
||||
BenchmarkSum256_16 143.66 MB/s
|
||||
BenchmarkSum256_64 628.75 MB/s
|
||||
BenchmarkSum256_1K 3621.71 MB/s
|
||||
BenchmarkSum256_8K 5039.64 MB/s
|
||||
BenchmarkSum256_1M 5279.79 MB/s
|
||||
BenchmarkSum256_5M 5474.60 MB/s
|
||||
BenchmarkSum256_10M 5621.73 MB/s
|
||||
BenchmarkSum256_25M 5250.47 MB/s
|
||||
```
|
||||
|
||||
### ppc64le Performance
|
||||
|
||||
The ppc64le accelerated version is roughly 10x faster compared to the non-optimized version:
|
||||
|
||||
```
|
||||
benchmark old MB/s new MB/s speedup
|
||||
BenchmarkWrite_8K 531.19 5566.41 10.48x
|
||||
BenchmarkSum64_8K 518.86 4971.88 9.58x
|
||||
BenchmarkSum256_8K 502.45 4474.20 8.90x
|
||||
```
|
||||
|
||||
### Performance compared to other hashing techniques
|
||||
|
||||
On a Skylake CPU (3.0 GHz Xeon Platinum 8124M) the table below shows how HighwayHash compares to other hashing techniques for 5 MB messages (single core performance, all Golang implementations, see [benchmark](https://github.com/fwessels/HashCompare/blob/master/benchmarks_test.go)).
|
||||
|
||||
```
|
||||
BenchmarkHighwayHash 11986.98 MB/s
|
||||
BenchmarkSHA256_AVX512 3552.74 MB/s
|
||||
BenchmarkBlake2b 972.38 MB/s
|
||||
BenchmarkSHA1 950.64 MB/s
|
||||
BenchmarkMD5 684.18 MB/s
|
||||
BenchmarkSHA512 562.04 MB/s
|
||||
BenchmarkSHA256 383.07 MB/s
|
||||
```
|
||||
|
||||
*Note: the AVX512 version of SHA256 uses the [multi-buffer crypto library](https://github.com/intel/intel-ipsec-mb) technique as developed by Intel, more details can be found in [sha256-simd](https://github.com/minio/sha256-simd/).*
|
||||
|
||||
### Qualitative assessment
|
||||
|
||||
We have performed a 'qualitative' assessment of how HighwayHash compares to Blake2b in terms of the distribution of the checksums for varying numbers of messages. It shows that HighwayHash behaves similarly according to the following graph:
|
||||
|
||||
|
||||
|
||||
More information can be found in [HashCompare](https://github.com/fwessels/HashCompare).
|
||||
|
||||
### Requirements
|
||||
|
||||
All Go versions >= 1.11 are supported (needed for required assembly support for the different platforms).
|
||||
|
||||
### Contributing
|
||||
|
||||
Contributions are welcome, please send PRs for any enhancements.
|
||||
+250
@@ -0,0 +1,250 @@
|
||||
// Copyright (c) 2017 Minio Inc. All rights reserved.
|
||||
// Use of this source code is governed by a license that can be
|
||||
// found in the LICENSE file.
|
||||
|
||||
// Package highwayhash implements the pseudo-random-function (PRF) HighwayHash.
|
||||
// HighwayHash is a fast hash function designed to defend hash-flooding attacks
|
||||
// or to authenticate short-lived messages.
|
||||
//
|
||||
// HighwayHash is not a general purpose cryptographic hash function and does not
|
||||
// provide (strong) collision resistance.
|
||||
package highwayhash
|
||||
|
||||
import (
|
||||
"encoding/binary"
|
||||
"errors"
|
||||
"hash"
|
||||
)
|
||||
|
||||
const (
|
||||
// Size is the size of HighwayHash-256 checksum in bytes.
|
||||
Size = 32
|
||||
// Size128 is the size of HighwayHash-128 checksum in bytes.
|
||||
Size128 = 16
|
||||
// Size64 is the size of HighwayHash-64 checksum in bytes.
|
||||
Size64 = 8
|
||||
)
|
||||
|
||||
// These will error at compile time if the interface is not conformant.
|
||||
var _ hash.Hash = &Digest{}
|
||||
var _ hash.Hash = &Digest64{}
|
||||
|
||||
var errKeySize = errors.New("highwayhash: invalid key size")
|
||||
|
||||
// New returns a hash.Hash computing the HighwayHash-256 checksum.
|
||||
// It returns a non-nil error if the key is not 32 bytes long.
|
||||
func New(key []byte) (hash.Hash, error) {
|
||||
return NewDigest(key)
|
||||
}
|
||||
|
||||
// New128 returns a hash.Hash computing the HighwayHash-128 checksum.
|
||||
// It returns a non-nil error if the key is not 32 bytes long.
|
||||
func New128(key []byte) (hash.Hash, error) {
|
||||
return NewDigest128(key)
|
||||
}
|
||||
|
||||
// New64 returns a hash.Hash64 computing the HighwayHash-64 checksum.
|
||||
// It returns a non-nil error if the key is not 32 bytes long.
|
||||
func New64(key []byte) (hash.Hash64, error) {
|
||||
return NewDigest64(key)
|
||||
}
|
||||
|
||||
// NewDigest returns a *Digest that conforms to hash.Hash computing
|
||||
// the HighwayHash-256 checksum.
|
||||
// It returns a non-nil error if the key is not 32 bytes long.
|
||||
func NewDigest(key []byte) (*Digest, error) {
|
||||
if len(key) != Size {
|
||||
return nil, errKeySize
|
||||
}
|
||||
h := &Digest{size: Size}
|
||||
copy(h.key[:], key)
|
||||
h.Reset()
|
||||
return h, nil
|
||||
}
|
||||
|
||||
// NewDigest128 returns a *Digest that conforms to hash.Hash computing
|
||||
// the HighwayHash-128 checksum.
|
||||
// It returns a non-nil error if the key is not 32 bytes long.
|
||||
func NewDigest128(key []byte) (*Digest, error) {
|
||||
if len(key) != Size {
|
||||
return nil, errKeySize
|
||||
}
|
||||
h := &Digest{size: Size128}
|
||||
copy(h.key[:], key)
|
||||
h.Reset()
|
||||
return h, nil
|
||||
}
|
||||
|
||||
// NewDigest64 returns a *Digest that conforms to hash.Hash computing
|
||||
// the HighwayHash-64 checksum.
|
||||
// It returns a non-nil error if the key is not 32 bytes long.
|
||||
func NewDigest64(key []byte) (*Digest64, error) {
|
||||
if len(key) != Size {
|
||||
return nil, errKeySize
|
||||
}
|
||||
h := new(Digest64)
|
||||
h.size = Size64
|
||||
copy(h.key[:], key)
|
||||
h.Reset()
|
||||
return h, nil
|
||||
}
|
||||
|
||||
// Sum computes the HighwayHash-256 checksum of data.
|
||||
// It panics if the key is not 32 bytes long.
|
||||
func Sum(data, key []byte) [Size]byte {
|
||||
if len(key) != Size {
|
||||
panic(errKeySize)
|
||||
}
|
||||
var state [16]uint64
|
||||
initialize(&state, key)
|
||||
if n := len(data) & (^(Size - 1)); n > 0 {
|
||||
update(&state, data[:n])
|
||||
data = data[n:]
|
||||
}
|
||||
if len(data) > 0 {
|
||||
var block [Size]byte
|
||||
offset := copy(block[:], data)
|
||||
hashBuffer(&state, &block, offset)
|
||||
}
|
||||
var hash [Size]byte
|
||||
finalize(hash[:], &state)
|
||||
return hash
|
||||
}
|
||||
|
||||
// Sum128 computes the HighwayHash-128 checksum of data.
|
||||
// It panics if the key is not 32 bytes long.
|
||||
func Sum128(data, key []byte) [Size128]byte {
|
||||
if len(key) != Size {
|
||||
panic(errKeySize)
|
||||
}
|
||||
var state [16]uint64
|
||||
initialize(&state, key)
|
||||
if n := len(data) & (^(Size - 1)); n > 0 {
|
||||
update(&state, data[:n])
|
||||
data = data[n:]
|
||||
}
|
||||
if len(data) > 0 {
|
||||
var block [Size]byte
|
||||
offset := copy(block[:], data)
|
||||
hashBuffer(&state, &block, offset)
|
||||
}
|
||||
var hash [Size128]byte
|
||||
finalize(hash[:], &state)
|
||||
return hash
|
||||
}
|
||||
|
||||
// Sum64 computes the HighwayHash-64 checksum of data.
|
||||
// It panics if the key is not 32 bytes long.
|
||||
func Sum64(data, key []byte) uint64 {
|
||||
if len(key) != Size {
|
||||
panic(errKeySize)
|
||||
}
|
||||
var state [16]uint64
|
||||
initialize(&state, key)
|
||||
if n := len(data) & (^(Size - 1)); n > 0 {
|
||||
update(&state, data[:n])
|
||||
data = data[n:]
|
||||
}
|
||||
if len(data) > 0 {
|
||||
var block [Size]byte
|
||||
offset := copy(block[:], data)
|
||||
hashBuffer(&state, &block, offset)
|
||||
}
|
||||
var hash [Size64]byte
|
||||
finalize(hash[:], &state)
|
||||
return binary.LittleEndian.Uint64(hash[:])
|
||||
}
|
||||
|
||||
type Digest64 struct{ Digest }
|
||||
|
||||
func (d *Digest64) Sum64() uint64 {
|
||||
state := d.state
|
||||
if d.offset > 0 {
|
||||
hashBuffer(&state, &d.buffer, d.offset)
|
||||
}
|
||||
var hash [8]byte
|
||||
finalize(hash[:], &state)
|
||||
return binary.LittleEndian.Uint64(hash[:])
|
||||
}
|
||||
|
||||
type Digest struct {
|
||||
state [16]uint64 // v0 | v1 | mul0 | mul1
|
||||
|
||||
key, buffer [Size]byte
|
||||
offset int
|
||||
|
||||
size int
|
||||
}
|
||||
|
||||
func (d *Digest) Size() int { return d.size }
|
||||
|
||||
func (d *Digest) BlockSize() int { return Size }
|
||||
|
||||
func (d *Digest) Reset() {
|
||||
initialize(&d.state, d.key[:])
|
||||
d.offset = 0
|
||||
}
|
||||
|
||||
func (d *Digest) Write(p []byte) (n int, err error) {
|
||||
n = len(p)
|
||||
if d.offset > 0 {
|
||||
remaining := Size - d.offset
|
||||
if n < remaining {
|
||||
d.offset += copy(d.buffer[d.offset:], p)
|
||||
return
|
||||
}
|
||||
copy(d.buffer[d.offset:], p[:remaining])
|
||||
update(&d.state, d.buffer[:])
|
||||
p = p[remaining:]
|
||||
d.offset = 0
|
||||
}
|
||||
if nn := len(p) & (^(Size - 1)); nn > 0 {
|
||||
update(&d.state, p[:nn])
|
||||
p = p[nn:]
|
||||
}
|
||||
if len(p) > 0 {
|
||||
d.offset = copy(d.buffer[d.offset:], p)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func (d *Digest) Sum(b []byte) []byte {
|
||||
state := d.state
|
||||
if d.offset > 0 {
|
||||
hashBuffer(&state, &d.buffer, d.offset)
|
||||
}
|
||||
var hash [Size]byte
|
||||
finalize(hash[:d.size], &state)
|
||||
return append(b, hash[:d.size]...)
|
||||
}
|
||||
|
||||
func hashBuffer(state *[16]uint64, buffer *[32]byte, offset int) {
|
||||
var block [Size]byte
|
||||
mod32 := (uint64(offset) << 32) + uint64(offset)
|
||||
for i := range state[:4] {
|
||||
state[i] += mod32
|
||||
}
|
||||
for i := range state[4:8] {
|
||||
t0 := uint32(state[i+4])
|
||||
t0 = (t0 << uint(offset)) | (t0 >> uint(32-offset))
|
||||
|
||||
t1 := uint32(state[i+4] >> 32)
|
||||
t1 = (t1 << uint(offset)) | (t1 >> uint(32-offset))
|
||||
|
||||
state[i+4] = (uint64(t1) << 32) | uint64(t0)
|
||||
}
|
||||
|
||||
mod4 := offset & 3
|
||||
remain := offset - mod4
|
||||
|
||||
copy(block[:], buffer[:remain])
|
||||
if offset >= 16 {
|
||||
copy(block[28:], buffer[offset-4:])
|
||||
} else if mod4 != 0 {
|
||||
last := uint32(buffer[remain])
|
||||
last += uint32(buffer[remain+mod4>>1]) << 8
|
||||
last += uint32(buffer[offset-1]) << 16
|
||||
binary.LittleEndian.PutUint32(block[16:], last)
|
||||
}
|
||||
update(state, block[:])
|
||||
}
|
||||
+248
@@ -0,0 +1,248 @@
|
||||
// Copyright (c) 2017 Minio Inc. All rights reserved.
|
||||
// Use of this source code is governed by a license that can be
|
||||
// found in the LICENSE file.
|
||||
|
||||
// +build amd64,!gccgo,!appengine,!nacl,!noasm
|
||||
|
||||
#include "textflag.h"
|
||||
|
||||
DATA ·consAVX2<>+0x00(SB)/8, $0xdbe6d5d5fe4cce2f
|
||||
DATA ·consAVX2<>+0x08(SB)/8, $0xa4093822299f31d0
|
||||
DATA ·consAVX2<>+0x10(SB)/8, $0x13198a2e03707344
|
||||
DATA ·consAVX2<>+0x18(SB)/8, $0x243f6a8885a308d3
|
||||
DATA ·consAVX2<>+0x20(SB)/8, $0x3bd39e10cb0ef593
|
||||
DATA ·consAVX2<>+0x28(SB)/8, $0xc0acf169b5f18a8c
|
||||
DATA ·consAVX2<>+0x30(SB)/8, $0xbe5466cf34e90c6c
|
||||
DATA ·consAVX2<>+0x38(SB)/8, $0x452821e638d01377
|
||||
GLOBL ·consAVX2<>(SB), (NOPTR+RODATA), $64
|
||||
|
||||
DATA ·zipperMergeAVX2<>+0x00(SB)/8, $0xf010e05020c03
|
||||
DATA ·zipperMergeAVX2<>+0x08(SB)/8, $0x70806090d0a040b
|
||||
DATA ·zipperMergeAVX2<>+0x10(SB)/8, $0xf010e05020c03
|
||||
DATA ·zipperMergeAVX2<>+0x18(SB)/8, $0x70806090d0a040b
|
||||
GLOBL ·zipperMergeAVX2<>(SB), (NOPTR+RODATA), $32
|
||||
|
||||
#define REDUCE_MOD(x0, x1, x2, x3, tmp0, tmp1, y0, y1) \
|
||||
MOVQ $0x3FFFFFFFFFFFFFFF, tmp0 \
|
||||
ANDQ tmp0, x3 \
|
||||
MOVQ x2, y0 \
|
||||
MOVQ x3, y1 \
|
||||
\
|
||||
MOVQ x2, tmp0 \
|
||||
MOVQ x3, tmp1 \
|
||||
SHLQ $1, tmp1 \
|
||||
SHRQ $63, tmp0 \
|
||||
MOVQ tmp1, x3 \
|
||||
ORQ tmp0, x3 \
|
||||
\
|
||||
SHLQ $1, x2 \
|
||||
\
|
||||
MOVQ y0, tmp0 \
|
||||
MOVQ y1, tmp1 \
|
||||
SHLQ $2, tmp1 \
|
||||
SHRQ $62, tmp0 \
|
||||
MOVQ tmp1, y1 \
|
||||
ORQ tmp0, y1 \
|
||||
\
|
||||
SHLQ $2, y0 \
|
||||
\
|
||||
XORQ x0, y0 \
|
||||
XORQ x2, y0 \
|
||||
XORQ x1, y1 \
|
||||
XORQ x3, y1
|
||||
|
||||
#define UPDATE(msg) \
|
||||
VPADDQ msg, Y2, Y2 \
|
||||
VPADDQ Y3, Y2, Y2 \
|
||||
\
|
||||
VPSRLQ $32, Y1, Y0 \
|
||||
BYTE $0xC5; BYTE $0xFD; BYTE $0xF4; BYTE $0xC2 \ // VPMULUDQ Y2, Y0, Y0
|
||||
VPXOR Y0, Y3, Y3 \
|
||||
\
|
||||
VPADDQ Y4, Y1, Y1 \
|
||||
\
|
||||
VPSRLQ $32, Y2, Y0 \
|
||||
BYTE $0xC5; BYTE $0xFD; BYTE $0xF4; BYTE $0xC1 \ // VPMULUDQ Y1, Y0, Y0
|
||||
VPXOR Y0, Y4, Y4 \
|
||||
\
|
||||
VPSHUFB Y5, Y2, Y0 \
|
||||
VPADDQ Y0, Y1, Y1 \
|
||||
\
|
||||
VPSHUFB Y5, Y1, Y0 \
|
||||
VPADDQ Y0, Y2, Y2
|
||||
|
||||
// func initializeAVX2(state *[16]uint64, key []byte)
|
||||
TEXT ·initializeAVX2(SB), 4, $0-32
|
||||
MOVQ state+0(FP), AX
|
||||
MOVQ key_base+8(FP), BX
|
||||
MOVQ $·consAVX2<>(SB), CX
|
||||
|
||||
VMOVDQU 0(BX), Y1
|
||||
VPSHUFD $177, Y1, Y2
|
||||
|
||||
VMOVDQU 0(CX), Y3
|
||||
VMOVDQU 32(CX), Y4
|
||||
|
||||
VPXOR Y3, Y1, Y1
|
||||
VPXOR Y4, Y2, Y2
|
||||
|
||||
VMOVDQU Y1, 0(AX)
|
||||
VMOVDQU Y2, 32(AX)
|
||||
VMOVDQU Y3, 64(AX)
|
||||
VMOVDQU Y4, 96(AX)
|
||||
VZEROUPPER
|
||||
RET
|
||||
|
||||
// func updateAVX2(state *[16]uint64, msg []byte)
|
||||
TEXT ·updateAVX2(SB), 4, $0-32
|
||||
MOVQ state+0(FP), AX
|
||||
MOVQ msg_base+8(FP), BX
|
||||
MOVQ msg_len+16(FP), CX
|
||||
|
||||
CMPQ CX, $32
|
||||
JB DONE
|
||||
|
||||
VMOVDQU 0(AX), Y1
|
||||
VMOVDQU 32(AX), Y2
|
||||
VMOVDQU 64(AX), Y3
|
||||
VMOVDQU 96(AX), Y4
|
||||
|
||||
VMOVDQU ·zipperMergeAVX2<>(SB), Y5
|
||||
|
||||
LOOP:
|
||||
VMOVDQU 0(BX), Y0
|
||||
UPDATE(Y0)
|
||||
|
||||
ADDQ $32, BX
|
||||
SUBQ $32, CX
|
||||
JA LOOP
|
||||
|
||||
VMOVDQU Y1, 0(AX)
|
||||
VMOVDQU Y2, 32(AX)
|
||||
VMOVDQU Y3, 64(AX)
|
||||
VMOVDQU Y4, 96(AX)
|
||||
VZEROUPPER
|
||||
|
||||
DONE:
|
||||
RET
|
||||
|
||||
// func finalizeAVX2(out []byte, state *[16]uint64)
|
||||
TEXT ·finalizeAVX2(SB), 4, $0-32
|
||||
MOVQ state+24(FP), AX
|
||||
MOVQ out_base+0(FP), BX
|
||||
MOVQ out_len+8(FP), CX
|
||||
|
||||
VMOVDQU 0(AX), Y1
|
||||
VMOVDQU 32(AX), Y2
|
||||
VMOVDQU 64(AX), Y3
|
||||
VMOVDQU 96(AX), Y4
|
||||
|
||||
VMOVDQU ·zipperMergeAVX2<>(SB), Y5
|
||||
|
||||
VPERM2I128 $1, Y1, Y1, Y0
|
||||
VPSHUFD $177, Y0, Y0
|
||||
UPDATE(Y0)
|
||||
|
||||
VPERM2I128 $1, Y1, Y1, Y0
|
||||
VPSHUFD $177, Y0, Y0
|
||||
UPDATE(Y0)
|
||||
|
||||
VPERM2I128 $1, Y1, Y1, Y0
|
||||
VPSHUFD $177, Y0, Y0
|
||||
UPDATE(Y0)
|
||||
|
||||
VPERM2I128 $1, Y1, Y1, Y0
|
||||
VPSHUFD $177, Y0, Y0
|
||||
UPDATE(Y0)
|
||||
|
||||
CMPQ CX, $8
|
||||
JE skipUpdate // Just 4 rounds for 64-bit checksum
|
||||
|
||||
VPERM2I128 $1, Y1, Y1, Y0
|
||||
VPSHUFD $177, Y0, Y0
|
||||
UPDATE(Y0)
|
||||
|
||||
VPERM2I128 $1, Y1, Y1, Y0
|
||||
VPSHUFD $177, Y0, Y0
|
||||
UPDATE(Y0)
|
||||
|
||||
CMPQ CX, $16
|
||||
JE skipUpdate // 6 rounds for 128-bit checksum
|
||||
|
||||
VPERM2I128 $1, Y1, Y1, Y0
|
||||
VPSHUFD $177, Y0, Y0
|
||||
UPDATE(Y0)
|
||||
|
||||
VPERM2I128 $1, Y1, Y1, Y0
|
||||
VPSHUFD $177, Y0, Y0
|
||||
UPDATE(Y0)
|
||||
|
||||
VPERM2I128 $1, Y1, Y1, Y0
|
||||
VPSHUFD $177, Y0, Y0
|
||||
UPDATE(Y0)
|
||||
|
||||
VPERM2I128 $1, Y1, Y1, Y0
|
||||
VPSHUFD $177, Y0, Y0
|
||||
UPDATE(Y0)
|
||||
|
||||
skipUpdate:
|
||||
VMOVDQU Y1, 0(AX)
|
||||
VMOVDQU Y2, 32(AX)
|
||||
VMOVDQU Y3, 64(AX)
|
||||
VMOVDQU Y4, 96(AX)
|
||||
VZEROUPPER
|
||||
|
||||
CMPQ CX, $8
|
||||
JE hash64
|
||||
CMPQ CX, $16
|
||||
JE hash128
|
||||
|
||||
// 256-bit checksum
|
||||
MOVQ 0*8(AX), R8
|
||||
MOVQ 1*8(AX), R9
|
||||
MOVQ 4*8(AX), R10
|
||||
MOVQ 5*8(AX), R11
|
||||
ADDQ 8*8(AX), R8
|
||||
ADDQ 9*8(AX), R9
|
||||
ADDQ 12*8(AX), R10
|
||||
ADDQ 13*8(AX), R11
|
||||
|
||||
REDUCE_MOD(R8, R9, R10, R11, R12, R13, R14, R15)
|
||||
MOVQ R14, 0(BX)
|
||||
MOVQ R15, 8(BX)
|
||||
|
||||
MOVQ 2*8(AX), R8
|
||||
MOVQ 3*8(AX), R9
|
||||
MOVQ 6*8(AX), R10
|
||||
MOVQ 7*8(AX), R11
|
||||
ADDQ 10*8(AX), R8
|
||||
ADDQ 11*8(AX), R9
|
||||
ADDQ 14*8(AX), R10
|
||||
ADDQ 15*8(AX), R11
|
||||
|
||||
REDUCE_MOD(R8, R9, R10, R11, R12, R13, R14, R15)
|
||||
MOVQ R14, 16(BX)
|
||||
MOVQ R15, 24(BX)
|
||||
RET
|
||||
|
||||
hash128:
|
||||
MOVQ 0*8(AX), R8
|
||||
MOVQ 1*8(AX), R9
|
||||
ADDQ 6*8(AX), R8
|
||||
ADDQ 7*8(AX), R9
|
||||
ADDQ 8*8(AX), R8
|
||||
ADDQ 9*8(AX), R9
|
||||
ADDQ 14*8(AX), R8
|
||||
ADDQ 15*8(AX), R9
|
||||
MOVQ R8, 0(BX)
|
||||
MOVQ R9, 8(BX)
|
||||
RET
|
||||
|
||||
hash64:
|
||||
MOVQ 0*8(AX), DX
|
||||
ADDQ 4*8(AX), DX
|
||||
ADDQ 8*8(AX), DX
|
||||
ADDQ 12*8(AX), DX
|
||||
MOVQ DX, 0(BX)
|
||||
RET
|
||||
|
||||
+132
@@ -0,0 +1,132 @@
|
||||
//
|
||||
// Copyright (c) 2024 Minio Inc. All rights reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
//
|
||||
|
||||
//+build !noasm,!appengine
|
||||
|
||||
#include "textflag.h"
|
||||
|
||||
TEXT ·getVectorLength(SB), NOSPLIT, $0
|
||||
WORD $0xd2800002 // mov x2, #0
|
||||
WORD $0x04225022 // addvl x2, x2, #1
|
||||
WORD $0xd37df042 // lsl x2, x2, #3
|
||||
WORD $0xd2800003 // mov x3, #0
|
||||
WORD $0x04635023 // addpl x3, x3, #1
|
||||
WORD $0xd37df063 // lsl x3, x3, #3
|
||||
MOVD R2, vl+0(FP)
|
||||
MOVD R3, pl+8(FP)
|
||||
RET
|
||||
|
||||
TEXT ·updateArm64Sve(SB), NOSPLIT, $0
|
||||
MOVD state+0(FP), R0
|
||||
MOVD msg_base+8(FP), R1
|
||||
MOVD msg_len+16(FP), R2 // length of message
|
||||
SUBS $32, R2
|
||||
BMI completeSve
|
||||
|
||||
WORD $0x2518e3e1 // ptrue p1.b
|
||||
WORD $0xa5e0a401 // ld1d z1.d, p1/z, [x0]
|
||||
WORD $0xa5e1a402 // ld1d z2.d, p1/z, [x0, #1, MUL VL]
|
||||
WORD $0xa5e2a403 // ld1d z3.d, p1/z, [x0, #2, MUL VL]
|
||||
WORD $0xa5e3a404 // ld1d z4.d, p1/z, [x0, #3, MUL VL]
|
||||
|
||||
// Load zipper merge constants table pointer
|
||||
MOVD $·zipperMergeSve(SB), R3
|
||||
WORD $0xa5e0a465 // ld1d z5.d, p1/z, [x3]
|
||||
WORD $0x25b8c006 // mov z6.s, #0
|
||||
WORD $0x25d8e3e2 // ptrue p2.d /* set every other lane for "s" type */
|
||||
|
||||
loopSve:
|
||||
WORD $0xa5e0a420 // ld1d z0.d, p1/z, [x1]
|
||||
ADD $32, R1
|
||||
|
||||
WORD $0x04e00042 // add z2.d, z2.d, z0.d
|
||||
WORD $0x04e30042 // add z2.d, z2.d, z3.d
|
||||
WORD $0x04e09420 // lsr z0.d, z1.d, #32
|
||||
WORD $0x05a6c847 // sel z7.s, p2, z2.s, z6.s
|
||||
WORD $0x04d004e0 // mul z0.d, p1/m, z0.d, z7.d
|
||||
WORD $0x04a33003 // eor z3.d, z0.d, z3.d
|
||||
WORD $0x04e10081 // add z1.d, z4.d, z1.d
|
||||
WORD $0x04e09440 // lsr z0.d, z2.d, #32
|
||||
WORD $0x05a6c827 // sel z7.s, p2, z1.s, z6.s
|
||||
WORD $0x04d004e0 // mul z0.d, p1/m, z0.d, z7.d
|
||||
WORD $0x04a43004 // eor z4.d, z0.d, z4.d
|
||||
WORD $0x05253040 // tbl z0.b, z2.b, z5.b
|
||||
WORD $0x04e00021 // add z1.d, z1.d, z0.d
|
||||
WORD $0x05253020 // tbl z0.b, z1.b, z5.b
|
||||
WORD $0x04e00042 // add z2.d, z2.d, z0.d
|
||||
|
||||
SUBS $32, R2
|
||||
BPL loopSve
|
||||
|
||||
WORD $0xe5e0e401 // st1d z1.d, p1, [x0]
|
||||
WORD $0xe5e1e402 // st1d z2.d, p1, [x0, #1, MUL VL]
|
||||
WORD $0xe5e2e403 // st1d z3.d, p1, [x0, #2, MUL VL]
|
||||
WORD $0xe5e3e404 // st1d z4.d, p1, [x0, #3, MUL VL]
|
||||
|
||||
completeSve:
|
||||
RET
|
||||
|
||||
TEXT ·updateArm64Sve2(SB), NOSPLIT, $0
|
||||
MOVD state+0(FP), R0
|
||||
MOVD msg_base+8(FP), R1
|
||||
MOVD msg_len+16(FP), R2 // length of message
|
||||
SUBS $32, R2
|
||||
BMI completeSve2
|
||||
|
||||
WORD $0x2518e3e1 // ptrue p1.b
|
||||
WORD $0xa5e0a401 // ld1d z1.d, p1/z, [x0]
|
||||
WORD $0xa5e1a402 // ld1d z2.d, p1/z, [x0, #1, MUL VL]
|
||||
WORD $0xa5e2a403 // ld1d z3.d, p1/z, [x0, #2, MUL VL]
|
||||
WORD $0xa5e3a404 // ld1d z4.d, p1/z, [x0, #3, MUL VL]
|
||||
|
||||
// Load zipper merge constants table pointer
|
||||
MOVD $·zipperMergeSve(SB), R3
|
||||
WORD $0xa5e0a465 // ld1d z5.d, p1/z, [x3]
|
||||
|
||||
loopSve2:
|
||||
WORD $0xa5e0a420 // ld1d z0.d, p1/z, [x1]
|
||||
ADD $32, R1
|
||||
|
||||
WORD $0x04e00042 // add z2.d, z2.d, z0.d
|
||||
WORD $0x04e30042 // add z2.d, z2.d, z3.d
|
||||
WORD $0x04e09420 // lsr z0.d, z1.d, #32
|
||||
WORD $0x45c27800 // umullb z0.d, z0.s, z2.s
|
||||
WORD $0x04a33003 // eor z3.d, z0.d, z3.d
|
||||
WORD $0x04e10081 // add z1.d, z4.d, z1.d
|
||||
WORD $0x04e09440 // lsr z0.d, z2.d, #32
|
||||
WORD $0x45c17800 // umullb z0.d, z0.s, z1.s
|
||||
WORD $0x04a43004 // eor z4.d, z0.d, z4.d
|
||||
WORD $0x05253040 // tbl z0.b, z2.b, z5.b
|
||||
WORD $0x04e00021 // add z1.d, z1.d, z0.d
|
||||
WORD $0x05253020 // tbl z0.b, z1.b, z5.b
|
||||
WORD $0x04e00042 // add z2.d, z2.d, z0.d
|
||||
|
||||
SUBS $32, R2
|
||||
BPL loopSve2
|
||||
|
||||
WORD $0xe5e0e401 // st1d z1.d, p1, [x0]
|
||||
WORD $0xe5e1e402 // st1d z2.d, p1, [x0, #1, MUL VL]
|
||||
WORD $0xe5e2e403 // st1d z3.d, p1, [x0, #2, MUL VL]
|
||||
WORD $0xe5e3e404 // st1d z4.d, p1, [x0, #3, MUL VL]
|
||||
|
||||
completeSve2:
|
||||
RET
|
||||
|
||||
DATA ·zipperMergeSve+0x00(SB)/8, $0x000f010e05020c03
|
||||
DATA ·zipperMergeSve+0x08(SB)/8, $0x070806090d0a040b
|
||||
DATA ·zipperMergeSve+0x10(SB)/8, $0x101f111e15121c13
|
||||
DATA ·zipperMergeSve+0x18(SB)/8, $0x171816191d1a141b
|
||||
GLOBL ·zipperMergeSve(SB), (NOPTR+RODATA), $32
|
||||
+70
@@ -0,0 +1,70 @@
|
||||
// Copyright (c) 2017 Minio Inc. All rights reserved.
|
||||
// Use of this source code is governed by a license that can be
|
||||
// found in the LICENSE file.
|
||||
|
||||
//go:build amd64 && !gccgo && !appengine && !nacl && !noasm
|
||||
// +build amd64,!gccgo,!appengine,!nacl,!noasm
|
||||
|
||||
package highwayhash
|
||||
|
||||
import "golang.org/x/sys/cpu"
|
||||
|
||||
var (
|
||||
useSSE4 = cpu.X86.HasSSE41
|
||||
useAVX2 = cpu.X86.HasAVX2
|
||||
useNEON = false
|
||||
useSVE = false
|
||||
useSVE2 = false
|
||||
useVMX = false
|
||||
)
|
||||
|
||||
//go:noescape
|
||||
func initializeSSE4(state *[16]uint64, key []byte)
|
||||
|
||||
//go:noescape
|
||||
func initializeAVX2(state *[16]uint64, key []byte)
|
||||
|
||||
//go:noescape
|
||||
func updateSSE4(state *[16]uint64, msg []byte)
|
||||
|
||||
//go:noescape
|
||||
func updateAVX2(state *[16]uint64, msg []byte)
|
||||
|
||||
//go:noescape
|
||||
func finalizeSSE4(out []byte, state *[16]uint64)
|
||||
|
||||
//go:noescape
|
||||
func finalizeAVX2(out []byte, state *[16]uint64)
|
||||
|
||||
func initialize(state *[16]uint64, key []byte) {
|
||||
switch {
|
||||
case useAVX2:
|
||||
initializeAVX2(state, key)
|
||||
case useSSE4:
|
||||
initializeSSE4(state, key)
|
||||
default:
|
||||
initializeGeneric(state, key)
|
||||
}
|
||||
}
|
||||
|
||||
func update(state *[16]uint64, msg []byte) {
|
||||
switch {
|
||||
case useAVX2:
|
||||
updateAVX2(state, msg)
|
||||
case useSSE4:
|
||||
updateSSE4(state, msg)
|
||||
default:
|
||||
updateGeneric(state, msg)
|
||||
}
|
||||
}
|
||||
|
||||
func finalize(out []byte, state *[16]uint64) {
|
||||
switch {
|
||||
case useAVX2:
|
||||
finalizeAVX2(out, state)
|
||||
case useSSE4:
|
||||
finalizeSSE4(out, state)
|
||||
default:
|
||||
finalizeGeneric(out, state)
|
||||
}
|
||||
}
|
||||
+294
@@ -0,0 +1,294 @@
|
||||
// Copyright (c) 2017 Minio Inc. All rights reserved.
|
||||
// Use of this source code is governed by a license that can be
|
||||
// found in the LICENSE file.
|
||||
|
||||
// +build amd64 !gccgo !appengine !nacl
|
||||
|
||||
#include "textflag.h"
|
||||
|
||||
DATA ·asmConstants<>+0x00(SB)/8, $0xdbe6d5d5fe4cce2f
|
||||
DATA ·asmConstants<>+0x08(SB)/8, $0xa4093822299f31d0
|
||||
DATA ·asmConstants<>+0x10(SB)/8, $0x13198a2e03707344
|
||||
DATA ·asmConstants<>+0x18(SB)/8, $0x243f6a8885a308d3
|
||||
DATA ·asmConstants<>+0x20(SB)/8, $0x3bd39e10cb0ef593
|
||||
DATA ·asmConstants<>+0x28(SB)/8, $0xc0acf169b5f18a8c
|
||||
DATA ·asmConstants<>+0x30(SB)/8, $0xbe5466cf34e90c6c
|
||||
DATA ·asmConstants<>+0x38(SB)/8, $0x452821e638d01377
|
||||
GLOBL ·asmConstants<>(SB), (NOPTR+RODATA), $64
|
||||
|
||||
DATA ·asmZipperMerge<>+0x00(SB)/8, $0xf010e05020c03
|
||||
DATA ·asmZipperMerge<>+0x08(SB)/8, $0x70806090d0a040b
|
||||
GLOBL ·asmZipperMerge<>(SB), (NOPTR+RODATA), $16
|
||||
|
||||
#define v00 X0
|
||||
#define v01 X1
|
||||
#define v10 X2
|
||||
#define v11 X3
|
||||
#define m00 X4
|
||||
#define m01 X5
|
||||
#define m10 X6
|
||||
#define m11 X7
|
||||
|
||||
#define t0 X8
|
||||
#define t1 X9
|
||||
#define t2 X10
|
||||
|
||||
#define REDUCE_MOD(x0, x1, x2, x3, tmp0, tmp1, y0, y1) \
|
||||
MOVQ $0x3FFFFFFFFFFFFFFF, tmp0 \
|
||||
ANDQ tmp0, x3 \
|
||||
MOVQ x2, y0 \
|
||||
MOVQ x3, y1 \
|
||||
\
|
||||
MOVQ x2, tmp0 \
|
||||
MOVQ x3, tmp1 \
|
||||
SHLQ $1, tmp1 \
|
||||
SHRQ $63, tmp0 \
|
||||
MOVQ tmp1, x3 \
|
||||
ORQ tmp0, x3 \
|
||||
\
|
||||
SHLQ $1, x2 \
|
||||
\
|
||||
MOVQ y0, tmp0 \
|
||||
MOVQ y1, tmp1 \
|
||||
SHLQ $2, tmp1 \
|
||||
SHRQ $62, tmp0 \
|
||||
MOVQ tmp1, y1 \
|
||||
ORQ tmp0, y1 \
|
||||
\
|
||||
SHLQ $2, y0 \
|
||||
\
|
||||
XORQ x0, y0 \
|
||||
XORQ x2, y0 \
|
||||
XORQ x1, y1 \
|
||||
XORQ x3, y1
|
||||
|
||||
#define UPDATE(msg0, msg1) \
|
||||
PADDQ msg0, v10 \
|
||||
PADDQ m00, v10 \
|
||||
PADDQ msg1, v11 \
|
||||
PADDQ m01, v11 \
|
||||
\
|
||||
MOVO v00, t0 \
|
||||
MOVO v01, t1 \
|
||||
PSRLQ $32, t0 \
|
||||
PSRLQ $32, t1 \
|
||||
PMULULQ v10, t0 \
|
||||
PMULULQ v11, t1 \
|
||||
PXOR t0, m00 \
|
||||
PXOR t1, m01 \
|
||||
\
|
||||
PADDQ m10, v00 \
|
||||
PADDQ m11, v01 \
|
||||
\
|
||||
MOVO v10, t0 \
|
||||
MOVO v11, t1 \
|
||||
PSRLQ $32, t0 \
|
||||
PSRLQ $32, t1 \
|
||||
PMULULQ v00, t0 \
|
||||
PMULULQ v01, t1 \
|
||||
PXOR t0, m10 \
|
||||
PXOR t1, m11 \
|
||||
\
|
||||
MOVO v10, t0 \
|
||||
PSHUFB t2, t0 \
|
||||
MOVO v11, t1 \
|
||||
PSHUFB t2, t1 \
|
||||
PADDQ t0, v00 \
|
||||
PADDQ t1, v01 \
|
||||
\
|
||||
MOVO v00, t0 \
|
||||
PSHUFB t2, t0 \
|
||||
MOVO v01, t1 \
|
||||
PSHUFB t2, t1 \
|
||||
PADDQ t0, v10 \
|
||||
PADDQ t1, v11
|
||||
|
||||
// func initializeSSE4(state *[16]uint64, key []byte)
|
||||
TEXT ·initializeSSE4(SB), NOSPLIT, $0-32
|
||||
MOVQ state+0(FP), AX
|
||||
MOVQ key_base+8(FP), BX
|
||||
MOVQ $·asmConstants<>(SB), CX
|
||||
|
||||
MOVOU 0(BX), v00
|
||||
MOVOU 16(BX), v01
|
||||
|
||||
PSHUFD $177, v00, v10
|
||||
PSHUFD $177, v01, v11
|
||||
|
||||
MOVOU 0(CX), m00
|
||||
MOVOU 16(CX), m01
|
||||
MOVOU 32(CX), m10
|
||||
MOVOU 48(CX), m11
|
||||
|
||||
PXOR m00, v00
|
||||
PXOR m01, v01
|
||||
PXOR m10, v10
|
||||
PXOR m11, v11
|
||||
|
||||
MOVOU v00, 0(AX)
|
||||
MOVOU v01, 16(AX)
|
||||
MOVOU v10, 32(AX)
|
||||
MOVOU v11, 48(AX)
|
||||
MOVOU m00, 64(AX)
|
||||
MOVOU m01, 80(AX)
|
||||
MOVOU m10, 96(AX)
|
||||
MOVOU m11, 112(AX)
|
||||
RET
|
||||
|
||||
// func updateSSE4(state *[16]uint64, msg []byte)
|
||||
TEXT ·updateSSE4(SB), NOSPLIT, $0-32
|
||||
MOVQ state+0(FP), AX
|
||||
MOVQ msg_base+8(FP), BX
|
||||
MOVQ msg_len+16(FP), CX
|
||||
|
||||
CMPQ CX, $32
|
||||
JB DONE
|
||||
|
||||
MOVOU 0(AX), v00
|
||||
MOVOU 16(AX), v01
|
||||
MOVOU 32(AX), v10
|
||||
MOVOU 48(AX), v11
|
||||
MOVOU 64(AX), m00
|
||||
MOVOU 80(AX), m01
|
||||
MOVOU 96(AX), m10
|
||||
MOVOU 112(AX), m11
|
||||
|
||||
MOVOU ·asmZipperMerge<>(SB), t2
|
||||
|
||||
LOOP:
|
||||
MOVOU 0(BX), t0
|
||||
MOVOU 16(BX), t1
|
||||
|
||||
UPDATE(t0, t1)
|
||||
|
||||
ADDQ $32, BX
|
||||
SUBQ $32, CX
|
||||
JA LOOP
|
||||
|
||||
MOVOU v00, 0(AX)
|
||||
MOVOU v01, 16(AX)
|
||||
MOVOU v10, 32(AX)
|
||||
MOVOU v11, 48(AX)
|
||||
MOVOU m00, 64(AX)
|
||||
MOVOU m01, 80(AX)
|
||||
MOVOU m10, 96(AX)
|
||||
MOVOU m11, 112(AX)
|
||||
|
||||
DONE:
|
||||
RET
|
||||
|
||||
// func finalizeSSE4(out []byte, state *[16]uint64)
|
||||
TEXT ·finalizeSSE4(SB), NOSPLIT, $0-32
|
||||
MOVQ state+24(FP), AX
|
||||
MOVQ out_base+0(FP), BX
|
||||
MOVQ out_len+8(FP), CX
|
||||
|
||||
MOVOU 0(AX), v00
|
||||
MOVOU 16(AX), v01
|
||||
MOVOU 32(AX), v10
|
||||
MOVOU 48(AX), v11
|
||||
MOVOU 64(AX), m00
|
||||
MOVOU 80(AX), m01
|
||||
MOVOU 96(AX), m10
|
||||
MOVOU 112(AX), m11
|
||||
|
||||
MOVOU ·asmZipperMerge<>(SB), t2
|
||||
|
||||
PSHUFD $177, v01, t0
|
||||
PSHUFD $177, v00, t1
|
||||
UPDATE(t0, t1)
|
||||
|
||||
PSHUFD $177, v01, t0
|
||||
PSHUFD $177, v00, t1
|
||||
UPDATE(t0, t1)
|
||||
|
||||
PSHUFD $177, v01, t0
|
||||
PSHUFD $177, v00, t1
|
||||
UPDATE(t0, t1)
|
||||
|
||||
PSHUFD $177, v01, t0
|
||||
PSHUFD $177, v00, t1
|
||||
UPDATE(t0, t1)
|
||||
|
||||
CMPQ CX, $8
|
||||
JE skipUpdate // Just 4 rounds for 64-bit checksum
|
||||
|
||||
PSHUFD $177, v01, t0
|
||||
PSHUFD $177, v00, t1
|
||||
UPDATE(t0, t1)
|
||||
|
||||
PSHUFD $177, v01, t0
|
||||
PSHUFD $177, v00, t1
|
||||
UPDATE(t0, t1)
|
||||
|
||||
CMPQ CX, $16
|
||||
JE skipUpdate // 6 rounds for 128-bit checksum
|
||||
|
||||
PSHUFD $177, v01, t0
|
||||
PSHUFD $177, v00, t1
|
||||
UPDATE(t0, t1)
|
||||
|
||||
PSHUFD $177, v01, t0
|
||||
PSHUFD $177, v00, t1
|
||||
UPDATE(t0, t1)
|
||||
|
||||
PSHUFD $177, v01, t0
|
||||
PSHUFD $177, v00, t1
|
||||
UPDATE(t0, t1)
|
||||
|
||||
PSHUFD $177, v01, t0
|
||||
PSHUFD $177, v00, t1
|
||||
UPDATE(t0, t1)
|
||||
|
||||
skipUpdate:
|
||||
MOVOU v00, 0(AX)
|
||||
MOVOU v01, 16(AX)
|
||||
MOVOU v10, 32(AX)
|
||||
MOVOU v11, 48(AX)
|
||||
MOVOU m00, 64(AX)
|
||||
MOVOU m01, 80(AX)
|
||||
MOVOU m10, 96(AX)
|
||||
MOVOU m11, 112(AX)
|
||||
|
||||
CMPQ CX, $8
|
||||
JE hash64
|
||||
CMPQ CX, $16
|
||||
JE hash128
|
||||
|
||||
// 256-bit checksum
|
||||
PADDQ v00, m00
|
||||
PADDQ v10, m10
|
||||
PADDQ v01, m01
|
||||
PADDQ v11, m11
|
||||
|
||||
MOVQ m00, R8
|
||||
PEXTRQ $1, m00, R9
|
||||
MOVQ m10, R10
|
||||
PEXTRQ $1, m10, R11
|
||||
REDUCE_MOD(R8, R9, R10, R11, R12, R13, R14, R15)
|
||||
MOVQ R14, 0(BX)
|
||||
MOVQ R15, 8(BX)
|
||||
|
||||
MOVQ m01, R8
|
||||
PEXTRQ $1, m01, R9
|
||||
MOVQ m11, R10
|
||||
PEXTRQ $1, m11, R11
|
||||
REDUCE_MOD(R8, R9, R10, R11, R12, R13, R14, R15)
|
||||
MOVQ R14, 16(BX)
|
||||
MOVQ R15, 24(BX)
|
||||
RET
|
||||
|
||||
hash128:
|
||||
PADDQ v00, v11
|
||||
PADDQ m00, m11
|
||||
PADDQ v11, m11
|
||||
MOVOU m11, 0(BX)
|
||||
RET
|
||||
|
||||
hash64:
|
||||
PADDQ v00, v10
|
||||
PADDQ m00, m10
|
||||
PADDQ v10, m10
|
||||
MOVQ m10, DX
|
||||
MOVQ DX, 0(BX)
|
||||
RET
|
||||
+81
@@ -0,0 +1,81 @@
|
||||
// Copyright (c) 2017-2024 Minio Inc. All rights reserved.
|
||||
// Use of this source code is governed by a license that can be
|
||||
// found in the LICENSE file.
|
||||
|
||||
//go:build !noasm && !appengine
|
||||
// +build !noasm,!appengine
|
||||
|
||||
package highwayhash
|
||||
|
||||
import (
|
||||
"golang.org/x/sys/cpu"
|
||||
)
|
||||
|
||||
var (
|
||||
useSSE4 = false
|
||||
useAVX2 = false
|
||||
useNEON = cpu.ARM64.HasASIMD
|
||||
useSVE = cpu.ARM64.HasSVE
|
||||
useSVE2 = false // cpu.ARM64.HasSVE2 -- disable until tested on real hardware
|
||||
useVMX = false
|
||||
)
|
||||
|
||||
func init() {
|
||||
if useSVE {
|
||||
if vl, _ := getVectorLength(); vl != 256 {
|
||||
//
|
||||
// Since HighwayHash is designed for AVX2,
|
||||
// SVE/SVE2 instructions only run correctly
|
||||
// for vector length of 256
|
||||
//
|
||||
useSVE2 = false
|
||||
useSVE = false
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
//go:noescape
|
||||
func initializeArm64(state *[16]uint64, key []byte)
|
||||
|
||||
//go:noescape
|
||||
func updateArm64(state *[16]uint64, msg []byte)
|
||||
|
||||
//go:noescape
|
||||
func getVectorLength() (vl, pl uint64)
|
||||
|
||||
//go:noescape
|
||||
func updateArm64Sve(state *[16]uint64, msg []byte)
|
||||
|
||||
//go:noescape
|
||||
func updateArm64Sve2(state *[16]uint64, msg []byte)
|
||||
|
||||
//go:noescape
|
||||
func finalizeArm64(out []byte, state *[16]uint64)
|
||||
|
||||
func initialize(state *[16]uint64, key []byte) {
|
||||
if useNEON {
|
||||
initializeArm64(state, key)
|
||||
} else {
|
||||
initializeGeneric(state, key)
|
||||
}
|
||||
}
|
||||
|
||||
func update(state *[16]uint64, msg []byte) {
|
||||
if useSVE2 {
|
||||
updateArm64Sve2(state, msg)
|
||||
} else if useSVE {
|
||||
updateArm64Sve(state, msg)
|
||||
} else if useNEON {
|
||||
updateArm64(state, msg)
|
||||
} else {
|
||||
updateGeneric(state, msg)
|
||||
}
|
||||
}
|
||||
|
||||
func finalize(out []byte, state *[16]uint64) {
|
||||
if useNEON {
|
||||
finalizeArm64(out, state)
|
||||
} else {
|
||||
finalizeGeneric(out, state)
|
||||
}
|
||||
}
|
||||
+324
@@ -0,0 +1,324 @@
|
||||
//
|
||||
// Minio Cloud Storage, (C) 2017 Minio, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
//
|
||||
|
||||
//+build !noasm,!appengine
|
||||
|
||||
// Use github.com/minio/asm2plan9s on this file to assemble ARM instructions to
|
||||
// the opcodes of their Plan9 equivalents
|
||||
|
||||
#include "textflag.h"
|
||||
|
||||
#define REDUCE_MOD(x0, x1, x2, x3, tmp0, tmp1, y0, y1) \
|
||||
MOVD $0x3FFFFFFFFFFFFFFF, tmp0 \
|
||||
AND tmp0, x3 \
|
||||
MOVD x2, y0 \
|
||||
MOVD x3, y1 \
|
||||
\
|
||||
MOVD x2, tmp0 \
|
||||
MOVD x3, tmp1 \
|
||||
LSL $1, tmp1 \
|
||||
LSR $63, tmp0 \
|
||||
MOVD tmp1, x3 \
|
||||
ORR tmp0, x3 \
|
||||
\
|
||||
LSL $1, x2 \
|
||||
\
|
||||
MOVD y0, tmp0 \
|
||||
MOVD y1, tmp1 \
|
||||
LSL $2, tmp1 \
|
||||
LSR $62, tmp0 \
|
||||
MOVD tmp1, y1 \
|
||||
ORR tmp0, y1 \
|
||||
\
|
||||
LSL $2, y0 \
|
||||
\
|
||||
EOR x0, y0 \
|
||||
EOR x2, y0 \
|
||||
EOR x1, y1 \
|
||||
EOR x3, y1
|
||||
|
||||
#define UPDATE(MSG1, MSG2) \
|
||||
\ // Add message
|
||||
VADD MSG1.D2, V2.D2, V2.D2 \
|
||||
VADD MSG2.D2, V3.D2, V3.D2 \
|
||||
\
|
||||
\ // v1 += mul0
|
||||
VADD V4.D2, V2.D2, V2.D2 \
|
||||
VADD V5.D2, V3.D2, V3.D2 \
|
||||
\
|
||||
\ // First pair of multiplies
|
||||
VTBL V29.B16, [V0.B16, V1.B16], V10.B16 \
|
||||
VTBL V30.B16, [V2.B16, V3.B16], V11.B16 \
|
||||
\
|
||||
\ // VUMULL V10.S2, V11.S2, V12.D2 /* assembler support missing */
|
||||
\ // VUMULL2 V10.S4, V11.S4, V13.D2 /* assembler support missing */
|
||||
WORD $0x2eaac16c \ // umull v12.2d, v11.2s, v10.2s
|
||||
WORD $0x6eaac16d \ // umull2 v13.2d, v11.4s, v10.4s
|
||||
\
|
||||
\ // v0 += mul1
|
||||
VADD V6.D2, V0.D2, V0.D2 \
|
||||
VADD V7.D2, V1.D2, V1.D2 \
|
||||
\
|
||||
\ // Second pair of multiplies
|
||||
VTBL V29.B16, [V2.B16, V3.B16], V15.B16 \
|
||||
VTBL V30.B16, [V0.B16, V1.B16], V14.B16 \
|
||||
\
|
||||
\ // EOR multiplication result in
|
||||
VEOR V12.B16, V4.B16, V4.B16 \
|
||||
VEOR V13.B16, V5.B16, V5.B16 \
|
||||
\
|
||||
\ // VUMULL V14.S2, V15.S2, V16.D2 /* assembler support missing */
|
||||
\ // VUMULL2 V14.S4, V15.S4, V17.D2 /* assembler support missing */
|
||||
WORD $0x2eaec1f0 \ // umull v16.2d, v15.2s, v14.2s
|
||||
WORD $0x6eaec1f1 \ // umull2 v17.2d, v15.4s, v14.4s
|
||||
\
|
||||
\ // First pair of zipper-merges
|
||||
VTBL V28.B16, [V2.B16], V18.B16 \
|
||||
VADD V18.D2, V0.D2, V0.D2 \
|
||||
VTBL V28.B16, [V3.B16], V19.B16 \
|
||||
VADD V19.D2, V1.D2, V1.D2 \
|
||||
\
|
||||
\ // Second pair of zipper-merges
|
||||
VTBL V28.B16, [V0.B16], V20.B16 \
|
||||
VADD V20.D2, V2.D2, V2.D2 \
|
||||
VTBL V28.B16, [V1.B16], V21.B16 \
|
||||
VADD V21.D2, V3.D2, V3.D2 \
|
||||
\
|
||||
\ // EOR multiplication result in
|
||||
VEOR V16.B16, V6.B16, V6.B16 \
|
||||
VEOR V17.B16, V7.B16, V7.B16
|
||||
|
||||
// func initializeArm64(state *[16]uint64, key []byte)
|
||||
TEXT ·initializeArm64(SB), NOSPLIT, $0
|
||||
MOVD state+0(FP), R0
|
||||
MOVD key_base+8(FP), R1
|
||||
|
||||
VLD1 (R1), [V1.S4, V2.S4]
|
||||
|
||||
VREV64 V1.S4, V3.S4
|
||||
VREV64 V2.S4, V4.S4
|
||||
|
||||
MOVD $·asmConstants(SB), R3
|
||||
VLD1 (R3), [V5.S4, V6.S4, V7.S4, V8.S4]
|
||||
VEOR V5.B16, V1.B16, V1.B16
|
||||
VEOR V6.B16, V2.B16, V2.B16
|
||||
VEOR V7.B16, V3.B16, V3.B16
|
||||
VEOR V8.B16, V4.B16, V4.B16
|
||||
|
||||
VST1.P [V1.D2, V2.D2, V3.D2, V4.D2], 64(R0)
|
||||
VST1 [V5.D2, V6.D2, V7.D2, V8.D2], (R0)
|
||||
RET
|
||||
|
||||
TEXT ·updateArm64(SB), NOSPLIT, $0
|
||||
MOVD state+0(FP), R0
|
||||
MOVD msg_base+8(FP), R1
|
||||
MOVD msg_len+16(FP), R2 // length of message
|
||||
SUBS $32, R2
|
||||
BMI complete
|
||||
|
||||
// Definition of registers
|
||||
// v0 = v0.lo
|
||||
// v1 = v0.hi
|
||||
// v2 = v1.lo
|
||||
// v3 = v1.hi
|
||||
// v4 = mul0.lo
|
||||
// v5 = mul0.hi
|
||||
// v6 = mul1.lo
|
||||
// v7 = mul1.hi
|
||||
|
||||
// Load zipper merge constants table pointer
|
||||
MOVD $·asmZipperMerge(SB), R3
|
||||
|
||||
// and load zipper merge constants into v28, v29, and v30
|
||||
VLD1 (R3), [V28.B16, V29.B16, V30.B16]
|
||||
|
||||
VLD1.P 64(R0), [V0.D2, V1.D2, V2.D2, V3.D2]
|
||||
VLD1 (R0), [V4.D2, V5.D2, V6.D2, V7.D2]
|
||||
SUBS $64, R0
|
||||
|
||||
loop:
|
||||
// Main loop
|
||||
VLD1.P 32(R1), [V26.S4, V27.S4]
|
||||
|
||||
UPDATE(V26, V27)
|
||||
|
||||
SUBS $32, R2
|
||||
BPL loop
|
||||
|
||||
// Store result
|
||||
VST1.P [V0.D2, V1.D2, V2.D2, V3.D2], 64(R0)
|
||||
VST1 [V4.D2, V5.D2, V6.D2, V7.D2], (R0)
|
||||
|
||||
complete:
|
||||
RET
|
||||
|
||||
// func finalizeArm64(out []byte, state *[16]uint64)
|
||||
TEXT ·finalizeArm64(SB), NOSPLIT, $0-32
|
||||
MOVD state+24(FP), R0
|
||||
MOVD out_base+0(FP), R1
|
||||
MOVD out_len+8(FP), R2
|
||||
|
||||
// Load zipper merge constants table pointer
|
||||
MOVD $·asmZipperMerge(SB), R3
|
||||
|
||||
// and load zipper merge constants into v28, v29, and v30
|
||||
VLD1 (R3), [V28.B16, V29.B16, V30.B16]
|
||||
|
||||
VLD1.P 64(R0), [V0.D2, V1.D2, V2.D2, V3.D2]
|
||||
VLD1 (R0), [V4.D2, V5.D2, V6.D2, V7.D2]
|
||||
SUB $64, R0
|
||||
|
||||
VREV64 V1.S4, V26.S4
|
||||
VREV64 V0.S4, V27.S4
|
||||
UPDATE(V26, V27)
|
||||
|
||||
VREV64 V1.S4, V26.S4
|
||||
VREV64 V0.S4, V27.S4
|
||||
UPDATE(V26, V27)
|
||||
|
||||
VREV64 V1.S4, V26.S4
|
||||
VREV64 V0.S4, V27.S4
|
||||
UPDATE(V26, V27)
|
||||
|
||||
VREV64 V1.S4, V26.S4
|
||||
VREV64 V0.S4, V27.S4
|
||||
UPDATE(V26, V27)
|
||||
|
||||
CMP $8, R2
|
||||
BEQ skipUpdate // Just 4 rounds for 64-bit checksum
|
||||
|
||||
VREV64 V1.S4, V26.S4
|
||||
VREV64 V0.S4, V27.S4
|
||||
UPDATE(V26, V27)
|
||||
|
||||
VREV64 V1.S4, V26.S4
|
||||
VREV64 V0.S4, V27.S4
|
||||
UPDATE(V26, V27)
|
||||
|
||||
CMP $16, R2
|
||||
BEQ skipUpdate // 6 rounds for 128-bit checksum
|
||||
|
||||
VREV64 V1.S4, V26.S4
|
||||
VREV64 V0.S4, V27.S4
|
||||
UPDATE(V26, V27)
|
||||
|
||||
VREV64 V1.S4, V26.S4
|
||||
VREV64 V0.S4, V27.S4
|
||||
UPDATE(V26, V27)
|
||||
|
||||
VREV64 V1.S4, V26.S4
|
||||
VREV64 V0.S4, V27.S4
|
||||
UPDATE(V26, V27)
|
||||
|
||||
VREV64 V1.S4, V26.S4
|
||||
VREV64 V0.S4, V27.S4
|
||||
UPDATE(V26, V27)
|
||||
|
||||
skipUpdate:
|
||||
// Store result
|
||||
VST1.P [V0.D2, V1.D2, V2.D2, V3.D2], 64(R0)
|
||||
VST1 [V4.D2, V5.D2, V6.D2, V7.D2], (R0)
|
||||
SUB $64, R0
|
||||
|
||||
CMP $8, R2
|
||||
BEQ hash64
|
||||
CMP $16, R2
|
||||
BEQ hash128
|
||||
|
||||
// 256-bit checksum
|
||||
MOVD 0*8(R0), R8
|
||||
MOVD 1*8(R0), R9
|
||||
MOVD 4*8(R0), R10
|
||||
MOVD 5*8(R0), R11
|
||||
MOVD 8*8(R0), R4
|
||||
MOVD 9*8(R0), R5
|
||||
MOVD 12*8(R0), R6
|
||||
MOVD 13*8(R0), R7
|
||||
ADD R4, R8
|
||||
ADD R5, R9
|
||||
ADD R6, R10
|
||||
ADD R7, R11
|
||||
|
||||
REDUCE_MOD(R8, R9, R10, R11, R4, R5, R6, R7)
|
||||
MOVD R6, 0(R1)
|
||||
MOVD R7, 8(R1)
|
||||
|
||||
MOVD 2*8(R0), R8
|
||||
MOVD 3*8(R0), R9
|
||||
MOVD 6*8(R0), R10
|
||||
MOVD 7*8(R0), R11
|
||||
MOVD 10*8(R0), R4
|
||||
MOVD 11*8(R0), R5
|
||||
MOVD 14*8(R0), R6
|
||||
MOVD 15*8(R0), R7
|
||||
ADD R4, R8
|
||||
ADD R5, R9
|
||||
ADD R6, R10
|
||||
ADD R7, R11
|
||||
|
||||
REDUCE_MOD(R8, R9, R10, R11, R4, R5, R6, R7)
|
||||
MOVD R6, 16(R1)
|
||||
MOVD R7, 24(R1)
|
||||
RET
|
||||
|
||||
hash128:
|
||||
MOVD 0*8(R0), R8
|
||||
MOVD 1*8(R0), R9
|
||||
MOVD 6*8(R0), R10
|
||||
MOVD 7*8(R0), R11
|
||||
ADD R10, R8
|
||||
ADD R11, R9
|
||||
MOVD 8*8(R0), R10
|
||||
MOVD 9*8(R0), R11
|
||||
ADD R10, R8
|
||||
ADD R11, R9
|
||||
MOVD 14*8(R0), R10
|
||||
MOVD 15*8(R0), R11
|
||||
ADD R10, R8
|
||||
ADD R11, R9
|
||||
MOVD R8, 0(R1)
|
||||
MOVD R9, 8(R1)
|
||||
RET
|
||||
|
||||
hash64:
|
||||
MOVD 0*8(R0), R4
|
||||
MOVD 4*8(R0), R5
|
||||
MOVD 8*8(R0), R6
|
||||
MOVD 12*8(R0), R7
|
||||
ADD R5, R4
|
||||
ADD R7, R6
|
||||
ADD R6, R4
|
||||
MOVD R4, (R1)
|
||||
RET
|
||||
|
||||
DATA ·asmConstants+0x00(SB)/8, $0xdbe6d5d5fe4cce2f
|
||||
DATA ·asmConstants+0x08(SB)/8, $0xa4093822299f31d0
|
||||
DATA ·asmConstants+0x10(SB)/8, $0x13198a2e03707344
|
||||
DATA ·asmConstants+0x18(SB)/8, $0x243f6a8885a308d3
|
||||
DATA ·asmConstants+0x20(SB)/8, $0x3bd39e10cb0ef593
|
||||
DATA ·asmConstants+0x28(SB)/8, $0xc0acf169b5f18a8c
|
||||
DATA ·asmConstants+0x30(SB)/8, $0xbe5466cf34e90c6c
|
||||
DATA ·asmConstants+0x38(SB)/8, $0x452821e638d01377
|
||||
GLOBL ·asmConstants(SB), 8, $64
|
||||
|
||||
// Constants for TBL instructions
|
||||
DATA ·asmZipperMerge+0x0(SB)/8, $0x000f010e05020c03 // zipper merge constant
|
||||
DATA ·asmZipperMerge+0x8(SB)/8, $0x070806090d0a040b
|
||||
DATA ·asmZipperMerge+0x10(SB)/8, $0x0f0e0d0c07060504 // setup first register for multiply
|
||||
DATA ·asmZipperMerge+0x18(SB)/8, $0x1f1e1d1c17161514
|
||||
DATA ·asmZipperMerge+0x20(SB)/8, $0x0b0a090803020100 // setup second register for multiply
|
||||
DATA ·asmZipperMerge+0x28(SB)/8, $0x1b1a191813121110
|
||||
GLOBL ·asmZipperMerge(SB), 8, $48
|
||||
+338
@@ -0,0 +1,338 @@
|
||||
// Copyright (c) 2017 Minio Inc. All rights reserved.
|
||||
// Use of this source code is governed by a license that can be
|
||||
// found in the LICENSE file.
|
||||
|
||||
package highwayhash
|
||||
|
||||
import (
|
||||
"encoding/binary"
|
||||
)
|
||||
|
||||
const (
|
||||
v0 = 0
|
||||
v1 = 4
|
||||
mul0 = 8
|
||||
mul1 = 12
|
||||
)
|
||||
|
||||
var (
|
||||
init0 = [4]uint64{0xdbe6d5d5fe4cce2f, 0xa4093822299f31d0, 0x13198a2e03707344, 0x243f6a8885a308d3}
|
||||
init1 = [4]uint64{0x3bd39e10cb0ef593, 0xc0acf169b5f18a8c, 0xbe5466cf34e90c6c, 0x452821e638d01377}
|
||||
)
|
||||
|
||||
func initializeGeneric(state *[16]uint64, k []byte) {
|
||||
var key [4]uint64
|
||||
|
||||
key[0] = binary.LittleEndian.Uint64(k[0:])
|
||||
key[1] = binary.LittleEndian.Uint64(k[8:])
|
||||
key[2] = binary.LittleEndian.Uint64(k[16:])
|
||||
key[3] = binary.LittleEndian.Uint64(k[24:])
|
||||
|
||||
copy(state[mul0:], init0[:])
|
||||
copy(state[mul1:], init1[:])
|
||||
|
||||
for i, k := range key {
|
||||
state[v0+i] = init0[i] ^ k
|
||||
}
|
||||
|
||||
key[0] = key[0]>>32 | key[0]<<32
|
||||
key[1] = key[1]>>32 | key[1]<<32
|
||||
key[2] = key[2]>>32 | key[2]<<32
|
||||
key[3] = key[3]>>32 | key[3]<<32
|
||||
|
||||
for i, k := range key {
|
||||
state[v1+i] = init1[i] ^ k
|
||||
}
|
||||
}
|
||||
|
||||
func updateGeneric(state *[16]uint64, msg []byte) {
|
||||
for len(msg) >= 32 {
|
||||
m := msg[:32]
|
||||
|
||||
// add message + mul0
|
||||
// Interleave operations to hide multiplication
|
||||
state[v1+0] += binary.LittleEndian.Uint64(m) + state[mul0+0]
|
||||
state[mul0+0] ^= uint64(uint32(state[v1+0])) * (state[v0+0] >> 32)
|
||||
state[v0+0] += state[mul1+0]
|
||||
state[mul1+0] ^= uint64(uint32(state[v0+0])) * (state[v1+0] >> 32)
|
||||
|
||||
state[v1+1] += binary.LittleEndian.Uint64(m[8:]) + state[mul0+1]
|
||||
state[mul0+1] ^= uint64(uint32(state[v1+1])) * (state[v0+1] >> 32)
|
||||
state[v0+1] += state[mul1+1]
|
||||
state[mul1+1] ^= uint64(uint32(state[v0+1])) * (state[v1+1] >> 32)
|
||||
|
||||
state[v1+2] += binary.LittleEndian.Uint64(m[16:]) + state[mul0+2]
|
||||
state[mul0+2] ^= uint64(uint32(state[v1+2])) * (state[v0+2] >> 32)
|
||||
state[v0+2] += state[mul1+2]
|
||||
state[mul1+2] ^= uint64(uint32(state[v0+2])) * (state[v1+2] >> 32)
|
||||
|
||||
state[v1+3] += binary.LittleEndian.Uint64(m[24:]) + state[mul0+3]
|
||||
state[mul0+3] ^= uint64(uint32(state[v1+3])) * (state[v0+3] >> 32)
|
||||
state[v0+3] += state[mul1+3]
|
||||
state[mul1+3] ^= uint64(uint32(state[v0+3])) * (state[v1+3] >> 32)
|
||||
|
||||
// inlined: zipperMerge(state[v1+0], state[v1+1], &state[v0+0], &state[v0+1])
|
||||
{
|
||||
val0 := state[v1+0]
|
||||
val1 := state[v1+1]
|
||||
res := val0 & (0xff << (2 * 8))
|
||||
res2 := (val0 & (0xff << (7 * 8))) + (val1 & (0xff << (2 * 8)))
|
||||
res += (val1 & (0xff << (7 * 8))) >> 8
|
||||
res2 += (val0 & (0xff << (6 * 8))) >> 8
|
||||
res += ((val0 & (0xff << (5 * 8))) + (val1 & (0xff << (6 * 8)))) >> 16
|
||||
res2 += (val1 & (0xff << (5 * 8))) >> 16
|
||||
res += ((val0 & (0xff << (3 * 8))) + (val1 & (0xff << (4 * 8)))) >> 24
|
||||
res2 += ((val1 & (0xff << (3 * 8))) + (val0 & (0xff << (4 * 8)))) >> 24
|
||||
res += (val0 & (0xff << (1 * 8))) << 32
|
||||
res2 += (val1 & 0xff) << 48
|
||||
res += val0 << 56
|
||||
res2 += (val1 & (0xff << (1 * 8))) << 24
|
||||
|
||||
state[v0+0] += res
|
||||
state[v0+1] += res2
|
||||
}
|
||||
// zipperMerge(state[v1+2], state[v1+3], &state[v0+2], &state[v0+3])
|
||||
{
|
||||
val0 := state[v1+2]
|
||||
val1 := state[v1+3]
|
||||
res := val0 & (0xff << (2 * 8))
|
||||
res2 := (val0 & (0xff << (7 * 8))) + (val1 & (0xff << (2 * 8)))
|
||||
res += (val1 & (0xff << (7 * 8))) >> 8
|
||||
res2 += (val0 & (0xff << (6 * 8))) >> 8
|
||||
res += ((val0 & (0xff << (5 * 8))) + (val1 & (0xff << (6 * 8)))) >> 16
|
||||
res2 += (val1 & (0xff << (5 * 8))) >> 16
|
||||
res += ((val0 & (0xff << (3 * 8))) + (val1 & (0xff << (4 * 8)))) >> 24
|
||||
res2 += ((val1 & (0xff << (3 * 8))) + (val0 & (0xff << (4 * 8)))) >> 24
|
||||
res += (val0 & (0xff << (1 * 8))) << 32
|
||||
res2 += (val1 & 0xff) << 48
|
||||
res += val0 << 56
|
||||
res2 += (val1 & (0xff << (1 * 8))) << 24
|
||||
|
||||
state[v0+2] += res
|
||||
state[v0+3] += res2
|
||||
}
|
||||
|
||||
// inlined: zipperMerge(state[v0+0], state[v0+1], &state[v1+0], &state[v1+1])
|
||||
{
|
||||
val0 := state[v0+0]
|
||||
val1 := state[v0+1]
|
||||
res := val0 & (0xff << (2 * 8))
|
||||
res2 := (val0 & (0xff << (7 * 8))) + (val1 & (0xff << (2 * 8)))
|
||||
res += (val1 & (0xff << (7 * 8))) >> 8
|
||||
res2 += (val0 & (0xff << (6 * 8))) >> 8
|
||||
res += ((val0 & (0xff << (5 * 8))) + (val1 & (0xff << (6 * 8)))) >> 16
|
||||
res2 += (val1 & (0xff << (5 * 8))) >> 16
|
||||
res += ((val0 & (0xff << (3 * 8))) + (val1 & (0xff << (4 * 8)))) >> 24
|
||||
res2 += ((val1 & (0xff << (3 * 8))) + (val0 & (0xff << (4 * 8)))) >> 24
|
||||
res += (val0 & (0xff << (1 * 8))) << 32
|
||||
res2 += (val1 & 0xff) << 48
|
||||
res += val0 << 56
|
||||
res2 += (val1 & (0xff << (1 * 8))) << 24
|
||||
|
||||
state[v1+0] += res
|
||||
state[v1+1] += res2
|
||||
}
|
||||
|
||||
//inlined: zipperMerge(state[v0+2], state[v0+3], &state[v1+2], &state[v1+3])
|
||||
{
|
||||
val0 := state[v0+2]
|
||||
val1 := state[v0+3]
|
||||
res := val0 & (0xff << (2 * 8))
|
||||
res2 := (val0 & (0xff << (7 * 8))) + (val1 & (0xff << (2 * 8)))
|
||||
res += (val1 & (0xff << (7 * 8))) >> 8
|
||||
res2 += (val0 & (0xff << (6 * 8))) >> 8
|
||||
res += ((val0 & (0xff << (5 * 8))) + (val1 & (0xff << (6 * 8)))) >> 16
|
||||
res2 += (val1 & (0xff << (5 * 8))) >> 16
|
||||
res += ((val0 & (0xff << (3 * 8))) + (val1 & (0xff << (4 * 8)))) >> 24
|
||||
res2 += ((val1 & (0xff << (3 * 8))) + (val0 & (0xff << (4 * 8)))) >> 24
|
||||
res += (val0 & (0xff << (1 * 8))) << 32
|
||||
res2 += (val1 & 0xff) << 48
|
||||
res += val0 << 56
|
||||
res2 += (val1 & (0xff << (1 * 8))) << 24
|
||||
|
||||
state[v1+2] += res
|
||||
state[v1+3] += res2
|
||||
}
|
||||
msg = msg[32:]
|
||||
}
|
||||
}
|
||||
|
||||
func finalizeGeneric(out []byte, state *[16]uint64) {
|
||||
var perm [4]uint64
|
||||
var tmp [32]byte
|
||||
runs := 4
|
||||
if len(out) == 16 {
|
||||
runs = 6
|
||||
} else if len(out) == 32 {
|
||||
runs = 10
|
||||
}
|
||||
for i := 0; i < runs; i++ {
|
||||
perm[0] = state[v0+2]>>32 | state[v0+2]<<32
|
||||
perm[1] = state[v0+3]>>32 | state[v0+3]<<32
|
||||
perm[2] = state[v0+0]>>32 | state[v0+0]<<32
|
||||
perm[3] = state[v0+1]>>32 | state[v0+1]<<32
|
||||
|
||||
binary.LittleEndian.PutUint64(tmp[0:], perm[0])
|
||||
binary.LittleEndian.PutUint64(tmp[8:], perm[1])
|
||||
binary.LittleEndian.PutUint64(tmp[16:], perm[2])
|
||||
binary.LittleEndian.PutUint64(tmp[24:], perm[3])
|
||||
|
||||
update(state, tmp[:])
|
||||
}
|
||||
|
||||
switch len(out) {
|
||||
case 8:
|
||||
binary.LittleEndian.PutUint64(out, state[v0+0]+state[v1+0]+state[mul0+0]+state[mul1+0])
|
||||
case 16:
|
||||
binary.LittleEndian.PutUint64(out, state[v0+0]+state[v1+2]+state[mul0+0]+state[mul1+2])
|
||||
binary.LittleEndian.PutUint64(out[8:], state[v0+1]+state[v1+3]+state[mul0+1]+state[mul1+3])
|
||||
case 32:
|
||||
h0, h1 := reduceMod(state[v0+0]+state[mul0+0], state[v0+1]+state[mul0+1], state[v1+0]+state[mul1+0], state[v1+1]+state[mul1+1])
|
||||
binary.LittleEndian.PutUint64(out[0:], h0)
|
||||
binary.LittleEndian.PutUint64(out[8:], h1)
|
||||
|
||||
h0, h1 = reduceMod(state[v0+2]+state[mul0+2], state[v0+3]+state[mul0+3], state[v1+2]+state[mul1+2], state[v1+3]+state[mul1+3])
|
||||
binary.LittleEndian.PutUint64(out[16:], h0)
|
||||
binary.LittleEndian.PutUint64(out[24:], h1)
|
||||
}
|
||||
}
|
||||
|
||||
// Experiments on variations left for future reference...
|
||||
/*
|
||||
func zipperMerge(v0, v1 uint64, d0, d1 *uint64) {
|
||||
if true {
|
||||
// fastest. original interleaved...
|
||||
res := v0 & (0xff << (2 * 8))
|
||||
res2 := (v0 & (0xff << (7 * 8))) + (v1 & (0xff << (2 * 8)))
|
||||
res += (v1 & (0xff << (7 * 8))) >> 8
|
||||
res2 += (v0 & (0xff << (6 * 8))) >> 8
|
||||
res += ((v0 & (0xff << (5 * 8))) + (v1 & (0xff << (6 * 8)))) >> 16
|
||||
res2 += (v1 & (0xff << (5 * 8))) >> 16
|
||||
res += ((v0 & (0xff << (3 * 8))) + (v1 & (0xff << (4 * 8)))) >> 24
|
||||
res2 += ((v1 & (0xff << (3 * 8))) + (v0 & (0xff << (4 * 8)))) >> 24
|
||||
res += (v0 & (0xff << (1 * 8))) << 32
|
||||
res2 += (v1 & 0xff) << 48
|
||||
res += v0 << 56
|
||||
res2 += (v1 & (0xff << (1 * 8))) << 24
|
||||
|
||||
*d0 += res
|
||||
*d1 += res2
|
||||
} else if false {
|
||||
// Reading bytes and combining into uint64
|
||||
var v0b [8]byte
|
||||
binary.LittleEndian.PutUint64(v0b[:], v0)
|
||||
var v1b [8]byte
|
||||
binary.LittleEndian.PutUint64(v1b[:], v1)
|
||||
var res, res2 uint64
|
||||
|
||||
res = uint64(v0b[0]) << (7 * 8)
|
||||
res2 = uint64(v1b[0]) << (6 * 8)
|
||||
res |= uint64(v0b[1]) << (5 * 8)
|
||||
res2 |= uint64(v1b[1]) << (4 * 8)
|
||||
res |= uint64(v0b[2]) << (2 * 8)
|
||||
res2 |= uint64(v1b[2]) << (2 * 8)
|
||||
res |= uint64(v0b[3])
|
||||
res2 |= uint64(v0b[4]) << (1 * 8)
|
||||
res |= uint64(v0b[5]) << (3 * 8)
|
||||
res2 |= uint64(v0b[6]) << (5 * 8)
|
||||
res |= uint64(v1b[4]) << (1 * 8)
|
||||
res2 |= uint64(v0b[7]) << (7 * 8)
|
||||
res |= uint64(v1b[6]) << (4 * 8)
|
||||
res2 |= uint64(v1b[3])
|
||||
res |= uint64(v1b[7]) << (6 * 8)
|
||||
res2 |= uint64(v1b[5]) << (3 * 8)
|
||||
|
||||
*d0 += res
|
||||
*d1 += res2
|
||||
|
||||
} else if false {
|
||||
// bytes to bytes shuffle
|
||||
var v0b [8]byte
|
||||
binary.LittleEndian.PutUint64(v0b[:], v0)
|
||||
var v1b [8]byte
|
||||
binary.LittleEndian.PutUint64(v1b[:], v1)
|
||||
var res [8]byte
|
||||
|
||||
//res += ((v0 & (0xff << (3 * 8))) + (v1 & (0xff << (4 * 8)))) >> 24
|
||||
res[0] = v0b[3]
|
||||
res[1] = v1b[4]
|
||||
|
||||
// res := v0 & (0xff << (2 * 8))
|
||||
res[2] = v0b[2]
|
||||
|
||||
//res += ((v0 & (0xff << (5 * 8))) + (v1 & (0xff << (6 * 8)))) >> 16
|
||||
res[3] = v0b[5]
|
||||
res[4] = v1b[6]
|
||||
|
||||
//res += (v0 & (0xff << (1 * 8))) << 32
|
||||
res[5] = v0b[1]
|
||||
|
||||
//res += (v1 & (0xff << (7 * 8))) >> 8
|
||||
res[6] += v1b[7]
|
||||
|
||||
//res += v0 << 56
|
||||
res[7] = v0b[0]
|
||||
v0 = binary.LittleEndian.Uint64(res[:])
|
||||
*d0 += v0
|
||||
|
||||
//res += ((v1 & (0xff << (3 * 8))) + (v0 & (0xff << (4 * 8)))) >> 24
|
||||
res[0] = v1b[3]
|
||||
res[1] = v0b[4]
|
||||
|
||||
res[2] = v1b[2]
|
||||
|
||||
// res += (v1 & (0xff << (5 * 8))) >> 16
|
||||
res[3] = v1b[5]
|
||||
|
||||
//res += (v1 & (0xff << (1 * 8))) << 24
|
||||
res[4] = v1b[1]
|
||||
|
||||
// res += (v0 & (0xff << (6 * 8))) >> 8
|
||||
res[5] = v0b[6]
|
||||
|
||||
//res := (v0 & (0xff << (7 * 8))) + (v1 & (0xff << (2 * 8)))
|
||||
res[7] = v0b[7]
|
||||
|
||||
//res += (v1 & 0xff) << 48
|
||||
res[6] = v1b[0]
|
||||
|
||||
v0 = binary.LittleEndian.Uint64(res[:])
|
||||
*d1 += v0
|
||||
} else {
|
||||
// original.
|
||||
res := v0 & (0xff << (2 * 8))
|
||||
res += (v1 & (0xff << (7 * 8))) >> 8
|
||||
res += ((v0 & (0xff << (5 * 8))) + (v1 & (0xff << (6 * 8)))) >> 16
|
||||
res += ((v0 & (0xff << (3 * 8))) + (v1 & (0xff << (4 * 8)))) >> 24
|
||||
res += (v0 & (0xff << (1 * 8))) << 32
|
||||
res += v0 << 56
|
||||
|
||||
*d0 += res
|
||||
|
||||
res = (v0 & (0xff << (7 * 8))) + (v1 & (0xff << (2 * 8)))
|
||||
res += (v0 & (0xff << (6 * 8))) >> 8
|
||||
res += (v1 & (0xff << (5 * 8))) >> 16
|
||||
res += ((v1 & (0xff << (3 * 8))) + (v0 & (0xff << (4 * 8)))) >> 24
|
||||
res += (v1 & 0xff) << 48
|
||||
res += (v1 & (0xff << (1 * 8))) << 24
|
||||
|
||||
*d1 += res
|
||||
}
|
||||
}
|
||||
*/
|
||||
|
||||
// reduce v = [v0, v1, v2, v3] mod the irreducible polynomial x^128 + x^2 + x
|
||||
func reduceMod(v0, v1, v2, v3 uint64) (r0, r1 uint64) {
|
||||
v3 &= 0x3FFFFFFFFFFFFFFF
|
||||
|
||||
r0, r1 = v2, v3
|
||||
|
||||
v3 = (v3 << 1) | (v2 >> (64 - 1))
|
||||
v2 <<= 1
|
||||
r1 = (r1 << 2) | (r0 >> (64 - 2))
|
||||
r0 <<= 2
|
||||
|
||||
r0 ^= v0 ^ v2
|
||||
r1 ^= v1 ^ v3
|
||||
return
|
||||
}
|
||||
+36
@@ -0,0 +1,36 @@
|
||||
// Copyright (c) 2017 Minio Inc. All rights reserved.
|
||||
// Use of this source code is governed by a license that can be
|
||||
// found in the LICENSE file.
|
||||
|
||||
//go:build !noasm && !appengine
|
||||
// +build !noasm,!appengine
|
||||
|
||||
package highwayhash
|
||||
|
||||
var (
|
||||
useSSE4 = false
|
||||
useAVX2 = false
|
||||
useNEON = false
|
||||
useSVE = false
|
||||
useSVE2 = false
|
||||
useVMX = true
|
||||
)
|
||||
|
||||
//go:noescape
|
||||
func updatePpc64Le(state *[16]uint64, msg []byte)
|
||||
|
||||
func initialize(state *[16]uint64, key []byte) {
|
||||
initializeGeneric(state, key)
|
||||
}
|
||||
|
||||
func update(state *[16]uint64, msg []byte) {
|
||||
if useVMX {
|
||||
updatePpc64Le(state, msg)
|
||||
} else {
|
||||
updateGeneric(state, msg)
|
||||
}
|
||||
}
|
||||
|
||||
func finalize(out []byte, state *[16]uint64) {
|
||||
finalizeGeneric(out, state)
|
||||
}
|
||||
+182
@@ -0,0 +1,182 @@
|
||||
//
|
||||
// Minio Cloud Storage, (C) 2018 Minio, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
//
|
||||
|
||||
//+build !noasm,!appengine
|
||||
|
||||
#include "textflag.h"
|
||||
|
||||
// Definition of registers
|
||||
#define V0_LO VS32
|
||||
#define V0_LO_ V0
|
||||
#define V0_HI VS33
|
||||
#define V0_HI_ V1
|
||||
#define V1_LO VS34
|
||||
#define V1_LO_ V2
|
||||
#define V1_HI VS35
|
||||
#define V1_HI_ V3
|
||||
#define MUL0_LO VS36
|
||||
#define MUL0_LO_ V4
|
||||
#define MUL0_HI VS37
|
||||
#define MUL0_HI_ V5
|
||||
#define MUL1_LO VS38
|
||||
#define MUL1_LO_ V6
|
||||
#define MUL1_HI VS39
|
||||
#define MUL1_HI_ V7
|
||||
|
||||
// Message
|
||||
#define MSG_LO VS40
|
||||
#define MSG_LO_ V8
|
||||
#define MSG_HI VS41
|
||||
|
||||
// Constants
|
||||
#define ROTATE VS42
|
||||
#define ROTATE_ V10
|
||||
#define MASK VS43
|
||||
#define MASK_ V11
|
||||
|
||||
// Temps
|
||||
#define TEMP1 VS44
|
||||
#define TEMP1_ V12
|
||||
#define TEMP2 VS45
|
||||
#define TEMP2_ V13
|
||||
#define TEMP3 VS46
|
||||
#define TEMP3_ V14
|
||||
#define TEMP4_ V15
|
||||
#define TEMP5_ V16
|
||||
#define TEMP6_ V17
|
||||
#define TEMP7_ V18
|
||||
|
||||
// Regular registers
|
||||
#define STATE R3
|
||||
#define MSG_BASE R4
|
||||
#define MSG_LEN R5
|
||||
#define CONSTANTS R6
|
||||
#define P1 R7
|
||||
#define P2 R8
|
||||
#define P3 R9
|
||||
#define P4 R10
|
||||
#define P5 R11
|
||||
#define P6 R12
|
||||
#define P7 R14 // avoid using R13
|
||||
|
||||
TEXT ·updatePpc64Le(SB), NOFRAME|NOSPLIT, $0-32
|
||||
MOVD state+0(FP), STATE
|
||||
MOVD msg_base+8(FP), MSG_BASE
|
||||
MOVD msg_len+16(FP), MSG_LEN // length of message
|
||||
|
||||
// Sanity check for length
|
||||
CMPU MSG_LEN, $31
|
||||
BLE complete
|
||||
|
||||
// Setup offsets
|
||||
MOVD $16, P1
|
||||
MOVD $32, P2
|
||||
MOVD $48, P3
|
||||
MOVD $64, P4
|
||||
MOVD $80, P5
|
||||
MOVD $96, P6
|
||||
MOVD $112, P7
|
||||
|
||||
// Load state
|
||||
LXVD2X (STATE)(R0), V0_LO
|
||||
LXVD2X (STATE)(P1), V0_HI
|
||||
LXVD2X (STATE)(P2), V1_LO
|
||||
LXVD2X (STATE)(P3), V1_HI
|
||||
LXVD2X (STATE)(P4), MUL0_LO
|
||||
LXVD2X (STATE)(P5), MUL0_HI
|
||||
LXVD2X (STATE)(P6), MUL1_LO
|
||||
LXVD2X (STATE)(P7), MUL1_HI
|
||||
XXPERMDI V0_LO, V0_LO, $2, V0_LO
|
||||
XXPERMDI V0_HI, V0_HI, $2, V0_HI
|
||||
XXPERMDI V1_LO, V1_LO, $2, V1_LO
|
||||
XXPERMDI V1_HI, V1_HI, $2, V1_HI
|
||||
XXPERMDI MUL0_LO, MUL0_LO, $2, MUL0_LO
|
||||
XXPERMDI MUL0_HI, MUL0_HI, $2, MUL0_HI
|
||||
XXPERMDI MUL1_LO, MUL1_LO, $2, MUL1_LO
|
||||
XXPERMDI MUL1_HI, MUL1_HI, $2, MUL1_HI
|
||||
|
||||
// Load asmConstants table pointer
|
||||
MOVD $·asmConstants(SB), CONSTANTS
|
||||
LXVD2X (CONSTANTS)(R0), ROTATE
|
||||
LXVD2X (CONSTANTS)(P1), MASK
|
||||
XXLNAND MASK, MASK, MASK
|
||||
|
||||
loop:
|
||||
// Main highwayhash update loop
|
||||
LXVD2X (MSG_BASE)(R0), MSG_LO
|
||||
VADDUDM V0_LO_, MUL1_LO_, TEMP1_
|
||||
VRLD V0_LO_, ROTATE_, TEMP2_
|
||||
VADDUDM MUL1_HI_, V0_HI_, TEMP3_
|
||||
LXVD2X (MSG_BASE)(P1), MSG_HI
|
||||
ADD $32, MSG_BASE, MSG_BASE
|
||||
XXPERMDI MSG_LO, MSG_LO, $2, MSG_LO
|
||||
XXPERMDI MSG_HI, MSG_HI, $2, V0_LO
|
||||
VADDUDM MSG_LO_, MUL0_LO_, MSG_LO_
|
||||
VADDUDM V0_LO_, MUL0_HI_, V0_LO_
|
||||
VADDUDM MSG_LO_, V1_LO_, V1_LO_
|
||||
VSRD V0_HI_, ROTATE_, MSG_LO_
|
||||
VADDUDM V0_LO_, V1_HI_, V1_HI_
|
||||
VPERM V1_LO_, V1_LO_, MASK_, V0_LO_
|
||||
VMULOUW V1_LO_, TEMP2_, TEMP2_
|
||||
VPERM V1_HI_, V1_HI_, MASK_, TEMP7_
|
||||
VADDUDM V0_LO_, TEMP1_, V0_LO_
|
||||
VMULOUW V1_HI_, MSG_LO_, MSG_LO_
|
||||
VADDUDM TEMP7_, TEMP3_, V0_HI_
|
||||
VPERM V0_LO_, V0_LO_, MASK_, TEMP6_
|
||||
VRLD V1_LO_, ROTATE_, TEMP4_
|
||||
VSRD V1_HI_, ROTATE_, TEMP5_
|
||||
VPERM V0_HI_, V0_HI_, MASK_, TEMP7_
|
||||
XXLXOR MUL0_LO, TEMP2, MUL0_LO
|
||||
VMULOUW TEMP1_, TEMP4_, TEMP1_
|
||||
VMULOUW TEMP3_, TEMP5_, TEMP3_
|
||||
XXLXOR MUL0_HI, MSG_LO, MUL0_HI
|
||||
XXLXOR MUL1_LO, TEMP1, MUL1_LO
|
||||
XXLXOR MUL1_HI, TEMP3, MUL1_HI
|
||||
VADDUDM TEMP6_, V1_LO_, V1_LO_
|
||||
VADDUDM TEMP7_, V1_HI_, V1_HI_
|
||||
|
||||
SUB $32, MSG_LEN, MSG_LEN
|
||||
CMPU MSG_LEN, $32
|
||||
BGE loop
|
||||
|
||||
// Save state
|
||||
XXPERMDI V0_LO, V0_LO, $2, V0_LO
|
||||
XXPERMDI V0_HI, V0_HI, $2, V0_HI
|
||||
XXPERMDI V1_LO, V1_LO, $2, V1_LO
|
||||
XXPERMDI V1_HI, V1_HI, $2, V1_HI
|
||||
XXPERMDI MUL0_LO, MUL0_LO, $2, MUL0_LO
|
||||
XXPERMDI MUL0_HI, MUL0_HI, $2, MUL0_HI
|
||||
XXPERMDI MUL1_LO, MUL1_LO, $2, MUL1_LO
|
||||
XXPERMDI MUL1_HI, MUL1_HI, $2, MUL1_HI
|
||||
STXVD2X V0_LO, (STATE)(R0)
|
||||
STXVD2X V0_HI, (STATE)(P1)
|
||||
STXVD2X V1_LO, (STATE)(P2)
|
||||
STXVD2X V1_HI, (STATE)(P3)
|
||||
STXVD2X MUL0_LO, (STATE)(P4)
|
||||
STXVD2X MUL0_HI, (STATE)(P5)
|
||||
STXVD2X MUL1_LO, (STATE)(P6)
|
||||
STXVD2X MUL1_HI, (STATE)(P7)
|
||||
|
||||
complete:
|
||||
RET
|
||||
|
||||
// Constants table
|
||||
DATA ·asmConstants+0x0(SB)/8, $0x0000000000000020
|
||||
DATA ·asmConstants+0x8(SB)/8, $0x0000000000000020
|
||||
DATA ·asmConstants+0x10(SB)/8, $0x070806090d0a040b // zipper merge constant
|
||||
DATA ·asmConstants+0x18(SB)/8, $0x000f010e05020c03 // zipper merge constant
|
||||
|
||||
GLOBL ·asmConstants(SB), 8, $32
|
||||
+29
@@ -0,0 +1,29 @@
|
||||
// Copyright (c) 2017 Minio Inc. All rights reserved.
|
||||
// Use of this source code is governed by a license that can be
|
||||
// found in the LICENSE file.
|
||||
|
||||
//go:build noasm || (!amd64 && !arm64 && !ppc64le)
|
||||
// +build noasm !amd64,!arm64,!ppc64le
|
||||
|
||||
package highwayhash
|
||||
|
||||
var (
|
||||
useSSE4 = false
|
||||
useAVX2 = false
|
||||
useNEON = false
|
||||
useSVE = false
|
||||
useSVE2 = false
|
||||
useVMX = false
|
||||
)
|
||||
|
||||
func initialize(state *[16]uint64, k []byte) {
|
||||
initializeGeneric(state, k)
|
||||
}
|
||||
|
||||
func update(state *[16]uint64, msg []byte) {
|
||||
updateGeneric(state, msg)
|
||||
}
|
||||
|
||||
func finalize(out []byte, state *[16]uint64) {
|
||||
finalizeGeneric(out, state)
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user