Merge pull request 'The forge bridge, the indexer and the teller run beside today's mesh-delivery, and the view shows candidates and fingerprints (design 47 R3c, issues 1235-1241, 1209)' (#5) from feat/delivery-parts into main
This commit was merged in pull request #5.
This commit is contained in:
@@ -0,0 +1 @@
|
||||
vendor/
|
||||
+55
-2
@@ -4,18 +4,71 @@
|
||||
# This repository's own check, mesh/repo-check (novox/hq ADR 0238 §4), written with its first commit (ADR 0428 §4):
|
||||
# every Go module in the tree is vetted, its unit tests run, and then its integration tests (-tags integration).
|
||||
# A tree with no Go module has nothing to test, and the check says so and passes.
|
||||
#
|
||||
# No module commits its dependencies (no vendor directory: .gitignore). A module's dependencies are fetched, or a
|
||||
# requirement of the mesh's own forge (the pinned mesh-sdk commit) is taken from its repository's clone beside the check
|
||||
# (MESH_CHECK_BESIDE), through a go.mod of its own (-modfile), by mesh-catalog's helper checks/deps, built from the
|
||||
# clone of mesh-catalog beside this check and not copied here: the build seat forgets the forge credential before the
|
||||
# check runs, so a commit the seat's module cache does not hold cannot be fetched from the forge (novox/hq issue 385). A
|
||||
# module whose dependencies cannot be had **fails the check**, by name and why: its tests could not run, and a test that
|
||||
# could not run is a failure, never a note. Run by hand, without MESH_CHECK_BESIDE, the dependencies are fetched.
|
||||
set -eu
|
||||
cd "$(dirname "$0")"
|
||||
|
||||
# The toolchain the check runs in sets -mod=vendor; there is no vendor directory, so the modules are used as they are.
|
||||
GOFLAGS="$(printf '%s' "${GOFLAGS:-}" | sed 's/-mod=[a-z]*//g') -mod=mod"
|
||||
export GOFLAGS
|
||||
export GOPRIVATE=git.novox.be
|
||||
|
||||
modules=$(find . -name go.mod -not -path '*/vendor/*' -not -path './.git/*' | sort)
|
||||
if [ -z "$modules" ]; then
|
||||
echo "This tree holds no Go module, so there are no tests to run; the check passes."
|
||||
exit 0
|
||||
fi
|
||||
|
||||
work=$(mktemp -d)
|
||||
trap 'rm -rf "$work"' EXIT
|
||||
deps=""
|
||||
helper="${MESH_CHECK_BESIDE:-}/mesh-catalog/checks/deps"
|
||||
if [ -n "${MESH_CHECK_BESIDE:-}" ] && [ -f "$helper/main.go" ]; then
|
||||
if ! (cd "$helper" && go build -o "$work/deps" .); then
|
||||
echo "FAILED: the dependency helper of mesh-catalog beside this check could not be built"
|
||||
exit 1
|
||||
fi
|
||||
deps="$work/deps"
|
||||
elif [ -n "${MESH_CHECK_BESIDE:-}" ]; then
|
||||
echo "NOT TAKEN FROM BESIDE: no mesh-catalog/checks/deps beside this check ($MESH_CHECK_BESIDE); the dependencies are fetched"
|
||||
fi
|
||||
|
||||
untestable=""
|
||||
for mod in $modules; do
|
||||
dir=$(dirname "$mod")
|
||||
modfile=""
|
||||
if [ -n "$deps" ]; then
|
||||
if ! modfile=$("$deps" -beside "$MESH_CHECK_BESIDE" -work "$work" "$dir"); then
|
||||
echo "FAILED: $dir could not be tested: $modfile"
|
||||
untestable="$untestable $dir"
|
||||
continue
|
||||
fi
|
||||
[ -n "$modfile" ] && echo "$dir takes its forge dependencies from the clones beside the check"
|
||||
elif ! fetched=$(cd "$dir" && go mod download 2>&1); then
|
||||
echo "FAILED: $dir could not be tested: its dependencies cannot be fetched: $(printf '%s\n' "$fetched" | tail -n 1)"
|
||||
untestable="$untestable $dir"
|
||||
continue
|
||||
fi
|
||||
mf=""
|
||||
[ -n "$modfile" ] && mf="-modfile=$modfile"
|
||||
unformatted=$(cd "$dir" && gofmt -l .)
|
||||
if [ -n "$unformatted" ]; then
|
||||
echo "$dir is not gofmt'd: $unformatted"
|
||||
exit 1
|
||||
fi
|
||||
echo "== $dir: go vet and the unit tests"
|
||||
(cd "$dir" && go vet ./... && go test ./...)
|
||||
(cd "$dir" && go vet $mf ./... && go test $mf ./...)
|
||||
echo "== $dir: the integration tests (-tags integration)"
|
||||
(cd "$dir" && go vet -tags integration ./... && go test -tags integration ./...)
|
||||
(cd "$dir" && go vet $mf -tags integration ./... && go test $mf -tags integration ./...)
|
||||
done
|
||||
if [ -n "$untestable" ]; then
|
||||
echo "the check fails: the tests of$untestable could not run (novox/hq issue 385)"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
@@ -0,0 +1,91 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
"mesh-delivery-forge-bridge/internal/bridge"
|
||||
)
|
||||
|
||||
// callTimeout bounds a call to a tool (mesh-sdk go/process/call.go CallTimeout).
|
||||
const callTimeout = 30 * time.Second
|
||||
|
||||
// caller calls a module's tool by its manifest key (`<module>.<tool>`) with a JSON body and returns the tool's result.
|
||||
type caller func(ctx context.Context, key string, body any) (json.RawMessage, error)
|
||||
|
||||
// callerOn calls tools as the bridge's own principal. It copies mesh-sdk go/process/call.go (Call, callSubjects,
|
||||
// request; hand-written here until the SDK's helper is used, issue 1242): the request is the tool's arguments as
|
||||
// JSON, published on the caller-named subject `mesh.mod.<module>.call.<tool>.<user, dots as ~>`, or on the subject that
|
||||
// names no caller, `mesh.mod.<module>.tool.<tool>`, when nothing answers the first; the reply is
|
||||
// {"result", "error", "node"} and a non-empty error is the call's failure (mesh-tools node-tools/internal/bus/bus.go
|
||||
// answerOf). The grant to publish on these subjects comes from the manifest's invokes alone (mesh-control
|
||||
// internal/broker/nats.go invokedSubjects).
|
||||
func callerOn(nc *nats.Conn, user string) caller {
|
||||
token := strings.ReplaceAll(user, ".", "~")
|
||||
request := func(ctx context.Context, subject string, data []byte) (json.RawMessage, error) {
|
||||
ctx, cancel := context.WithTimeout(ctx, callTimeout)
|
||||
defer cancel()
|
||||
msg, err := nc.RequestWithContext(ctx, subject, data)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var r struct {
|
||||
Result json.RawMessage `json:"result"`
|
||||
Error string `json:"error"`
|
||||
Node string `json:"node"`
|
||||
}
|
||||
if err := json.Unmarshal(msg.Data, &r); err != nil {
|
||||
return nil, fmt.Errorf("%s answered something that is not a tool's answer: %w", subject, err)
|
||||
}
|
||||
if r.Error != "" {
|
||||
return nil, errors.New(r.Error)
|
||||
}
|
||||
return r.Result, nil
|
||||
}
|
||||
return func(ctx context.Context, key string, body any) (json.RawMessage, error) {
|
||||
module, tool, ok := strings.Cut(key, ".")
|
||||
if !ok || module == "" || tool == "" {
|
||||
return nil, fmt.Errorf("%q is not <module>.<tool>", key)
|
||||
}
|
||||
data, err := json.Marshal(body)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
result, err := request(ctx, "mesh.mod."+module+".call."+tool+"."+token, data)
|
||||
if errors.Is(err, nats.ErrNoResponders) {
|
||||
result, err = request(ctx, "mesh.mod."+module+".tool."+tool, data)
|
||||
}
|
||||
return result, err
|
||||
}
|
||||
}
|
||||
|
||||
// commitTreesTool is the forge's tool for a commit's trees (mesh-catalog modules/gitea, issue 1234): arguments owner,
|
||||
// repo and sha, answer {root, trees}, the root directory spelled "" (1234/AC4).
|
||||
const commitTreesTool = "gitea.gitea_commit_trees"
|
||||
|
||||
// treesVia asks the forge for a commit's trees through call.
|
||||
func treesVia(ctx context.Context, call caller) bridge.Trees {
|
||||
return func(owner, repo, sha string) (string, map[string]string, error) {
|
||||
raw, err := call(ctx, commitTreesTool, map[string]string{"owner": owner, "repo": repo, "sha": sha})
|
||||
if err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
var a struct {
|
||||
Root string `json:"root"`
|
||||
Trees map[string]string `json:"trees"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &a); err != nil {
|
||||
return "", nil, fmt.Errorf("the answer of %s is not {root, trees}: %w", commitTreesTool, err)
|
||||
}
|
||||
if a.Root == "" {
|
||||
return "", nil, fmt.Errorf("the answer of %s names no root tree", commitTreesTool)
|
||||
}
|
||||
return a.Root, a.Trees, nil
|
||||
}
|
||||
}
|
||||
+295
@@ -0,0 +1,295 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats-server/v2/server"
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// busConf is the shape of the composed accounts file (mesh-controller internal/broker): one account with JetStream,
|
||||
// the controller's user and the bridge's own principal <node>.<module>.
|
||||
const busConf = `
|
||||
listen: 127.0.0.1:-1
|
||||
jetstream { store_dir: %q }
|
||||
accounts {
|
||||
MESH {
|
||||
jetstream: enabled
|
||||
users = [
|
||||
{ user: "ctl", password: "pw" }
|
||||
{ user: "n1.mesh-delivery-forge-bridge", password: "pw" }
|
||||
]
|
||||
}
|
||||
}
|
||||
`
|
||||
|
||||
func startBus(t *testing.T) *server.Server {
|
||||
t.Helper()
|
||||
dir, err := os.MkdirTemp(os.Getenv("TMPDIR"), "b")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { os.RemoveAll(dir) })
|
||||
conf := filepath.Join(dir, "bus.conf")
|
||||
if err := os.WriteFile(conf, []byte(fmt.Sprintf(busConf, filepath.Join(dir, "js"))), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
opts, err := server.ProcessConfigFile(conf)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
opts.NoLog, opts.NoSigs = true, true
|
||||
srv, err := server.NewServer(opts)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
go srv.Start()
|
||||
if !srv.ReadyForConnections(10 * time.Second) {
|
||||
t.Fatal("the bus did not come up")
|
||||
}
|
||||
t.Cleanup(srv.Shutdown)
|
||||
return srv
|
||||
}
|
||||
|
||||
// startBridge runs the program against an embedded bus with the stream EVENTS, the bridge's stream FORGE and its durable
|
||||
// consumer <node>_<module> on EVENTS, as the controller makes them (mesh-control internal/broker streams.go:192,
|
||||
// nats.go:1387-1396, modstreams.go). It returns a control connection, its JetStream and a context.
|
||||
func startBridge(t *testing.T, openReply string) (*nats.Conn, jetstream.JetStream, context.Context, <-chan error) {
|
||||
srv := startBus(t)
|
||||
ctl, err := nats.Connect(srv.ClientURL(), nats.UserInfo("ctl", "pw"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(ctl.Close)
|
||||
js, err := jetstream.New(ctl)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
|
||||
t.Cleanup(cancel)
|
||||
if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: "EVENTS", Subjects: []string{"mesh.mod.*.event.>"}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: "FORGE", Subjects: []string{"mesh.stream.mesh-delivery-forge-bridge.event.forge.>"}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := js.CreateOrUpdateConsumer(ctx, "EVENTS", jetstream.ConsumerConfig{
|
||||
Durable: "n1_mesh-delivery-forge-bridge", AckPolicy: jetstream.AckExplicitPolicy,
|
||||
FilterSubjects: []string{"mesh.mod.gitea.event.pull.updated", "mesh.mod.gitea.event.pull.merged", "mesh.mod.gitea.event.pull.closed"},
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// A stand-in answers the forge's gitea_open_pulls, which the bridge asks at its start, on its tool subject
|
||||
// (mesh-control internal/broker/nats.go invokedSubjects) in the reply envelope of mesh-sdk go/process/call.go.
|
||||
open, err := ctl.Subscribe("mesh.mod.gitea.tool.gitea_open_pulls", func(m *nats.Msg) { _ = m.Respond([]byte(openReply)) })
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _ = open.Unsubscribe() })
|
||||
if err := ctl.Flush(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
broker := filepath.Join(t.TempDir(), "broker")
|
||||
cred, _ := json.Marshal(credential{URL: srv.ClientURL(), Node: "n1", Module: "mesh-delivery-forge-bridge", User: "n1.mesh-delivery-forge-bridge", Password: "pw"})
|
||||
if err := os.WriteFile(broker, cred, 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
env := map[string]string{"MESH_BROKER_FILE": broker}
|
||||
runCtx, stop := context.WithCancel(ctx)
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- run(runCtx, func(k string) string { return env[k] }, io.Discard) }()
|
||||
t.Cleanup(stop)
|
||||
return ctl, js, ctx, done
|
||||
}
|
||||
|
||||
// TestBridgeWritesPullMessages starts the program the way the node does (its entry, run, with MESH_BROKER_FILE).
|
||||
func TestBridgeWritesPullMessages(t *testing.T) { // 1235/AC2
|
||||
_, js, ctx, _ := startBridge(t, `{"result":{"pulls":[]}}`)
|
||||
|
||||
// gitea.pull.updated as the forge module publishes it: the body of trunk.ts:233-250 on mesh.mod.gitea.event.pull.updated.
|
||||
ev, _ := json.Marshal(map[string]any{"owner": "novox", "repo": "mesh-delivery", "number": 12, "title": "t", "body": "b", "base": "main",
|
||||
"head": "feat/x", "head_sha": head, "clone_url": "u", "html_url": "h", "paths": []string{"modules/a/x.go"}, "paths_truncated": false,
|
||||
"removed": []string{}, "module_dirs": []string{"modules/a"}, "module_dirs_said": true})
|
||||
if _, err := js.Publish(ctx, "mesh.mod.gitea.event.pull.updated", ev); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
forge, err := js.Stream(ctx, "FORGE")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var info *jetstream.StreamInfo
|
||||
for deadline := time.Now().Add(5 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) {
|
||||
if info, err = forge.Info(ctx); err == nil && info.State.Msgs >= 1 {
|
||||
break
|
||||
}
|
||||
}
|
||||
if info.State.Msgs != 1 {
|
||||
t.Fatalf("FORGE holds %d messages within 5 s, want 1", info.State.Msgs)
|
||||
}
|
||||
raw, err := forge.GetLastMsgForSubject(ctx, "mesh.stream.mesh-delivery-forge-bridge.event.forge.novox.mesh-delivery.pull")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
f, err := delivery.DecodeForge(raw.Data)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := delivery.Forge{V: delivery.Version, Kind: "pull", Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12, Head: head,
|
||||
Action: "updated", Dirs: []string{"modules/a"}, DirsKnown: true}
|
||||
if !delivery.SameForge(f, want) {
|
||||
t.Errorf("message = %+v, want %+v", f, want)
|
||||
}
|
||||
|
||||
// The event is acknowledged: the consumer owes nothing and has nothing pending.
|
||||
cons, err := js.Consumer(ctx, "EVENTS", "n1_mesh-delivery-forge-bridge")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for deadline := time.Now().Add(5 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) {
|
||||
ci, err := cons.Info(ctx)
|
||||
if err == nil && ci.NumAckPending == 0 && ci.NumPending == 0 && ci.Delivered.Consumer >= 1 {
|
||||
return
|
||||
}
|
||||
}
|
||||
ci, _ := cons.Info(ctx)
|
||||
t.Errorf("the event is not acknowledged: %+v", ci)
|
||||
}
|
||||
|
||||
const merge = "89abcdef0123456789abcdef0123456789abcdef"
|
||||
|
||||
// TestBridgeWritesMergedMessage has a stand-in answer the forge's gitea_commit_trees on its tool subject
|
||||
// (mesh-control internal/broker/nats.go invokedSubjects: mesh.mod.<module>.tool.<tool>), in the reply envelope of
|
||||
// mesh-sdk go/process/call.go ({result, error, node}).
|
||||
func TestBridgeWritesMergedMessage(t *testing.T) { // 1236/AC4
|
||||
ctl, js, ctx, _ := startBridge(t, `{"result":{"pulls":[]}}`)
|
||||
var asked chan []byte = make(chan []byte, 4)
|
||||
sub, err := ctl.Subscribe("mesh.mod.gitea.tool.gitea_commit_trees", func(m *nats.Msg) {
|
||||
asked <- m.Data
|
||||
_ = m.Respond([]byte(`{"result":{"root":"r0","trees":{"":"t0","modules/a":"t1"}},"node":"n0"}`))
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _ = sub.Unsubscribe() })
|
||||
if err := ctl.Flush(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// gitea.pull.merged as mesh-catalog modules/gitea/index.ts:186-207 publishes it.
|
||||
ev, _ := json.Marshal(map[string]any{"owner": "novox", "repo": "mesh-delivery", "number": 12, "title": "t", "body": "b",
|
||||
"head_sha": head, "head": "feat/x", "base": "main", "merge_commit_sha": merge, "merged_at": "2026-10-19T12:00:00Z",
|
||||
"clone_url": "u", "html_url": "h", "paths": []string{"modules/a/x.go"}, "paths_truncated": false, "removed": []string{},
|
||||
"module_dirs": []string{"modules/a"}, "module_dirs_said": true})
|
||||
if _, err := js.Publish(ctx, "mesh.mod.gitea.event.pull.merged", ev); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
forge, err := js.Stream(ctx, "FORGE")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var info *jetstream.StreamInfo
|
||||
for deadline := time.Now().Add(5 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) {
|
||||
if info, err = forge.Info(ctx); err == nil && info.State.Msgs >= 1 {
|
||||
break
|
||||
}
|
||||
}
|
||||
if info.State.Msgs != 1 {
|
||||
t.Fatalf("FORGE holds %d messages within 5 s, want 1", info.State.Msgs)
|
||||
}
|
||||
raw, err := forge.GetLastMsgForSubject(ctx, "mesh.stream.mesh-delivery-forge-bridge.event.forge.novox.mesh-delivery.merged")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
f, err := delivery.DecodeForge(raw.Data)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := delivery.Forge{V: delivery.Version, Kind: "merged", Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12, Head: head,
|
||||
Merge: merge, Root: "r0", Trees: map[string]string{"": "t0", "modules/a": "t1"}}
|
||||
if !delivery.SameForge(f, want) {
|
||||
t.Errorf("message = %+v, want %+v", f, want)
|
||||
}
|
||||
select {
|
||||
case body := <-asked:
|
||||
if string(body) != `{"owner":"novox","repo":"mesh-delivery","sha":"`+merge+`"}` {
|
||||
t.Errorf("the tool was asked %s", body)
|
||||
}
|
||||
default:
|
||||
t.Error("the stand-in was not asked")
|
||||
}
|
||||
if len(asked) != 0 {
|
||||
t.Error("the tool was asked more than once")
|
||||
}
|
||||
cons, err := js.Consumer(ctx, "EVENTS", "n1_mesh-delivery-forge-bridge")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for deadline := time.Now().Add(5 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) {
|
||||
ci, err := cons.Info(ctx)
|
||||
if err == nil && ci.NumAckPending == 0 && ci.NumPending == 0 && ci.Delivered.Consumer >= 1 {
|
||||
return
|
||||
}
|
||||
}
|
||||
ci, _ := cons.Info(ctx)
|
||||
t.Errorf("the event is not acknowledged: %+v", ci)
|
||||
}
|
||||
|
||||
// TestBridgeSaysTheOpenPullsAtStart: the forge holds two open pull requests when the bridge starts; one message each is on FORGE.
|
||||
func TestBridgeSaysTheOpenPullsAtStart(t *testing.T) { // 1237/AC1
|
||||
// The entries are gitea_open_pulls's (mesh-catalog modules/gitea/client.ts:369-381, head the branch).
|
||||
_, js, ctx, _ := startBridge(t, `{"result":{"pulls":[
|
||||
{"owner":"novox","repo":"mesh-delivery","number":12,"base":"main","head":"feat/x","head_sha":"`+head+`","html_url":"h","required":[],"statuses":[],"checks":{},"labels":[]},
|
||||
{"owner":"novox","repo":"mesh-catalog","number":300,"base":"main","head":"feat/x","head_sha":"`+strings.Repeat("a", 40)+`","html_url":"h","required":[],"statuses":[],"checks":{},"labels":[]}]},"node":"n0"}`)
|
||||
forge, err := js.Stream(ctx, "FORGE")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var info *jetstream.StreamInfo
|
||||
for deadline := time.Now().Add(5 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) {
|
||||
if info, err = forge.Info(ctx); err == nil && info.State.Msgs >= 2 {
|
||||
break
|
||||
}
|
||||
}
|
||||
if info.State.Msgs != 2 {
|
||||
t.Fatalf("FORGE holds %d messages within 5 s, want 2", info.State.Msgs)
|
||||
}
|
||||
for _, repo := range []string{"mesh-delivery", "mesh-catalog"} {
|
||||
raw, err := forge.GetLastMsgForSubject(ctx, "mesh.stream.mesh-delivery-forge-bridge.event.forge.novox."+repo+".pull")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if f, err := delivery.DecodeForge(raw.Data); err != nil || f.Action != "updated" || f.DirsKnown || f.Branch != "feat/x" {
|
||||
t.Errorf("%s: %+v err %v", repo, f, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestBridgeExitsWhenTheOpenPullsCannotBeRead: the process manager starts it again.
|
||||
func TestBridgeExitsWhenTheOpenPullsCannotBeRead(t *testing.T) { // 1237/AC3
|
||||
_, _, _, done := startBridge(t, `{"error":"the forge said 502"}`)
|
||||
select {
|
||||
case err := <-done:
|
||||
if err == nil || !strings.Contains(err.Error(), "gitea.gitea_open_pulls") || !strings.Contains(err.Error(), "the forge said 502") {
|
||||
t.Errorf("run returned %v", err)
|
||||
}
|
||||
case <-time.After(10 * time.Second):
|
||||
t.Fatal("the bridge did not exit")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,205 @@
|
||||
// Command mesh-delivery-forge-bridge is delivery's forge bridge (novox/hq ADR 0437 §4, issue 1235): it hears the forge
|
||||
// module's pull request events and writes each as a pull message on its stream FORGE. It runs beside today's
|
||||
// mesh-delivery and merges nothing. It serves no request subject and holds no timer: every reaction is an event
|
||||
// (ADR 0437 §8); a refused act is tried again only on the next event touching its delivery.
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"crypto/tls"
|
||||
"crypto/x509"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"os"
|
||||
"os/signal"
|
||||
"strings"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
|
||||
"mesh-delivery-forge-bridge/internal/bridge"
|
||||
)
|
||||
|
||||
// eventsStream is the shared stream every module's events are on; the module's consumer on it is the durable
|
||||
// <node>_<module> the controller makes (mesh-control internal/broker/nats.go consumerStream, consumerDurable).
|
||||
const eventsStream = "EVENTS"
|
||||
|
||||
// The bridge's own stream FORGE takes `mesh.stream.<module>.event.<event>` (mesh-control internal/broker
|
||||
// modstreams.go ModuleStreamSubject).
|
||||
func forgeSubject(event string) string {
|
||||
return "mesh.stream." + delivery.ForgeBridgeModule + ".event." + event
|
||||
}
|
||||
|
||||
func main() {
|
||||
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
|
||||
defer stop()
|
||||
if err := run(ctx, os.Getenv, os.Stderr); err != nil {
|
||||
fmt.Fprintln(os.Stderr, "mesh-delivery-forge-bridge:", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
// credential is the broker file the controller issues to the module: JSON of the bus address, the certificate
|
||||
// fingerprint to pin, the principal and its password (as the watchman reads it; issue 1242 moves this into the SDK).
|
||||
type credential struct {
|
||||
URL string `json:"url"`
|
||||
Fingerprint string `json:"fingerprint"`
|
||||
Node string `json:"node"`
|
||||
Module string `json:"module"`
|
||||
User string `json:"user"`
|
||||
Password string `json:"password"`
|
||||
}
|
||||
|
||||
func readCredential(path string) (credential, error) {
|
||||
raw, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return credential{}, err
|
||||
}
|
||||
var c credential
|
||||
if err := json.Unmarshal(raw, &c); err != nil {
|
||||
return credential{}, fmt.Errorf("the broker credential is not one: %w", err)
|
||||
}
|
||||
if c.URL == "" || c.User == "" {
|
||||
return credential{}, errors.New("the broker credential names no bus address or user")
|
||||
}
|
||||
return c, nil
|
||||
}
|
||||
|
||||
// pinned accepts exactly the certificate with this SHA-256 and no other.
|
||||
func pinned(want string) *tls.Config {
|
||||
return &tls.Config{
|
||||
InsecureSkipVerify: true, //nolint:gosec // replaced by the pin below, which is stricter
|
||||
MinVersion: tls.VersionTLS12,
|
||||
VerifyPeerCertificate: func(rawCerts [][]byte, _ [][]*x509.Certificate) error {
|
||||
if len(rawCerts) == 0 {
|
||||
return errors.New("the bus presented no certificate")
|
||||
}
|
||||
sum := sha256.Sum256(rawCerts[0])
|
||||
if got := "sha256:" + hex.EncodeToString(sum[:]); got != want {
|
||||
return fmt.Errorf("the bus presented a certificate this mesh does not know (%.23s…)", got)
|
||||
}
|
||||
return nil
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// delivered is the part of a JetStream message the bridge uses.
|
||||
type delivered interface {
|
||||
Subject() string
|
||||
Data() []byte
|
||||
Headers() nats.Header
|
||||
Ack() error
|
||||
Term() error
|
||||
NakWithDelay(delay time.Duration) error
|
||||
}
|
||||
|
||||
// retryDelay is how long the server waits before it delivers again an event the forge could not be asked about (the
|
||||
// server's redelivery is the only clock; issue 1236/AC3).
|
||||
const retryDelay = 10 * time.Second
|
||||
|
||||
// publisher writes to FORGE.
|
||||
type publisher interface {
|
||||
Publish(ctx context.Context, subject string, payload []byte, opts ...jetstream.PublishOpt) (*jetstream.PubAck, error)
|
||||
}
|
||||
|
||||
// handle turns one event into its message on FORGE and acknowledges it. An event it cannot read is terminated, never
|
||||
// redelivered, and the log names its subject and why. A message it could not write stays unacknowledged: the server
|
||||
// delivers it again (the server's ack wait is the only clock).
|
||||
func handle(ctx context.Context, m delivered, pub publisher, trees bridge.Trees, logger *log.Logger) {
|
||||
msg, err := bridge.Convert(m.Subject(), m.Data(), trees)
|
||||
switch {
|
||||
case errors.Is(err, bridge.ErrRetry):
|
||||
logger.Printf("%v; heard again in %s", err, retryDelay)
|
||||
_ = m.NakWithDelay(retryDelay)
|
||||
return
|
||||
case errors.Is(err, bridge.ErrNoModules):
|
||||
logger.Printf("%v; no merged message", err)
|
||||
_ = m.Ack()
|
||||
return
|
||||
case errors.Is(err, bridge.ErrNotHandled):
|
||||
logger.Printf("%v; nothing written", err)
|
||||
_ = m.Ack()
|
||||
return
|
||||
case err != nil:
|
||||
logger.Printf("%v; the event is dropped", err)
|
||||
_ = m.Term()
|
||||
return
|
||||
}
|
||||
var opts []jetstream.PublishOpt
|
||||
// The forge's event id makes a redelivery after a lost acknowledgement the same message to the stream.
|
||||
if id := m.Headers().Get("x-event-id"); id != "" {
|
||||
opts = append(opts, jetstream.WithMsgID(id))
|
||||
}
|
||||
if _, err := pub.Publish(ctx, forgeSubject(msg.Event), msg.Data, opts...); err != nil {
|
||||
logger.Printf("%s: %s could not be written on FORGE, left for redelivery: %v", m.Subject(), msg.Event, err)
|
||||
return
|
||||
}
|
||||
_ = m.Ack()
|
||||
}
|
||||
|
||||
// run is the program: it returns when ctx ends or the bus cannot be used.
|
||||
func run(ctx context.Context, getenv func(string) string, logOut io.Writer) error {
|
||||
logger := log.New(logOut, "mesh-delivery-forge-bridge: ", log.LstdFlags)
|
||||
path := strings.TrimSpace(getenv("MESH_BROKER_FILE"))
|
||||
if path == "" {
|
||||
return errors.New("MESH_BROKER_FILE is not set, so there is no credential to hear the bus with")
|
||||
}
|
||||
cred, err := readCredential(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
again := make(chan struct{}, 1)
|
||||
opts := []nats.Option{
|
||||
nats.Name(cred.Node + "." + cred.Module),
|
||||
nats.UserInfo(cred.User, cred.Password),
|
||||
nats.CustomInboxPrefix("_INBOX." + cred.User),
|
||||
nats.MaxReconnects(-1),
|
||||
nats.RetryOnFailedConnect(true),
|
||||
nats.ReconnectHandler(func(*nats.Conn) {
|
||||
select {
|
||||
case again <- struct{}{}:
|
||||
default:
|
||||
}
|
||||
}),
|
||||
}
|
||||
if strings.TrimSpace(cred.Fingerprint) != "" {
|
||||
opts = append(opts, nats.Secure(pinned(cred.Fingerprint)))
|
||||
}
|
||||
conn, err := nats.Connect(cred.URL, opts...)
|
||||
if err != nil {
|
||||
return fmt.Errorf("connecting to the bus: %w", err)
|
||||
}
|
||||
defer conn.Close()
|
||||
js, err := jetstream.New(conn)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// What the forge holds open, said before any event is heard, and again at each reconnect (ADR 0429 §13).
|
||||
call := callerOn(conn, cred.User)
|
||||
sync := func(ctx context.Context) error { return syncOpen(ctx, call, js, logger) }
|
||||
if err := sync(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
go onReconnect(ctx, again, sync, logger)
|
||||
cons, err := js.Consumer(ctx, eventsStream, cred.Node+"_"+cred.Module)
|
||||
if err != nil {
|
||||
return fmt.Errorf("the consumer %s_%s on %s: %w", cred.Node, cred.Module, eventsStream, err)
|
||||
}
|
||||
cc, err := cons.Consume(func(m jetstream.Msg) { handle(ctx, m, js, treesVia(ctx, call), logger) },
|
||||
jetstream.ConsumeErrHandler(func(_ jetstream.ConsumeContext, err error) { logger.Printf("hearing the bus: %v", err) }))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer cc.Stop()
|
||||
<-ctx.Done()
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,173 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"log"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
const head = "0123456789abcdef0123456789abcdef01234567"
|
||||
|
||||
type fakeMsg struct {
|
||||
subject string
|
||||
data []byte
|
||||
acked, termd int
|
||||
nak []time.Duration
|
||||
}
|
||||
|
||||
func (f *fakeMsg) Subject() string { return f.subject }
|
||||
func (f *fakeMsg) Data() []byte { return f.data }
|
||||
func (f *fakeMsg) Headers() nats.Header { return nats.Header{"X-Event-Id": {"ev-1"}} }
|
||||
func (f *fakeMsg) Ack() error { f.acked++; return nil }
|
||||
func (f *fakeMsg) NakWithDelay(d time.Duration) error { f.nak = append(f.nak, d); return nil }
|
||||
func (f *fakeMsg) Term() error { f.termd++; return nil }
|
||||
|
||||
type fakePub struct {
|
||||
subjects []string
|
||||
data [][]byte
|
||||
err error
|
||||
}
|
||||
|
||||
func (p *fakePub) Publish(_ context.Context, subject string, payload []byte, _ ...jetstream.PublishOpt) (*jetstream.PubAck, error) {
|
||||
if p.err != nil {
|
||||
return nil, p.err
|
||||
}
|
||||
p.subjects, p.data = append(p.subjects, subject), append(p.data, payload)
|
||||
return &jetstream.PubAck{}, nil
|
||||
}
|
||||
|
||||
// body is gitea.pull.updated as mesh-catalog modules/gitea/trunk.ts:233-250 writes it (the fields the bridge reads).
|
||||
func body(m map[string]any) []byte {
|
||||
base := map[string]any{"owner": "novox", "repo": "mesh-delivery", "number": 12, "head": "feat/x", "head_sha": head,
|
||||
"module_dirs": []string{"modules/a"}, "module_dirs_said": true}
|
||||
for k, v := range m {
|
||||
if v == nil {
|
||||
delete(base, k)
|
||||
} else {
|
||||
base[k] = v
|
||||
}
|
||||
}
|
||||
b, _ := json.Marshal(base)
|
||||
return b
|
||||
}
|
||||
|
||||
func TestReadableEventIsWrittenOnForgeAndAcked(t *testing.T) { // 1235/AC2 (unit part; the bus is the integration test)
|
||||
var logs bytes.Buffer
|
||||
m, pub := &fakeMsg{subject: "mesh.mod.gitea.event.pull.updated", data: body(nil)}, &fakePub{}
|
||||
handle(context.Background(), m, pub, nil, log.New(&logs, "", 0))
|
||||
if len(pub.subjects) != 1 || pub.subjects[0] != "mesh.stream.mesh-delivery-forge-bridge.event.forge.novox.mesh-delivery.pull" {
|
||||
t.Fatalf("published on %v", pub.subjects)
|
||||
}
|
||||
f, err := delivery.DecodeForge(pub.data[0])
|
||||
if err != nil || f.Number != 12 || f.Branch != "feat/x" || f.Head != head || f.Action != "updated" || !f.DirsKnown {
|
||||
t.Errorf("message %+v err %v", f, err)
|
||||
}
|
||||
if m.acked != 1 || m.termd != 0 {
|
||||
t.Errorf("acked %d terminated %d", m.acked, m.termd)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUnreadableEventIsTerminatedAndLogged(t *testing.T) { // 1235/AC5
|
||||
var logs bytes.Buffer
|
||||
m, pub := &fakeMsg{subject: "mesh.mod.gitea.event.pull.updated", data: body(map[string]any{"head_sha": "short"})}, &fakePub{}
|
||||
handle(context.Background(), m, pub, nil, log.New(&logs, "", 0))
|
||||
if len(pub.subjects) != 0 {
|
||||
t.Errorf("a message was written: %v", pub.subjects)
|
||||
}
|
||||
if m.termd != 1 || m.acked != 0 {
|
||||
t.Errorf("terminated %d acked %d, want terminated once", m.termd, m.acked)
|
||||
}
|
||||
if !strings.Contains(logs.String(), "mesh.mod.gitea.event.pull.updated") || !strings.Contains(logs.String(), "head") {
|
||||
t.Errorf("log %q names neither the subject nor why", logs.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestUnwrittenMessageIsNotAcked(t *testing.T) { // 1235 (a failed write is redelivered by the server, not retried here)
|
||||
var logs bytes.Buffer
|
||||
m, pub := &fakeMsg{subject: "mesh.mod.gitea.event.pull.updated", data: body(nil)}, &fakePub{err: errors.New("no stream")}
|
||||
handle(context.Background(), m, pub, nil, log.New(&logs, "", 0))
|
||||
if m.acked != 0 || m.termd != 0 {
|
||||
t.Errorf("acked %d terminated %d, want neither", m.acked, m.termd)
|
||||
}
|
||||
if !strings.Contains(logs.String(), "no stream") {
|
||||
t.Errorf("log %q", logs.String())
|
||||
}
|
||||
}
|
||||
|
||||
// mergedEvent is gitea.pull.merged as mesh-catalog modules/gitea/index.ts:186-207 writes it (the fields the bridge reads).
|
||||
func mergedEvent() []byte {
|
||||
b, _ := json.Marshal(map[string]any{"owner": "novox", "repo": "mesh-delivery", "number": 12, "head": "feat/x", "head_sha": head,
|
||||
"base": "main", "merge_commit_sha": "89abcdef0123456789abcdef0123456789abcdef"})
|
||||
return b
|
||||
}
|
||||
|
||||
func TestTreesRefusedLeavesTheEventForTenSecondsAndLogs(t *testing.T) { // 1236/AC3
|
||||
var logs bytes.Buffer
|
||||
m, pub := &fakeMsg{subject: "mesh.mod.gitea.event.pull.merged", data: mergedEvent()}, &fakePub{}
|
||||
call := func(context.Context, string, any) (json.RawMessage, error) {
|
||||
return nil, errors.New("the tree listing of novox/mesh-delivery was cut")
|
||||
}
|
||||
handle(context.Background(), m, pub, treesVia(context.Background(), call), log.New(&logs, "", 0))
|
||||
if len(pub.subjects) != 0 {
|
||||
t.Errorf("a message was written: %v", pub.subjects)
|
||||
}
|
||||
if m.acked != 0 || m.termd != 0 || len(m.nak) != 1 || m.nak[0] < 10*time.Second {
|
||||
t.Errorf("acked %d terminated %d naks %v, want one nak of at least 10 s", m.acked, m.termd, m.nak)
|
||||
}
|
||||
for _, w := range []string{"novox/mesh-delivery", "89abcdef0123456789abcdef0123456789abcdef", "the tree listing of novox/mesh-delivery was cut"} {
|
||||
if !strings.Contains(logs.String(), w) {
|
||||
t.Errorf("log %q does not name %q", logs.String(), w)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestMergeCallsTheToolOnceAndWritesTheMergedMessage(t *testing.T) { // 1236/AC1 (through the tool's request and answer)
|
||||
var logs bytes.Buffer
|
||||
m, pub := &fakeMsg{subject: "mesh.mod.gitea.event.pull.merged", data: mergedEvent()}, &fakePub{}
|
||||
var keys []string
|
||||
var args []any
|
||||
call := func(_ context.Context, key string, body any) (json.RawMessage, error) {
|
||||
keys, args = append(keys, key), append(args, body)
|
||||
return json.RawMessage(`{"root":"r0","trees":{"":"t0","modules/a":"t1"}}`), nil
|
||||
}
|
||||
handle(context.Background(), m, pub, treesVia(context.Background(), call), log.New(&logs, "", 0))
|
||||
if len(keys) != 1 || keys[0] != "gitea.gitea_commit_trees" {
|
||||
t.Fatalf("calls %v", keys)
|
||||
}
|
||||
if got, _ := json.Marshal(args[0]); string(got) != `{"owner":"novox","repo":"mesh-delivery","sha":"89abcdef0123456789abcdef0123456789abcdef"}` {
|
||||
t.Errorf("arguments %s", got)
|
||||
}
|
||||
if len(pub.subjects) != 1 || pub.subjects[0] != "mesh.stream.mesh-delivery-forge-bridge.event.forge.novox.mesh-delivery.merged" {
|
||||
t.Fatalf("published on %v", pub.subjects)
|
||||
}
|
||||
f, err := delivery.DecodeForge(pub.data[0])
|
||||
if err != nil || f.Root != "r0" || f.Trees[""] != "t0" || f.Trees["modules/a"] != "t1" || f.Merge == "" {
|
||||
t.Errorf("message %+v err %v", f, err)
|
||||
}
|
||||
if m.acked != 1 {
|
||||
t.Errorf("acked %d", m.acked)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnAnswerWithoutARootIsNotUsed(t *testing.T) { // 1236 (a shape the tool does not answer)
|
||||
call := func(context.Context, string, any) (json.RawMessage, error) {
|
||||
return json.RawMessage(`{"trees":{"a":"t"}}`), nil
|
||||
}
|
||||
if _, _, err := treesVia(context.Background(), call)("o", "r", "s"); err == nil {
|
||||
t.Error("an answer with no root was taken")
|
||||
}
|
||||
call = func(context.Context, string, any) (json.RawMessage, error) { return json.RawMessage(`[1]`), nil }
|
||||
if _, _, err := treesVia(context.Background(), call)("o", "r", "s"); err == nil {
|
||||
t.Error("an answer that is not an object was taken")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,52 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log"
|
||||
|
||||
"mesh-delivery-forge-bridge/internal/bridge"
|
||||
)
|
||||
|
||||
// openPullsTool is the forge's tool for every open pull request (mesh-catalog modules/gitea/tools/index.ts
|
||||
// gitea_open_pulls: no arguments, answers {pulls: [...]}).
|
||||
const openPullsTool = "gitea.gitea_open_pulls"
|
||||
|
||||
// syncOpen reads the open pull requests from the forge and writes one pull message for each: what makes the indexer's
|
||||
// candidates match the forge after a restart or a gap in the forge's events (ADR 0429 §13). An error names the call and
|
||||
// the forge's words.
|
||||
func syncOpen(ctx context.Context, call caller, pub publisher, logger *log.Logger) error {
|
||||
raw, err := call(ctx, openPullsTool, struct{}{})
|
||||
if err != nil {
|
||||
return fmt.Errorf("%s: %w", openPullsTool, err)
|
||||
}
|
||||
msgs, skipped, err := bridge.OpenPulls(raw)
|
||||
if err != nil {
|
||||
return fmt.Errorf("%s: %w", openPullsTool, err)
|
||||
}
|
||||
for _, s := range skipped {
|
||||
logger.Printf("an open pull request is left out: %s", s)
|
||||
}
|
||||
for _, m := range msgs {
|
||||
if _, err := pub.Publish(ctx, forgeSubject(m.Event), m.Data); err != nil {
|
||||
return fmt.Errorf("the open pull request message %s could not be written on FORGE: %w", m.Event, err)
|
||||
}
|
||||
}
|
||||
logger.Printf("%d open pull requests said on FORGE", len(msgs))
|
||||
return nil
|
||||
}
|
||||
|
||||
// onReconnect reads the open pull requests again each time the bus connection comes back, until ctx ends; a failure is
|
||||
// logged and the bridge goes on (the next reconnect or event tries again).
|
||||
func onReconnect(ctx context.Context, again <-chan struct{}, sync func(context.Context) error, logger *log.Logger) {
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-again:
|
||||
if err := sync(ctx); err != nil {
|
||||
logger.Printf("after a reconnect: %v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,114 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"log"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// openAnswer is gitea_open_pulls's answer as mesh-catalog modules/gitea/client.ts:369-381 builds its entries inside {pulls}.
|
||||
func openAnswer() json.RawMessage {
|
||||
return json.RawMessage(`{"pulls":[
|
||||
{"owner":"novox","repo":"mesh-delivery","number":12,"base":"main","head":"feat/x","head_sha":"` + head + `","html_url":"h","required":[],"statuses":[],"checks":{},"labels":[]},
|
||||
{"owner":"novox","repo":"mesh-catalog","number":300,"base":"main","head":"feat/x","head_sha":"` + strings.Repeat("a", 40) + `","html_url":"h","required":[],"statuses":[],"checks":{},"labels":[]},
|
||||
{"owner":"novox","repo":"mesh-catalog","number":301,"base":"feat/y","head":"feat/z","head_sha":"` + strings.Repeat("b", 40) + `","html_url":"h","required":[],"statuses":[],"checks":{},"labels":[]}]}`)
|
||||
}
|
||||
|
||||
func TestSyncOpenWritesOneUpdatedMessagePerOpenPull(t *testing.T) { // 1237/AC1, AC4
|
||||
var logs bytes.Buffer
|
||||
pub := &fakePub{}
|
||||
var keys []string
|
||||
call := func(_ context.Context, key string, _ any) (json.RawMessage, error) {
|
||||
keys = append(keys, key)
|
||||
return openAnswer(), nil
|
||||
}
|
||||
if err := syncOpen(context.Background(), call, pub, log.New(&logs, "", 0)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(keys) != 1 || keys[0] != "gitea.gitea_open_pulls" {
|
||||
t.Errorf("calls %v", keys)
|
||||
}
|
||||
if len(pub.subjects) != 2 {
|
||||
t.Fatalf("%d messages written (%v), want two: the pull request into feat/y gets none", len(pub.subjects), pub.subjects)
|
||||
}
|
||||
want := []string{"mesh.stream.mesh-delivery-forge-bridge.event.forge.novox.mesh-delivery.pull", "mesh.stream.mesh-delivery-forge-bridge.event.forge.novox.mesh-catalog.pull"}
|
||||
for i, w := range want {
|
||||
f, err := delivery.DecodeForge(pub.data[i])
|
||||
if pub.subjects[i] != w || err != nil || f.Action != "updated" || f.DirsKnown || f.Branch != "feat/x" {
|
||||
t.Errorf("message %d on %s = %+v err %v", i, pub.subjects[i], f, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestSyncOpenFailureNamesTheCallAndTheForgesWords(t *testing.T) { // 1237/AC3
|
||||
pub := &fakePub{}
|
||||
call := func(context.Context, string, any) (json.RawMessage, error) {
|
||||
return nil, errors.New("the forge said 502")
|
||||
}
|
||||
err := syncOpen(context.Background(), call, pub, log.New(&bytes.Buffer{}, "", 0))
|
||||
if err == nil || !strings.Contains(err.Error(), "gitea.gitea_open_pulls") || !strings.Contains(err.Error(), "the forge said 502") {
|
||||
t.Errorf("err = %v", err)
|
||||
}
|
||||
if len(pub.subjects) != 0 {
|
||||
t.Errorf("messages written: %v", pub.subjects)
|
||||
}
|
||||
}
|
||||
|
||||
func TestReconnectReadsTheOpenPullsAgainAndFailureGoesOn(t *testing.T) { // 1237/AC2, AC3
|
||||
var logs lockedBuffer
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
again := make(chan struct{}, 1)
|
||||
done := make(chan struct{})
|
||||
var mu sync.Mutex
|
||||
n := 0
|
||||
count := func() int { mu.Lock(); defer mu.Unlock(); return n }
|
||||
read := func(context.Context) error {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
n++
|
||||
if n == 1 {
|
||||
return errors.New("the forge said 502")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
go func() { onReconnect(ctx, again, read, log.New(&logs, "", 0)); close(done) }()
|
||||
again <- struct{}{}
|
||||
waitFor(t, func() bool { return count() >= 1 })
|
||||
again <- struct{}{} // the first read failed; the bridge goes on and reads again at the next reconnect
|
||||
waitFor(t, func() bool { return count() >= 2 })
|
||||
cancel()
|
||||
<-done
|
||||
if !strings.Contains(logs.String(), "the forge said 502") {
|
||||
t.Errorf("log %q does not carry the forge's words", logs.String())
|
||||
}
|
||||
}
|
||||
|
||||
type lockedBuffer struct {
|
||||
mu sync.Mutex
|
||||
b bytes.Buffer
|
||||
}
|
||||
|
||||
func (l *lockedBuffer) Write(p []byte) (int, error) {
|
||||
l.mu.Lock()
|
||||
defer l.mu.Unlock()
|
||||
return l.b.Write(p)
|
||||
}
|
||||
func (l *lockedBuffer) String() string { l.mu.Lock(); defer l.mu.Unlock(); return l.b.String() }
|
||||
|
||||
func waitFor(t *testing.T, ok func() bool) {
|
||||
t.Helper()
|
||||
for deadline := time.Now().Add(5 * time.Second); time.Now().Before(deadline); time.Sleep(5 * time.Millisecond) {
|
||||
if ok() {
|
||||
return
|
||||
}
|
||||
}
|
||||
t.Fatal("it did not happen within 5 s")
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
module mesh-delivery-forge-bridge
|
||||
|
||||
go 1.26.0
|
||||
|
||||
require (
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261019124204-9ade199740e3
|
||||
github.com/nats-io/nats-server/v2 v2.11.17
|
||||
github.com/nats-io/nats.go v1.54.0
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op // indirect
|
||||
github.com/google/go-tpm v0.9.8 // indirect
|
||||
github.com/klauspost/compress v1.20.0 // indirect
|
||||
github.com/minio/highwayhash v1.0.4 // indirect
|
||||
github.com/nats-io/jwt/v2 v2.8.1 // indirect
|
||||
github.com/nats-io/nkeys v0.4.16 // indirect
|
||||
github.com/nats-io/nuid v1.0.1 // indirect
|
||||
golang.org/x/crypto v0.57.0 // indirect
|
||||
golang.org/x/sys v0.48.0 // indirect
|
||||
golang.org/x/time v0.15.0 // indirect
|
||||
)
|
||||
@@ -0,0 +1,27 @@
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261019124204-9ade199740e3 h1:Mu2QiGbL/sZFU5ya4lGUyQOB5OKXPy/KB40XfXTf2EU=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261019124204-9ade199740e3/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
||||
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
|
||||
github.com/google/go-tpm v0.9.8/go.mod h1:h9jEsEECg7gtLis0upRBQU+GhYVH6jMjrFxI8u6bVUY=
|
||||
github.com/klauspost/compress v1.20.0 h1:a3C1ke2ohxFymNlb2HWAHjDeKCI90scRskErZkR0ezA=
|
||||
github.com/klauspost/compress v1.20.0/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI=
|
||||
github.com/minio/highwayhash v1.0.4 h1:asJizugGgchQod2ja9NJlGOWq4s7KsAWr5XUc9Clgl4=
|
||||
github.com/minio/highwayhash v1.0.4/go.mod h1:GGYsuwP/fPD6Y9hMiXuapVvlIUEhFhMTh0rxU3ik1LQ=
|
||||
github.com/nats-io/jwt/v2 v2.8.1 h1:V0xpGuD/N8Mi+fQNDynXohVvp7ZztevW5io8CUWlPmU=
|
||||
github.com/nats-io/jwt/v2 v2.8.1/go.mod h1:nWnOEEiVMiKHQpnAy4eXlizVEtSfzacZ1Q43LIRavZg=
|
||||
github.com/nats-io/nats-server/v2 v2.11.17 h1:GKEghcFK6A+aFx11Yf1LjgLC3txAwvyhnYzhBIQZA8I=
|
||||
github.com/nats-io/nats-server/v2 v2.11.17/go.mod h1:B1sFVz4StNosQ903ak4N1G01Fl/9f8e06mXpFIE2K24=
|
||||
github.com/nats-io/nats.go v1.54.0 h1:vsXoOxjHp/GmPUN+EcI7uOf/uB+iAP+kEsAFNQN0yzA=
|
||||
github.com/nats-io/nats.go v1.54.0/go.mod h1:y+DZoD1oBOYfZTU681eTUiUjI0vbqYGixNVFHcjHJ0k=
|
||||
github.com/nats-io/nkeys v0.4.16 h1:rd5oAuLOb8mnAycB0xleuEBNS1pVVnN0fv/FF34Eypg=
|
||||
github.com/nats-io/nkeys v0.4.16/go.mod h1:llLgWoI0o4z/Q57q2R1kHfmocyhGV6VG/U18Glg1Afs=
|
||||
github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw=
|
||||
github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c=
|
||||
golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M=
|
||||
golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA=
|
||||
golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo=
|
||||
golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
|
||||
golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U=
|
||||
golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno=
|
||||
@@ -0,0 +1,188 @@
|
||||
// Package bridge turns the forge module's pull request events into the messages the forge bridge writes on FORGE.
|
||||
package bridge
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"regexp"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// ErrUnreadable is wrapped by every error that says an event cannot be turned into a message; it is never
|
||||
// redelivered.
|
||||
var ErrUnreadable = errors.New("unreadable event")
|
||||
|
||||
// ErrNotHandled says the event is one the bridge hears but writes nothing for (yet).
|
||||
var ErrNotHandled = errors.New("event not handled")
|
||||
|
||||
// ErrRetry says the event is readable but the forge could not be asked what the message needs; the event is to be
|
||||
// heard again later.
|
||||
var ErrRetry = errors.New("the forge could not say")
|
||||
|
||||
// ErrNoModules says a merge into the trunk left a repository with no module.json anywhere: no merged message.
|
||||
var ErrNoModules = errors.New("no module.json in the merge")
|
||||
|
||||
// Trunk is the branch a merged message is written for.
|
||||
const Trunk = "main"
|
||||
|
||||
// Trees asks the forge for the trees of a commit: the root tree and the tree of every directory holding a module.json
|
||||
// (the root directory spelled ""), as the forge's gitea_commit_trees answers (issue 1234/AC4).
|
||||
type Trees func(owner, repo, sha string) (root string, trees map[string]string, err error)
|
||||
|
||||
// Message is a message for FORGE: the event name (forge.<owner>.<name>.<kind>) and its JSON body.
|
||||
type Message struct {
|
||||
Event string
|
||||
Data []byte
|
||||
}
|
||||
|
||||
// The subjects of the forge module's events the bridge writes pull messages for (mesh-catalog modules/gitea,
|
||||
// published by the module as mesh.mod.<module>.event.<event>).
|
||||
const (
|
||||
UpdatedSubject = "mesh.mod.gitea.event.pull.updated"
|
||||
ClosedSubject = "mesh.mod.gitea.event.pull.closed"
|
||||
MergedSubject = "mesh.mod.gitea.event.pull.merged"
|
||||
)
|
||||
|
||||
var sha40 = regexp.MustCompile(`^[0-9a-f]{40}$`)
|
||||
|
||||
// pull is the part of gitea.pull.merged (index.ts:186-207), gitea.pull.updated (mesh-catalog modules/gitea/trunk.ts:233-250) and gitea.pull.closed
|
||||
// (index.ts:127-128) the bridge reads; head is the branch and head_sha the commit.
|
||||
type pull struct {
|
||||
Owner string `json:"owner"`
|
||||
Repo string `json:"repo"`
|
||||
Number int `json:"number"`
|
||||
Head string `json:"head"`
|
||||
HeadSHA string `json:"head_sha"`
|
||||
ModuleDirs []string `json:"module_dirs"`
|
||||
ModuleDirsSaid bool `json:"module_dirs_said"`
|
||||
Base string `json:"base"`
|
||||
MergeSHA string `json:"merge_commit_sha"`
|
||||
}
|
||||
|
||||
func unreadable(subject, why string, args ...any) error {
|
||||
return fmt.Errorf("%w: %s: %s", ErrUnreadable, subject, fmt.Sprintf(why, args...))
|
||||
}
|
||||
|
||||
// Convert turns one event of the forge module, by its bus subject and body, into a message. An event it cannot read
|
||||
// is an ErrUnreadable error naming the subject and why; an event it writes nothing for is ErrNotHandled. A merge into
|
||||
// the trunk asks trees once and becomes a merged message; a merge into any other branch is a closed pull message, as the
|
||||
// pull request left its group. A forge that cannot say is ErrRetry; a merge with no module.json is ErrNoModules.
|
||||
func Convert(subject string, data []byte, trees Trees) (Message, error) {
|
||||
var action string
|
||||
switch subject {
|
||||
case UpdatedSubject:
|
||||
action = delivery.PullUpdated
|
||||
case ClosedSubject:
|
||||
action = delivery.PullClosed
|
||||
case MergedSubject:
|
||||
action = delivery.PullClosed // unless it merged into the trunk, below
|
||||
default:
|
||||
return Message{}, fmt.Errorf("%w: %s", ErrNotHandled, subject)
|
||||
}
|
||||
var p pull
|
||||
if err := json.Unmarshal(data, &p); err != nil {
|
||||
return Message{}, unreadable(subject, "the body is not json: %v", err)
|
||||
}
|
||||
switch {
|
||||
case p.Owner == "" || p.Repo == "":
|
||||
return Message{}, unreadable(subject, "no owner or repo")
|
||||
case p.Number <= 0:
|
||||
return Message{}, unreadable(subject, "no pull request number")
|
||||
case !sha40.MatchString(p.HeadSHA):
|
||||
return Message{}, unreadable(subject, "the head %q is not 40 hex digits", p.HeadSHA)
|
||||
case p.Head == "":
|
||||
return Message{}, unreadable(subject, "no branch")
|
||||
}
|
||||
if subject == MergedSubject && p.Base == Trunk {
|
||||
return merged(subject, p, trees)
|
||||
}
|
||||
f := delivery.Forge{Kind: delivery.ForgePull, Repository: p.Owner + "/" + p.Repo, Branch: p.Head, Number: p.Number,
|
||||
Head: p.HeadSHA, Action: action}
|
||||
// The directories are the forge's word only when it said it could read them (module_dirs_said).
|
||||
if action == delivery.PullUpdated && p.ModuleDirsSaid {
|
||||
f.Dirs, f.DirsKnown = p.ModuleDirs, true
|
||||
}
|
||||
event, err := delivery.ForgeEvent(f.Repository, delivery.ForgePull)
|
||||
if err != nil {
|
||||
return Message{}, unreadable(subject, "%v", err)
|
||||
}
|
||||
body, err := delivery.EncodeForge(f)
|
||||
if err != nil {
|
||||
return Message{}, unreadable(subject, "%v", err)
|
||||
}
|
||||
return Message{Event: event, Data: body}, nil
|
||||
}
|
||||
|
||||
// merged is the merged message of a merge into the trunk: the trees of the merge commit, asked once.
|
||||
func merged(subject string, p pull, trees Trees) (Message, error) {
|
||||
if !sha40.MatchString(p.MergeSHA) {
|
||||
return Message{}, unreadable(subject, "the merge commit %q is not 40 hex digits", p.MergeSHA)
|
||||
}
|
||||
repository := p.Owner + "/" + p.Repo
|
||||
root, dirs, err := trees(p.Owner, p.Repo, p.MergeSHA)
|
||||
if err != nil {
|
||||
return Message{}, fmt.Errorf("%w: %s: the trees of %s at the merge commit %s: %v", ErrRetry, subject, repository, p.MergeSHA, err)
|
||||
}
|
||||
if len(dirs) == 0 {
|
||||
return Message{}, fmt.Errorf("%w: %s: %s at the merge commit %s", ErrNoModules, subject, repository, p.MergeSHA)
|
||||
}
|
||||
event, err := delivery.ForgeEvent(repository, delivery.ForgeMerged)
|
||||
if err != nil {
|
||||
return Message{}, unreadable(subject, "%v", err)
|
||||
}
|
||||
body, err := delivery.EncodeForge(delivery.Forge{Kind: delivery.ForgeMerged, Repository: repository, Branch: p.Head, Number: p.Number,
|
||||
Head: p.HeadSHA, Merge: p.MergeSHA, Root: root, Trees: dirs})
|
||||
if err != nil {
|
||||
return Message{}, unreadable(subject, "%v", err)
|
||||
}
|
||||
return Message{Event: event, Data: body}, nil
|
||||
}
|
||||
|
||||
// openPull is an entry of gitea_open_pulls's answer (mesh-catalog modules/gitea/client.ts:369-381 openPulls, with the
|
||||
// branch as `head`, issue 1234/AC5), the part the bridge reads.
|
||||
type openPull struct {
|
||||
Owner string `json:"owner"`
|
||||
Repo string `json:"repo"`
|
||||
Number int `json:"number"`
|
||||
Base string `json:"base"`
|
||||
Head string `json:"head"`
|
||||
HeadSHA string `json:"head_sha"`
|
||||
}
|
||||
|
||||
// OpenPulls turns the answer of gitea_open_pulls, {pulls: [...]}, into one pull message per open pull request into
|
||||
// the trunk, action updated and its directories not known (the forge's answer does not say them). A pull request into
|
||||
// another branch gets none. An entry that cannot be read is named in skipped and gets none; an answer that is not
|
||||
// {pulls} is an error.
|
||||
func OpenPulls(answer []byte) (msgs []Message, skipped []string, err error) {
|
||||
var a struct {
|
||||
Pulls *[]openPull `json:"pulls"`
|
||||
}
|
||||
if err := json.Unmarshal(answer, &a); err != nil || a.Pulls == nil {
|
||||
return nil, nil, fmt.Errorf("the answer of gitea_open_pulls is not {pulls: [...]}: %v", err)
|
||||
}
|
||||
for _, p := range *a.Pulls {
|
||||
if p.Base != Trunk {
|
||||
continue
|
||||
}
|
||||
name := fmt.Sprintf("%s/%s#%d", p.Owner, p.Repo, p.Number)
|
||||
if p.Owner == "" || p.Repo == "" || p.Number <= 0 || p.Head == "" || !sha40.MatchString(p.HeadSHA) {
|
||||
skipped = append(skipped, name+": it names no owner, repo, number, branch or 40-hex head")
|
||||
continue
|
||||
}
|
||||
f := delivery.Forge{Kind: delivery.ForgePull, Repository: p.Owner + "/" + p.Repo, Branch: p.Head, Number: p.Number,
|
||||
Head: p.HeadSHA, Action: delivery.PullUpdated}
|
||||
event, err := delivery.ForgeEvent(f.Repository, delivery.ForgePull)
|
||||
var body []byte
|
||||
if err == nil {
|
||||
body, err = delivery.EncodeForge(f)
|
||||
}
|
||||
if err != nil {
|
||||
skipped = append(skipped, name+": "+err.Error())
|
||||
continue
|
||||
}
|
||||
msgs = append(msgs, Message{Event: event, Data: body})
|
||||
}
|
||||
return msgs, skipped, nil
|
||||
}
|
||||
@@ -0,0 +1,325 @@
|
||||
package bridge
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
const head = "0123456789abcdef0123456789abcdef01234567"
|
||||
|
||||
// updated is the body of gitea.pull.updated as mesh-catalog modules/gitea/trunk.ts:233-250 (announcementOf) writes
|
||||
// it: owner, repo, number, base_sha, title, body, base, head (the branch), head_sha, clone_url, html_url, paths,
|
||||
// paths_truncated, removed, module_dirs, module_dirs_said, merge_check, merge_check_said.
|
||||
func updated(extra map[string]any) []byte {
|
||||
m := map[string]any{
|
||||
"owner": "novox", "repo": "mesh-delivery", "number": 12,
|
||||
"base_sha": "ffffffffffffffffffffffffffffffffffffffff", "title": "A change", "body": "text", "base": "main",
|
||||
"head": "feat/x", "head_sha": head, "clone_url": "https://forge.example/novox/mesh-delivery.git",
|
||||
"html_url": "https://forge.example/novox/mesh-delivery/pulls/12",
|
||||
"paths": []string{"modules/a/x.go"}, "paths_truncated": false, "removed": []string{},
|
||||
"module_dirs": []string{"modules/a"}, "module_dirs_said": true,
|
||||
"merge_check": true, "merge_check_said": true,
|
||||
}
|
||||
for k, v := range extra {
|
||||
if v == nil {
|
||||
delete(m, k)
|
||||
} else {
|
||||
m[k] = v
|
||||
}
|
||||
}
|
||||
b, _ := json.Marshal(m)
|
||||
return b
|
||||
}
|
||||
|
||||
// closed is the body of gitea.pull.closed as mesh-catalog modules/gitea/index.ts:127-128 writes it: owner, repo,
|
||||
// number, title, head, head_sha, base, html_url.
|
||||
func closed() []byte {
|
||||
b, _ := json.Marshal(map[string]any{"owner": "novox", "repo": "mesh-delivery", "number": 12, "title": "A change",
|
||||
"head": "feat/x", "head_sha": head, "base": "main", "html_url": "https://forge.example/novox/mesh-delivery/pulls/12"})
|
||||
return b
|
||||
}
|
||||
|
||||
const updatedSubject = "mesh.mod.gitea.event.pull.updated"
|
||||
const closedSubject = "mesh.mod.gitea.event.pull.closed"
|
||||
|
||||
func TestUpdatedBecomesAPullMessage(t *testing.T) { // 1235/AC2 (the conversion; the bus is the integration test)
|
||||
msg, err := Convert(updatedSubject, updated(nil), nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if msg.Event != "forge.novox.mesh-delivery.pull" {
|
||||
t.Errorf("event = %q", msg.Event)
|
||||
}
|
||||
f, err := delivery.DecodeForge(msg.Data)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := delivery.Forge{V: delivery.Version, Kind: delivery.ForgePull, Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12,
|
||||
Head: head, Action: delivery.PullUpdated, Dirs: []string{"modules/a"}, DirsKnown: true}
|
||||
if !delivery.SameForge(f, want) {
|
||||
t.Errorf("message = %+v, want %+v", f, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDirsNotSaidAreNotKnown(t *testing.T) { // 1235/AC3
|
||||
msg, err := Convert(updatedSubject, updated(map[string]any{"module_dirs": nil, "module_dirs_said": nil}), nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
f, _ := delivery.DecodeForge(msg.Data)
|
||||
if f.DirsKnown || len(f.Dirs) != 0 {
|
||||
t.Errorf("dirs = %v known = %v, want not known", f.Dirs, f.DirsKnown)
|
||||
}
|
||||
// Said and empty is known and empty, not unknown.
|
||||
msg, err = Convert(updatedSubject, updated(map[string]any{"module_dirs": []string{}, "module_dirs_said": true}), nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
f, _ = delivery.DecodeForge(msg.Data)
|
||||
if !f.DirsKnown || len(f.Dirs) != 0 {
|
||||
t.Errorf("dirs = %v known = %v, want known and empty", f.Dirs, f.DirsKnown)
|
||||
}
|
||||
// Dirs listed without the forge saying it could read them are not trusted.
|
||||
msg, err = Convert(updatedSubject, updated(map[string]any{"module_dirs_said": nil}), nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if f, _ = delivery.DecodeForge(msg.Data); f.DirsKnown || len(f.Dirs) != 0 {
|
||||
t.Errorf("dirs %v known %v without module_dirs_said", f.Dirs, f.DirsKnown)
|
||||
}
|
||||
}
|
||||
|
||||
func TestClosedBecomesAClosedPullMessage(t *testing.T) { // 1235/AC4
|
||||
msg, err := Convert(closedSubject, closed(), nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if msg.Event != "forge.novox.mesh-delivery.pull" {
|
||||
t.Errorf("event = %q", msg.Event)
|
||||
}
|
||||
f, err := delivery.DecodeForge(msg.Data)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := delivery.Forge{V: delivery.Version, Kind: delivery.ForgePull, Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12,
|
||||
Head: head, Action: delivery.PullClosed}
|
||||
if !delivery.SameForge(f, want) {
|
||||
t.Errorf("message = %+v, want %+v", f, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUnreadableEventsWriteNothing(t *testing.T) { // 1235/AC5
|
||||
for name, c := range map[string]struct {
|
||||
subject string
|
||||
data []byte
|
||||
why string
|
||||
}{
|
||||
"no owner": {updatedSubject, updated(map[string]any{"owner": nil}), "owner"},
|
||||
"no repo": {updatedSubject, updated(map[string]any{"repo": nil}), "repo"},
|
||||
"no number": {updatedSubject, updated(map[string]any{"number": nil}), "number"},
|
||||
"number zero": {updatedSubject, updated(map[string]any{"number": 0}), "number"},
|
||||
"short head": {updatedSubject, updated(map[string]any{"head_sha": "abc123"}), "head"},
|
||||
"head not hex": {updatedSubject, updated(map[string]any{"head_sha": strings.Repeat("g", 40)}), "head"},
|
||||
"upper-case head": {updatedSubject, updated(map[string]any{"head_sha": strings.ToUpper(head)}), "head"},
|
||||
"no branch": {updatedSubject, updated(map[string]any{"head": nil}), "branch"},
|
||||
"not json": {updatedSubject, []byte("{"), "json"},
|
||||
"closed no head": {closedSubject, mustDrop(closed(), "head_sha"), "head"},
|
||||
"closed no repo": {closedSubject, mustDrop(closed(), "repo"), "repo"},
|
||||
} {
|
||||
msg, err := Convert(c.subject, c.data, nil)
|
||||
if err == nil {
|
||||
t.Errorf("%s: a message %q was made", name, msg.Event)
|
||||
continue
|
||||
}
|
||||
if !errors.Is(err, ErrUnreadable) {
|
||||
t.Errorf("%s: error %v is not ErrUnreadable", name, err)
|
||||
}
|
||||
if !strings.Contains(err.Error(), c.why) || !strings.Contains(err.Error(), c.subject) {
|
||||
t.Errorf("%s: error %q names neither its subject nor %q", name, err, c.why)
|
||||
}
|
||||
if msg.Data != nil {
|
||||
t.Errorf("%s: data was made", name)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func mustDrop(body []byte, key string) []byte {
|
||||
var m map[string]any
|
||||
_ = json.Unmarshal(body, &m)
|
||||
delete(m, key)
|
||||
b, _ := json.Marshal(m)
|
||||
return b
|
||||
}
|
||||
|
||||
func TestOtherSubjectsAreNotHandled(t *testing.T) { // 1235
|
||||
for _, s := range []string{"mesh.mod.gitea.event.something.else", "mesh.mod.other.event.pull.updated"} {
|
||||
if _, err := Convert(s, updated(nil), nil); !errors.Is(err, ErrNotHandled) {
|
||||
t.Errorf("%s: err = %v, want ErrNotHandled", s, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const merge = "89abcdef0123456789abcdef0123456789abcdef"
|
||||
|
||||
// mergedBody is gitea.pull.merged as mesh-catalog modules/gitea/index.ts:186-207 writes it.
|
||||
func mergedBody(extra map[string]any) []byte {
|
||||
m := map[string]any{
|
||||
"owner": "novox", "repo": "mesh-delivery", "number": 12, "title": "A change", "body": "text",
|
||||
"head_sha": head, "head_checks": map[string]string{"mesh/merge-gate": "success"}, "head": "feat/x", "base": "main",
|
||||
"merge_commit_sha": merge, "merged_at": "2026-10-19T12:00:00Z", "clone_url": "https://forge.example/novox/mesh-delivery.git",
|
||||
"html_url": "https://forge.example/novox/mesh-delivery/pulls/12", "paths": []string{"modules/a/x.go"}, "paths_truncated": false,
|
||||
"removed": []string{}, "module_dirs": []string{"modules/a"}, "module_dirs_said": true,
|
||||
}
|
||||
for k, v := range extra {
|
||||
if v == nil {
|
||||
delete(m, k)
|
||||
} else {
|
||||
m[k] = v
|
||||
}
|
||||
}
|
||||
b, _ := json.Marshal(m)
|
||||
return b
|
||||
}
|
||||
|
||||
const mergedSubject = "mesh.mod.gitea.event.pull.merged"
|
||||
|
||||
type call struct{ owner, repo, sha string }
|
||||
|
||||
// treesOf answers as gitea_commit_trees does (issue 1234/AC4: root, and the directory to tree hash, the root spelled "").
|
||||
func treesOf(calls *[]call, root string, trees map[string]string, err error) Trees {
|
||||
return func(owner, repo, sha string) (string, map[string]string, error) {
|
||||
*calls = append(*calls, call{owner, repo, sha})
|
||||
return root, trees, err
|
||||
}
|
||||
}
|
||||
|
||||
func TestMergeIntoMainBecomesAMergedMessage(t *testing.T) { // 1236/AC1
|
||||
var calls []call
|
||||
answer := map[string]string{"modules/a": "t1", "modules/b": "t2"}
|
||||
msg, err := Convert(mergedSubject, mergedBody(nil), treesOf(&calls, "r0", answer, nil))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if msg.Event != "forge.novox.mesh-delivery.merged" {
|
||||
t.Errorf("event = %q", msg.Event)
|
||||
}
|
||||
f, err := delivery.DecodeForge(msg.Data)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := delivery.Forge{V: delivery.Version, Kind: delivery.ForgeMerged, Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12,
|
||||
Head: head, Merge: merge, Root: "r0", Trees: answer}
|
||||
if !delivery.SameForge(f, want) {
|
||||
t.Errorf("message = %+v, want %+v", f, want)
|
||||
}
|
||||
if len(calls) != 1 || calls[0] != (call{"novox", "mesh-delivery", merge}) {
|
||||
t.Errorf("the tool was called %v, want once with novox, mesh-delivery and the merge commit", calls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMergeIntoAnotherBranchIsAClosedPull(t *testing.T) { // 1236/AC2
|
||||
var calls []call
|
||||
msg, err := Convert(mergedSubject, mergedBody(map[string]any{"base": "feat/y"}), treesOf(&calls, "r0", map[string]string{"a": "t"}, nil))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
f, err := delivery.DecodeForge(msg.Data)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := delivery.Forge{V: delivery.Version, Kind: delivery.ForgePull, Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12,
|
||||
Head: head, Action: delivery.PullClosed}
|
||||
if msg.Event != "forge.novox.mesh-delivery.pull" || !delivery.SameForge(f, want) {
|
||||
t.Errorf("event %q message %+v, want a closed pull message", msg.Event, f)
|
||||
}
|
||||
if len(calls) != 0 {
|
||||
t.Errorf("the tool was called for a merge into another branch: %v", calls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTreesRefusedWritesNothingAndSaysWhy(t *testing.T) { // 1236/AC3 (the conversion; the delay is in main_test)
|
||||
var calls []call
|
||||
msg, err := Convert(mergedSubject, mergedBody(nil), treesOf(&calls, "", nil, errors.New("the listing for novox/mesh-delivery was cut")))
|
||||
if !errors.Is(err, ErrRetry) || msg.Data != nil {
|
||||
t.Fatalf("err = %v data = %s, want ErrRetry and no message", err, msg.Data)
|
||||
}
|
||||
for _, w := range []string{"novox/mesh-delivery", merge, "the listing for novox/mesh-delivery was cut"} {
|
||||
if !strings.Contains(err.Error(), w) {
|
||||
t.Errorf("error %q does not name %q", err, w)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestNoModuleJSONAnywhereWritesNoMergedMessage(t *testing.T) { // 1236 (notes: a repository with no module.json)
|
||||
var calls []call
|
||||
msg, err := Convert(mergedSubject, mergedBody(nil), treesOf(&calls, "r0", map[string]string{}, nil))
|
||||
if !errors.Is(err, ErrNoModules) || msg.Data != nil {
|
||||
t.Errorf("err = %v data = %s", err, msg.Data)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMergedWithoutAMergeCommitIsUnreadable(t *testing.T) { // 1236
|
||||
var calls []call
|
||||
_, err := Convert(mergedSubject, mergedBody(map[string]any{"merge_commit_sha": "abc"}), treesOf(&calls, "r0", map[string]string{"a": "t"}, nil))
|
||||
if !errors.Is(err, ErrUnreadable) || len(calls) != 0 {
|
||||
t.Errorf("err = %v calls = %v", err, calls)
|
||||
}
|
||||
}
|
||||
|
||||
// openAnswer is gitea_open_pulls's answer as mesh-catalog modules/gitea/client.ts:369-381 builds each entry
|
||||
// (owner, repo, number, base, head, head_sha, html_url, required, statuses, checks, labels) inside {pulls}.
|
||||
func openAnswer(pulls ...map[string]any) []byte {
|
||||
b, _ := json.Marshal(map[string]any{"pulls": pulls})
|
||||
return b
|
||||
}
|
||||
|
||||
func openEntry(repo string, number int, base, branch, sha string) map[string]any {
|
||||
return map[string]any{"owner": "novox", "repo": repo, "number": number, "base": base, "head": branch, "head_sha": sha,
|
||||
"html_url": "https://forge.example/x", "required": []string{"mesh/merge-gate"}, "statuses": []string{}, "checks": map[string]string{}, "labels": []string{}}
|
||||
}
|
||||
|
||||
func TestOpenPullsAreUpdatedPullMessagesWithUnknownDirs(t *testing.T) { // 1237/AC1
|
||||
h2 := strings.Repeat("a", 40)
|
||||
msgs, skipped, err := OpenPulls(openAnswer(openEntry("mesh-delivery", 12, "main", "feat/x", head), openEntry("mesh-catalog", 300, "main", "feat/x", h2)))
|
||||
if err != nil || len(skipped) != 0 || len(msgs) != 2 {
|
||||
t.Fatalf("msgs %d skipped %v err %v", len(msgs), skipped, err)
|
||||
}
|
||||
for i, want := range []delivery.Forge{
|
||||
{V: delivery.Version, Kind: delivery.ForgePull, Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12, Head: head, Action: delivery.PullUpdated},
|
||||
{V: delivery.Version, Kind: delivery.ForgePull, Repository: "novox/mesh-catalog", Branch: "feat/x", Number: 300, Head: h2, Action: delivery.PullUpdated},
|
||||
} {
|
||||
f, err := delivery.DecodeForge(msgs[i].Data)
|
||||
if err != nil || !delivery.SameForge(f, want) {
|
||||
t.Errorf("message %d = %+v err %v, want %+v", i, f, err, want)
|
||||
}
|
||||
if f.DirsKnown {
|
||||
t.Errorf("message %d says its directories are known", i)
|
||||
}
|
||||
}
|
||||
if msgs[0].Event != "forge.novox.mesh-delivery.pull" || msgs[1].Event != "forge.novox.mesh-catalog.pull" {
|
||||
t.Errorf("events %q %q", msgs[0].Event, msgs[1].Event)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOpenPullIntoAnotherBranchGetsNoMessage(t *testing.T) { // 1237/AC4
|
||||
msgs, skipped, err := OpenPulls(openAnswer(openEntry("mesh-delivery", 12, "feat/y", "feat/x", head)))
|
||||
if err != nil || len(msgs) != 0 || len(skipped) != 0 {
|
||||
t.Errorf("msgs %d skipped %v err %v, want none", len(msgs), skipped, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOpenPullsUnreadableEntryIsNamedAndTheRestGo(t *testing.T) { // 1237
|
||||
msgs, skipped, err := OpenPulls(openAnswer(openEntry("a", 1, "main", "b", "short"), openEntry("mesh-delivery", 12, "main", "feat/x", head)))
|
||||
if err != nil || len(msgs) != 1 || len(skipped) != 1 || !strings.Contains(skipped[0], "novox/a#1") {
|
||||
t.Errorf("msgs %d skipped %v err %v", len(msgs), skipped, err)
|
||||
}
|
||||
for _, bad := range []string{`{"pulls":null}`, `{}`, `[]`, `x`} {
|
||||
if _, _, err := OpenPulls([]byte(bad)); err == nil {
|
||||
t.Errorf("%s was taken as an answer", bad)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,168 @@
|
||||
package manifest
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// The controller's manifest reader is mesh-controller internal/catalogue (Manifest, manifest.go: Consumes :521,
|
||||
// Invokes :590, Streams :613, Reads :659; StreamDeclaration, streams.go), an internal package of another repository.
|
||||
// These tests decode the manifest strictly into the fields those types name (DisallowUnknownFields) and hold the
|
||||
// rules of issue 1235 (ADR 0437 §4). The merge gate's module check proves the controller takes the manifest.
|
||||
|
||||
type stream struct {
|
||||
Name string `json:"name"`
|
||||
Events []string `json:"events"`
|
||||
}
|
||||
|
||||
type artifact struct {
|
||||
Name string `json:"name"`
|
||||
Kind string `json:"kind"`
|
||||
Language string `json:"language"`
|
||||
System string `json:"system"`
|
||||
From string `json:"from"`
|
||||
Binary string `json:"binary"`
|
||||
}
|
||||
|
||||
type manifest struct {
|
||||
Module string `json:"module"`
|
||||
Version string `json:"version"`
|
||||
Slug string `json:"slug"`
|
||||
Consumes []string `json:"consumes"`
|
||||
Streams []stream `json:"streams"`
|
||||
Reads []string `json:"reads"`
|
||||
Invokes []string `json:"invokes"`
|
||||
OwnSecrets map[string]string `json:"own-secrets"`
|
||||
SecretsOwner string `json:"secrets-owner"`
|
||||
Resources []map[string]any `json:"resources"`
|
||||
Build struct {
|
||||
Artifacts []artifact `json:"artifacts"`
|
||||
} `json:"build"`
|
||||
}
|
||||
|
||||
func load(t *testing.T) (manifest, map[string]json.RawMessage) {
|
||||
t.Helper()
|
||||
raw, err := os.ReadFile("module.json")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var m manifest
|
||||
dec := json.NewDecoder(bytes.NewReader(raw))
|
||||
dec.DisallowUnknownFields()
|
||||
if err := dec.Decode(&m); err != nil {
|
||||
t.Fatalf("the controller's reader refuses a key this manifest has: %v", err)
|
||||
}
|
||||
var top map[string]json.RawMessage
|
||||
if err := json.Unmarshal(raw, &top); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return m, top
|
||||
}
|
||||
|
||||
func sorted(s []string) []string {
|
||||
s = slices.Clone(s)
|
||||
slices.Sort(s)
|
||||
return s
|
||||
}
|
||||
|
||||
func TestManifestDeclarations(t *testing.T) { // 1235/AC1
|
||||
m, top := load(t)
|
||||
if m.Module != "mesh-delivery-forge-bridge" {
|
||||
t.Errorf("module = %q", m.Module)
|
||||
}
|
||||
if !reflect.DeepEqual(m.Streams, []stream{{Name: "forge", Events: []string{"forge.**"}}}) {
|
||||
t.Errorf("streams = %+v", m.Streams)
|
||||
}
|
||||
if want := []string{"gitea.pull.closed", "gitea.pull.merged", "gitea.pull.updated"}; !reflect.DeepEqual(sorted(m.Consumes), want) {
|
||||
t.Errorf("consumes = %v, want %v", m.Consumes, want)
|
||||
}
|
||||
if want := []string{"mesh-delivery-gate.merges", "mesh-delivery-gate.owed"}; !reflect.DeepEqual(sorted(m.Reads), want) {
|
||||
t.Errorf("reads = %v, want %v", m.Reads, want)
|
||||
}
|
||||
if want := []string{"gitea.gitea_add_comment", "gitea.gitea_commit_status", "gitea.gitea_commit_trees", "gitea.gitea_delivery_view",
|
||||
"gitea.gitea_merge_pull_request", "gitea.gitea_note_append", "gitea.gitea_open_pulls"}; !reflect.DeepEqual(sorted(m.Invokes), want) {
|
||||
t.Errorf("invokes = %v, want %v", m.Invokes, want)
|
||||
}
|
||||
if b, ok := m.OwnSecrets["broker"]; !ok || len(m.OwnSecrets) != 1 || b != "${dir:state}/broker" {
|
||||
t.Errorf("own-secrets = %v, want exactly broker", m.OwnSecrets)
|
||||
}
|
||||
if m.SecretsOwner != "mesh-delivery-forge-bridge" {
|
||||
t.Errorf("secrets-owner = %q", m.SecretsOwner)
|
||||
}
|
||||
// No state, no bus, no seat, no tool (ADR 0437 §4).
|
||||
for _, k := range []string{"state", "logs", "emits", "claims", "provides", "requires", "uses", "defines-seats", "tools", "seat", "seats"} {
|
||||
if _, ok := top[k]; ok {
|
||||
t.Errorf("the manifest declares %q: the bridge has none", k)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestManifestProcess(t *testing.T) { // 1235/AC1
|
||||
m, _ := load(t)
|
||||
var procs []map[string]any
|
||||
for _, r := range m.Resources {
|
||||
if r["type"] == "process" {
|
||||
procs = append(procs, r)
|
||||
}
|
||||
if r["type"] == "tool" || r["type"] == "seat" {
|
||||
t.Errorf("resource %v: the bridge serves no tool and holds no seat", r)
|
||||
}
|
||||
}
|
||||
if len(procs) != 1 {
|
||||
t.Fatalf("%d process resources, want one", len(procs))
|
||||
}
|
||||
p := procs[0]
|
||||
if p["artifact"] != "program" || !reflect.DeepEqual(p["run"], []any{"./mesh-delivery-forge-bridge"}) {
|
||||
t.Errorf("process = %v", p)
|
||||
}
|
||||
env, _ := p["env"].(map[string]any)
|
||||
if len(env) != 1 || env["MESH_BROKER_FILE"] != "${dir:state}/broker" {
|
||||
t.Errorf("env = %v", p["env"])
|
||||
}
|
||||
if _, bus := p["bus"]; bus {
|
||||
t.Errorf("the process declares a bus: it runs on its own broker credential")
|
||||
}
|
||||
if len(m.Build.Artifacts) != 1 {
|
||||
t.Fatalf("artifacts = %+v", m.Build.Artifacts)
|
||||
}
|
||||
a := m.Build.Artifacts[0]
|
||||
if st, err := os.Stat(a.From); err != nil || !st.IsDir() || a.Binary != "mesh-delivery-forge-bridge" || !strings.HasSuffix(a.From, a.Binary) {
|
||||
t.Errorf("artifact = %+v", a)
|
||||
}
|
||||
}
|
||||
|
||||
func TestServesNoRequestSubjectAndHoldsNoTicker(t *testing.T) { // 1235/AC6 (ADR 0437 §8)
|
||||
forbidden := []string{"Subscribe(", "QueueSubscribe", "ChanSubscribe", "time.NewTicker", "time.Tick", "time.NewTimer", "time.AfterFunc", "time.After(", "time.Sleep"}
|
||||
var files int
|
||||
for _, root := range []string{"cmd", "internal"} {
|
||||
err := filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error {
|
||||
if err != nil || d.IsDir() || !strings.HasSuffix(path, ".go") || strings.HasSuffix(path, "_test.go") {
|
||||
return err
|
||||
}
|
||||
files++
|
||||
src, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, f := range forbidden {
|
||||
if strings.Contains(string(src), f) {
|
||||
t.Errorf("%s holds %s: the bridge serves no request subject and holds no timer", path, f)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if files < 2 {
|
||||
t.Fatalf("read %d source files, want the program's", files)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,75 @@
|
||||
{
|
||||
"module": "mesh-delivery-forge-bridge",
|
||||
"version": "1",
|
||||
"slug": "forge-bridge",
|
||||
"consumes": [
|
||||
"gitea.pull.updated",
|
||||
"gitea.pull.merged",
|
||||
"gitea.pull.closed"
|
||||
],
|
||||
"streams": [
|
||||
{ "name": "forge", "events": ["forge.**"] }
|
||||
],
|
||||
"reads": [
|
||||
"mesh-delivery-gate.merges",
|
||||
"mesh-delivery-gate.owed"
|
||||
],
|
||||
"invokes": [
|
||||
"gitea.gitea_merge_pull_request",
|
||||
"gitea.gitea_commit_status",
|
||||
"gitea.gitea_delivery_view",
|
||||
"gitea.gitea_note_append",
|
||||
"gitea.gitea_add_comment",
|
||||
"gitea.gitea_open_pulls",
|
||||
"gitea.gitea_commit_trees"
|
||||
],
|
||||
"own-secrets": {
|
||||
"broker": "${dir:state}/broker"
|
||||
},
|
||||
"secrets-owner": "mesh-delivery-forge-bridge",
|
||||
"resources": [
|
||||
{
|
||||
"id": "account",
|
||||
"type": "user",
|
||||
"name": "mesh-delivery-forge-bridge",
|
||||
"shell": "/usr/bin/nologin",
|
||||
"home": "/var/lib/mesh-delivery-forge-bridge"
|
||||
},
|
||||
{
|
||||
"id": "state",
|
||||
"type": "directory",
|
||||
"mode": "0700",
|
||||
"owner": "mesh-delivery-forge-bridge",
|
||||
"place": ".",
|
||||
"unknown": "program"
|
||||
},
|
||||
{
|
||||
"id": "bridge",
|
||||
"type": "process",
|
||||
"name": "mesh-delivery-forge-bridge",
|
||||
"artifact": "program",
|
||||
"run": [
|
||||
"./mesh-delivery-forge-bridge"
|
||||
],
|
||||
"env": {
|
||||
"MESH_BROKER_FILE": "${dir:state}/broker"
|
||||
},
|
||||
"user": "mesh-delivery-forge-bridge",
|
||||
"health": {
|
||||
"kind": "unit"
|
||||
}
|
||||
}
|
||||
],
|
||||
"build": {
|
||||
"artifacts": [
|
||||
{
|
||||
"name": "program",
|
||||
"kind": "bundle",
|
||||
"language": "go",
|
||||
"system": "arch",
|
||||
"from": "cmd/mesh-delivery-forge-bridge",
|
||||
"binary": "mesh-delivery-forge-bridge"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,205 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats-server/v2/server"
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// busConf is the shape of the composed accounts file (mesh-controller internal/broker): one account with JetStream, the
|
||||
// controller's user and the indexer's own principal <node>.<module>.
|
||||
const busConf = `
|
||||
listen: 127.0.0.1:-1
|
||||
jetstream { store_dir: %q }
|
||||
accounts {
|
||||
MESH {
|
||||
jetstream: enabled
|
||||
users = [
|
||||
{ user: "ctl", password: "pw" }
|
||||
{ user: "n1.mesh-delivery-indexer", password: "pw" }
|
||||
]
|
||||
}
|
||||
}
|
||||
`
|
||||
|
||||
// TestIndexerWhole starts the program the way the node does (its entry, run, with MESH_BROKER_FILE) against an embedded
|
||||
// bus with the stream FORGE and the indexer's durable <node>_<module> on it, the buckets the controller makes from the
|
||||
// manifest's state (mesh-delivery-indexer_candidates, history 10) and mesh-controller_builds holding build records, and
|
||||
// no mesh-delivery-gate_source bucket.
|
||||
func TestIndexerWhole(t *testing.T) { // 1238/AC2, AC10
|
||||
js, ctx, candidates, _ := startIndexer(t, func(ctx context.Context, builds jetstream.KeyValue) {
|
||||
// Build records in the controller's layout (mesh-sdk go/delivery/testdata/build-record-build-agent.json, captured from
|
||||
// the controller's encoder), under delivery.BuildKey, with the newest pointer beside them.
|
||||
for _, b := range []struct{ repo, module, path, id string }{
|
||||
{"novox/mesh-delivery", "a", "modules/a", "build-1"}, {"novox/mesh-catalog", "c", "modules/c", "build-2"},
|
||||
} {
|
||||
rec, _ := json.Marshal(map[string]any{"id": b.id, "state": "built", "repository": b.repo, "ref": "main", "module": b.module,
|
||||
"commit": "7a74cee83f88e6585a5511c3cc53bdf0f5303d20", "path": b.path, "at": "2026-10-19T09:38:00Z"})
|
||||
if _, err := builds.Put(ctx, delivery.BuildKey(b.module, b.id), rec); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := builds.Put(ctx, delivery.NewestKey(b.module), []byte(b.id)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
})
|
||||
h1, h2 := "0123456789abcdef0123456789abcdef01234567", "89abcdef0123456789abcdef0123456789abcdef"
|
||||
for _, f := range []delivery.Forge{
|
||||
{Kind: delivery.ForgePull, Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12, Head: h1, Action: delivery.PullUpdated, Dirs: []string{"modules/a"}, DirsKnown: true},
|
||||
{Kind: delivery.ForgePull, Repository: "novox/mesh-catalog", Branch: "feat/x", Number: 300, Head: h2, Action: delivery.PullUpdated, Dirs: []string{"modules/c"}, DirsKnown: true},
|
||||
} {
|
||||
data, err := delivery.EncodeForge(f)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
event, _ := delivery.ForgeEvent(f.Repository, delivery.ForgePull)
|
||||
if _, err := js.Publish(ctx, "mesh.stream.mesh-delivery-forge-bridge.event."+event, data); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
hash, _ := delivery.HeadsHash([]delivery.Head{{Repository: "novox/mesh-delivery", Number: 12, Head: h1}, {Repository: "novox/mesh-catalog", Number: 300, Head: h2}})
|
||||
var c delivery.Candidate
|
||||
for deadline := time.Now().Add(5 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) {
|
||||
e, err := candidates.Get(ctx, "feat/x")
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
if c, err = delivery.DecodeCandidate(e.Value()); err == nil && len(c.Pulls) == 2 {
|
||||
break
|
||||
}
|
||||
}
|
||||
if c.Group != "feat/x" || c.Base != 0 || c.HeadsHash != hash || len(c.Pulls) != 2 {
|
||||
t.Fatalf("candidate = %+v within 5 s", c)
|
||||
}
|
||||
for _, p := range c.Pulls {
|
||||
want := map[string]string{"novox/mesh-delivery": "a", "novox/mesh-catalog": "c"}[p.Repository]
|
||||
if len(p.Modules) != 1 || p.Modules[0] != want {
|
||||
t.Errorf("%s#%d modules = %v, want [%s]", p.Repository, p.Number, p.Modules, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// startIndexer starts the bus, the stream FORGE with the indexer's durable consumer, the buckets, lets put fill
|
||||
// mesh-controller_builds, and runs the program. mesh-delivery-gate_source is never created.
|
||||
func startIndexer(t *testing.T, put func(context.Context, jetstream.KeyValue)) (jetstream.JetStream, context.Context, jetstream.KeyValue, jetstream.KeyValue) {
|
||||
dir, err := os.MkdirTemp(os.Getenv("TMPDIR"), "i")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { os.RemoveAll(dir) })
|
||||
conf := filepath.Join(dir, "bus.conf")
|
||||
if err := os.WriteFile(conf, []byte(fmt.Sprintf(busConf, filepath.Join(dir, "js"))), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
opts, err := server.ProcessConfigFile(conf)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
opts.NoLog, opts.NoSigs = true, true
|
||||
srv, err := server.NewServer(opts)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
go srv.Start()
|
||||
if !srv.ReadyForConnections(10 * time.Second) {
|
||||
t.Fatal("the bus did not come up")
|
||||
}
|
||||
t.Cleanup(srv.Shutdown)
|
||||
|
||||
ctl, err := nats.Connect(srv.ClientURL(), nats.UserInfo("ctl", "pw"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(ctl.Close)
|
||||
js, err := jetstream.New(ctl)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
|
||||
t.Cleanup(cancel)
|
||||
if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: "FORGE", Subjects: []string{"mesh.stream.mesh-delivery-forge-bridge.event.forge.>"}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := js.CreateOrUpdateConsumer(ctx, "FORGE", jetstream.ConsumerConfig{Durable: "n1_mesh-delivery-indexer", AckPolicy: jetstream.AckExplicitPolicy}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
candidates, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: delivery.CandidatesBucket, History: delivery.CandidatesHistory})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
modules, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: delivery.ModulesBucket, History: delivery.ModulesHistory})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
builds, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: delivery.BuildsBucket, History: 5})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
put(ctx, builds)
|
||||
|
||||
broker := filepath.Join(t.TempDir(), "broker")
|
||||
cred, _ := json.Marshal(credential{URL: srv.ClientURL(), Node: "n1", Module: "mesh-delivery-indexer", User: "n1.mesh-delivery-indexer", Password: "pw"})
|
||||
if err := os.WriteFile(broker, cred, 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
env := map[string]string{"MESH_BROKER_FILE": broker}
|
||||
runCtx, stop := context.WithCancel(ctx)
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- run(runCtx, func(k string) string { return env[k] }, io.Discard) }()
|
||||
t.Cleanup(func() { stop(); <-done })
|
||||
|
||||
return js, ctx, candidates, modules
|
||||
}
|
||||
|
||||
// TestIndexerWritesModulesAtAMerge: a module's record and its newest pointer are in mesh-controller_builds; a merged
|
||||
// message on FORGE makes modules.<m> within 5 s, the fingerprint of the merge's tree and the recorded inputs.
|
||||
func TestIndexerWritesModulesAtAMerge(t *testing.T) { // 1239/AC9
|
||||
// Inputs and record in the controller's layout: SourceInputs under its Go field names (mesh-sdk
|
||||
// go/delivery/testdata/build-record-build-agent.json, captured from the controller's encoder).
|
||||
const inputs = `{"Module":"a","Tree":"oldtree","Bases":["registry.example/base@sha256:aaa"],"Contexts":null,"Toolchains":{"code":"go sha256:ccc 0123456789abcdef"},"Unpinned":""}`
|
||||
js, ctx, _, modules := startIndexer(t, func(ctx context.Context, builds jetstream.KeyValue) {
|
||||
rec := `{"id":"build-1","state":"built","repository":"novox/mesh-delivery","ref":"main","module":"a","commit":"7a74cee83f88e6585a5511c3cc53bdf0f5303d20","path":"modules/a","at":"2026-10-19T09:38:00Z","source-inputs":` + inputs + `}`
|
||||
if _, err := builds.Put(ctx, delivery.BuildKey("a", "build-1"), []byte(rec)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := builds.Put(ctx, delivery.NewestKey("a"), []byte("build-1")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
})
|
||||
const merge = "89abcdef0123456789abcdef0123456789abcdef"
|
||||
data, err := delivery.EncodeForge(delivery.Forge{Kind: delivery.ForgeMerged, Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12,
|
||||
Head: "0123456789abcdef0123456789abcdef01234567", Merge: merge, Root: "r0", Trees: map[string]string{"modules/a": "newtree"}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := js.Publish(ctx, "mesh.stream.mesh-delivery-forge-bridge.event.forge.novox.mesh-delivery.merged", data); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want, _ := delivery.SourceFingerprint(delivery.SourceInputs{Module: "a", Tree: "newtree", Bases: []string{"registry.example/base@sha256:aaa"},
|
||||
Toolchains: map[string]string{"code": "go sha256:ccc 0123456789abcdef"}})
|
||||
var m delivery.Module
|
||||
for deadline := time.Now().Add(5 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) {
|
||||
if e, err := modules.Get(ctx, "a"); err == nil {
|
||||
if m, err = delivery.DecodeModule(e.Value()); err == nil {
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
if m.FP != want || m.Repository != "novox/mesh-delivery" || m.Commit != merge {
|
||||
t.Errorf("module = %+v within 5 s, want fp %s", m, want)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,284 @@
|
||||
// Command mesh-delivery-indexer is delivery's indexer (novox/hq ADR 0437 §4, §5, issue 1238): it hears the forge
|
||||
// bridge's messages on the stream FORGE and keeps the candidate groups (candidates.<group>): the pull requests that share
|
||||
// a branch, their heads and the modules each touches. It reads the controller's builds to know which module a changed
|
||||
// directory is. It serves no request subject and holds no timer: every reaction is a message or a watch (ADR 0437 §8).
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"crypto/tls"
|
||||
"crypto/x509"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"os"
|
||||
"os/signal"
|
||||
"strings"
|
||||
"syscall"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
|
||||
"mesh-delivery-indexer/internal/index"
|
||||
)
|
||||
|
||||
// forgeStream is the bridge's stream as the bus names it: its local name in capitals (mesh-control
|
||||
// internal/catalogue/streams.go StreamBusName). The module's consumer on it is the durable <node>_<module> the controller
|
||||
// makes (internal/broker/modstreams.go HeardIssued); the indexer binds it, pulls and acks.
|
||||
const forgeStream = "FORGE"
|
||||
|
||||
func main() {
|
||||
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
|
||||
defer stop()
|
||||
if err := run(ctx, os.Getenv, os.Stderr); err != nil {
|
||||
fmt.Fprintln(os.Stderr, "mesh-delivery-indexer:", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
// credential is the broker file the controller issues to the module: JSON of the bus address, the certificate
|
||||
// fingerprint to pin, the principal and its password (as the watchman reads it; issue 1242 moves this into the SDK).
|
||||
type credential struct {
|
||||
URL string `json:"url"`
|
||||
Fingerprint string `json:"fingerprint"`
|
||||
Node string `json:"node"`
|
||||
Module string `json:"module"`
|
||||
User string `json:"user"`
|
||||
Password string `json:"password"`
|
||||
}
|
||||
|
||||
func readCredential(path string) (credential, error) {
|
||||
raw, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return credential{}, err
|
||||
}
|
||||
var c credential
|
||||
if err := json.Unmarshal(raw, &c); err != nil {
|
||||
return credential{}, fmt.Errorf("the broker credential is not one: %w", err)
|
||||
}
|
||||
if c.URL == "" || c.User == "" {
|
||||
return credential{}, errors.New("the broker credential names no bus address or user")
|
||||
}
|
||||
return c, nil
|
||||
}
|
||||
|
||||
// pinned accepts exactly the certificate with this SHA-256 and no other.
|
||||
func pinned(want string) *tls.Config {
|
||||
return &tls.Config{
|
||||
InsecureSkipVerify: true, //nolint:gosec // replaced by the pin below, which is stricter
|
||||
MinVersion: tls.VersionTLS12,
|
||||
VerifyPeerCertificate: func(rawCerts [][]byte, _ [][]*x509.Certificate) error {
|
||||
if len(rawCerts) == 0 {
|
||||
return errors.New("the bus presented no certificate")
|
||||
}
|
||||
sum := sha256.Sum256(rawCerts[0])
|
||||
if got := "sha256:" + hex.EncodeToString(sum[:]); got != want {
|
||||
return fmt.Errorf("the bus presented a certificate this mesh does not know (%.23s…)", got)
|
||||
}
|
||||
return nil
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// kvStore is the candidates bucket as the indexer's store: compare-and-set through Create, Update and Delete at a revision.
|
||||
type kvStore struct{ kv jetstream.KeyValue }
|
||||
|
||||
func conflict(err error) error {
|
||||
if errors.Is(err, jetstream.ErrKeyExists) || errors.Is(err, jetstream.ErrKeyRevisionMismatch) {
|
||||
return index.ErrConflict
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
func (s kvStore) Get(ctx context.Context, key string) ([]byte, uint64, bool, error) {
|
||||
e, err := s.kv.Get(ctx, key)
|
||||
if errors.Is(err, jetstream.ErrKeyNotFound) {
|
||||
return nil, 0, false, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, 0, false, err
|
||||
}
|
||||
return e.Value(), e.Revision(), true, nil
|
||||
}
|
||||
|
||||
func (s kvStore) Create(ctx context.Context, key string, value []byte) error {
|
||||
_, err := s.kv.Create(ctx, key, value)
|
||||
return conflict(err)
|
||||
}
|
||||
|
||||
func (s kvStore) Update(ctx context.Context, key string, value []byte, revision uint64) error {
|
||||
_, err := s.kv.Update(ctx, key, value, revision)
|
||||
return conflict(err)
|
||||
}
|
||||
|
||||
func (s kvStore) Delete(ctx context.Context, key string, revision uint64) error {
|
||||
return conflict(s.kv.Delete(ctx, key, jetstream.LastRevision(revision)))
|
||||
}
|
||||
|
||||
// delivered is the part of a JetStream message the indexer uses.
|
||||
type delivered interface {
|
||||
Subject() string
|
||||
Data() []byte
|
||||
Ack() error
|
||||
Term() error
|
||||
}
|
||||
|
||||
// handle takes one message of FORGE into the candidates and acknowledges it. A message it can never read is terminated,
|
||||
// never redelivered, and the log names its subject and why. A write that failed stays unacknowledged: the server
|
||||
// delivers it again (the server's ack wait is the only clock).
|
||||
func handle(ctx context.Context, m delivered, ix *index.Indexer, logger *log.Logger) {
|
||||
f, err := delivery.DecodeForge(m.Data())
|
||||
if err != nil {
|
||||
logger.Printf("%s: not a forge message, dropped: %v", m.Subject(), err)
|
||||
_ = m.Term()
|
||||
return
|
||||
}
|
||||
switch err := ix.Apply(ctx, f); {
|
||||
case errors.Is(err, index.ErrUnreadable):
|
||||
logger.Printf("%s: %v; dropped", m.Subject(), err)
|
||||
_ = m.Term()
|
||||
case err != nil:
|
||||
logger.Printf("%s: %s#%d left for redelivery: %v", m.Subject(), f.Repository, f.Number, err)
|
||||
default:
|
||||
_ = m.Ack()
|
||||
}
|
||||
}
|
||||
|
||||
func opOf(e jetstream.KeyValueEntry) bool {
|
||||
return e.Operation() == jetstream.KeyValuePut
|
||||
}
|
||||
|
||||
// settle reads the entries a watcher holds now, up to its mark of the last values, and returns the watcher for the
|
||||
// changes that follow.
|
||||
func settle(ctx context.Context, kv jetstream.KeyValue, each func(jetstream.KeyValueEntry)) (jetstream.KeyWatcher, error) {
|
||||
w, err := kv.WatchAll(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
w.Stop()
|
||||
return nil, ctx.Err()
|
||||
case e, ok := <-w.Updates():
|
||||
if !ok {
|
||||
return nil, errors.New("the watch ended")
|
||||
}
|
||||
if e == nil {
|
||||
return w, nil
|
||||
}
|
||||
each(e)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// follow hands each change of a watcher to each until ctx ends; a watch that ends is an error, and the process
|
||||
// manager starts the program again.
|
||||
func follow(ctx context.Context, w jetstream.KeyWatcher, each func(jetstream.KeyValueEntry)) error {
|
||||
defer w.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return nil
|
||||
case e, ok := <-w.Updates():
|
||||
if !ok {
|
||||
return errors.New("a watch ended")
|
||||
}
|
||||
if e != nil {
|
||||
each(e)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// run is the program: it returns when ctx ends or the bus cannot be used.
|
||||
func run(ctx context.Context, getenv func(string) string, logOut io.Writer) error {
|
||||
logger := log.New(logOut, "mesh-delivery-indexer: ", log.LstdFlags)
|
||||
path := strings.TrimSpace(getenv("MESH_BROKER_FILE"))
|
||||
if path == "" {
|
||||
return errors.New("MESH_BROKER_FILE is not set, so there is no credential to hear the bus with")
|
||||
}
|
||||
cred, err := readCredential(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
opts := []nats.Option{
|
||||
nats.Name(cred.Node + "." + cred.Module),
|
||||
nats.UserInfo(cred.User, cred.Password),
|
||||
nats.CustomInboxPrefix("_INBOX." + cred.User),
|
||||
nats.MaxReconnects(-1),
|
||||
nats.RetryOnFailedConnect(true),
|
||||
}
|
||||
if strings.TrimSpace(cred.Fingerprint) != "" {
|
||||
opts = append(opts, nats.Secure(pinned(cred.Fingerprint)))
|
||||
}
|
||||
conn, err := nats.Connect(cred.URL, opts...)
|
||||
if err != nil {
|
||||
return fmt.Errorf("connecting to the bus: %w", err)
|
||||
}
|
||||
defer conn.Close()
|
||||
js, err := jetstream.New(conn)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
bucket := func(name string) (jetstream.KeyValue, error) {
|
||||
kv, err := js.KeyValue(ctx, name)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the bucket %s: %w", name, err)
|
||||
}
|
||||
return kv, nil
|
||||
}
|
||||
candidates, err := bucket(delivery.CandidatesBucket)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
modules, err := bucket(delivery.ModulesBucket)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
builds, err := bucket(delivery.BuildsBucket)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// The gate's source bucket (mesh-delivery-gate_source) is read from release 3d on; it does not exist before, and the
|
||||
// indexer does not wait for it.
|
||||
|
||||
known := index.NewBuilds()
|
||||
each := func(e jetstream.KeyValueEntry) {
|
||||
if opOf(e) {
|
||||
known.Put(e.Key(), e.Value())
|
||||
} else {
|
||||
known.Delete(e.Key())
|
||||
}
|
||||
}
|
||||
w, err := settle(ctx, builds, each)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
errs := make(chan error, 1)
|
||||
go func() { errs <- follow(ctx, w, each) }()
|
||||
|
||||
ix := &index.Indexer{Store: kvStore{candidates}, Modules: kvStore{modules}, Builds: known, Logf: logger.Printf}
|
||||
cons, err := js.Consumer(ctx, forgeStream, cred.Node+"_"+cred.Module)
|
||||
if err != nil {
|
||||
return fmt.Errorf("the consumer %s_%s on %s: %w", cred.Node, cred.Module, forgeStream, err)
|
||||
}
|
||||
cc, err := cons.Consume(func(m jetstream.Msg) { handle(ctx, m, ix, logger) },
|
||||
jetstream.ConsumeErrHandler(func(_ jetstream.ConsumeContext, err error) { logger.Printf("hearing the bus: %v", err) }))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer cc.Stop()
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return nil
|
||||
case err := <-errs:
|
||||
return err
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,62 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"log"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
|
||||
"mesh-delivery-indexer/internal/index"
|
||||
)
|
||||
|
||||
type fakeMsg struct {
|
||||
subject string
|
||||
data []byte
|
||||
acked, termd int
|
||||
}
|
||||
|
||||
func (f *fakeMsg) Subject() string { return f.subject }
|
||||
func (f *fakeMsg) Data() []byte { return f.data }
|
||||
func (f *fakeMsg) Ack() error { f.acked++; return nil }
|
||||
func (f *fakeMsg) Term() error { f.termd++; return nil }
|
||||
|
||||
type memStore struct{ vals map[string][]byte }
|
||||
|
||||
func (m *memStore) Get(_ context.Context, k string) ([]byte, uint64, bool, error) {
|
||||
v, ok := m.vals[k]
|
||||
return v, 1, ok, nil
|
||||
}
|
||||
func (m *memStore) Create(_ context.Context, k string, v []byte) error { m.vals[k] = v; return nil }
|
||||
func (m *memStore) Update(_ context.Context, k string, v []byte, _ uint64) error {
|
||||
m.vals[k] = v
|
||||
return nil
|
||||
}
|
||||
func (m *memStore) Delete(_ context.Context, k string, _ uint64) error { delete(m.vals, k); return nil }
|
||||
|
||||
const head = "0123456789abcdef0123456789abcdef01234567"
|
||||
|
||||
func TestAMessageIsIndexedAndAcked(t *testing.T) { // 1238/AC2 (unit part)
|
||||
st := &memStore{vals: map[string][]byte{}}
|
||||
ix := &index.Indexer{Store: st, Builds: index.NewBuilds()}
|
||||
data, _ := delivery.EncodeForge(delivery.Forge{Kind: delivery.ForgePull, Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12, Head: head,
|
||||
Action: delivery.PullUpdated, Dirs: []string{"modules/a"}, DirsKnown: true})
|
||||
m := &fakeMsg{subject: "mesh.stream.mesh-delivery-forge-bridge.event.forge.novox.mesh-delivery.pull", data: data}
|
||||
handle(context.Background(), m, ix, log.New(&bytes.Buffer{}, "", 0))
|
||||
c, err := delivery.DecodeCandidate(st.vals["feat/x"])
|
||||
if err != nil || len(c.Pulls) != 1 || c.Pulls[0].Modules[0] != "a" || m.acked != 1 || m.termd != 0 {
|
||||
t.Errorf("candidate %+v err %v acked %d terminated %d", c, err, m.acked, m.termd)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAMessageThatIsNoForgeMessageIsTerminatedAndLogged(t *testing.T) { // 1238 (never redelivered)
|
||||
var logs bytes.Buffer
|
||||
st := &memStore{vals: map[string][]byte{}}
|
||||
m := &fakeMsg{subject: "mesh.stream.mesh-delivery-forge-bridge.event.forge.x.y.pull", data: []byte(`{"v":1,"kind":"pull"}`)}
|
||||
handle(context.Background(), m, &index.Indexer{Store: st, Builds: index.NewBuilds()}, log.New(&logs, "", 0))
|
||||
if m.termd != 1 || m.acked != 0 || len(st.vals) != 0 || !strings.Contains(logs.String(), m.subject) {
|
||||
t.Errorf("terminated %d acked %d keys %d log %q", m.termd, m.acked, len(st.vals), logs.String())
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
module mesh-delivery-indexer
|
||||
|
||||
go 1.26.0
|
||||
|
||||
require (
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261019124204-9ade199740e3
|
||||
github.com/nats-io/nats-server/v2 v2.11.17
|
||||
github.com/nats-io/nats.go v1.54.0
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op // indirect
|
||||
github.com/google/go-tpm v0.9.8 // indirect
|
||||
github.com/klauspost/compress v1.20.0 // indirect
|
||||
github.com/minio/highwayhash v1.0.4 // indirect
|
||||
github.com/nats-io/jwt/v2 v2.8.1 // indirect
|
||||
github.com/nats-io/nkeys v0.4.16 // indirect
|
||||
github.com/nats-io/nuid v1.0.1 // indirect
|
||||
golang.org/x/crypto v0.57.0 // indirect
|
||||
golang.org/x/sys v0.48.0 // indirect
|
||||
golang.org/x/time v0.15.0 // indirect
|
||||
)
|
||||
@@ -0,0 +1,27 @@
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261019124204-9ade199740e3 h1:Mu2QiGbL/sZFU5ya4lGUyQOB5OKXPy/KB40XfXTf2EU=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261019124204-9ade199740e3/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
||||
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
|
||||
github.com/google/go-tpm v0.9.8/go.mod h1:h9jEsEECg7gtLis0upRBQU+GhYVH6jMjrFxI8u6bVUY=
|
||||
github.com/klauspost/compress v1.20.0 h1:a3C1ke2ohxFymNlb2HWAHjDeKCI90scRskErZkR0ezA=
|
||||
github.com/klauspost/compress v1.20.0/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI=
|
||||
github.com/minio/highwayhash v1.0.4 h1:asJizugGgchQod2ja9NJlGOWq4s7KsAWr5XUc9Clgl4=
|
||||
github.com/minio/highwayhash v1.0.4/go.mod h1:GGYsuwP/fPD6Y9hMiXuapVvlIUEhFhMTh0rxU3ik1LQ=
|
||||
github.com/nats-io/jwt/v2 v2.8.1 h1:V0xpGuD/N8Mi+fQNDynXohVvp7ZztevW5io8CUWlPmU=
|
||||
github.com/nats-io/jwt/v2 v2.8.1/go.mod h1:nWnOEEiVMiKHQpnAy4eXlizVEtSfzacZ1Q43LIRavZg=
|
||||
github.com/nats-io/nats-server/v2 v2.11.17 h1:GKEghcFK6A+aFx11Yf1LjgLC3txAwvyhnYzhBIQZA8I=
|
||||
github.com/nats-io/nats-server/v2 v2.11.17/go.mod h1:B1sFVz4StNosQ903ak4N1G01Fl/9f8e06mXpFIE2K24=
|
||||
github.com/nats-io/nats.go v1.54.0 h1:vsXoOxjHp/GmPUN+EcI7uOf/uB+iAP+kEsAFNQN0yzA=
|
||||
github.com/nats-io/nats.go v1.54.0/go.mod h1:y+DZoD1oBOYfZTU681eTUiUjI0vbqYGixNVFHcjHJ0k=
|
||||
github.com/nats-io/nkeys v0.4.16 h1:rd5oAuLOb8mnAycB0xleuEBNS1pVVnN0fv/FF34Eypg=
|
||||
github.com/nats-io/nkeys v0.4.16/go.mod h1:llLgWoI0o4z/Q57q2R1kHfmocyhGV6VG/U18Glg1Afs=
|
||||
github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw=
|
||||
github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c=
|
||||
golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M=
|
||||
golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA=
|
||||
golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo=
|
||||
golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
|
||||
golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U=
|
||||
golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno=
|
||||
@@ -0,0 +1,136 @@
|
||||
// Package index keeps the candidate groups (novox/hq ADR 0437 §5) the forge bridge's pull messages make.
|
||||
package index
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"path"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// Builds holds, from the controller's builds bucket (mesh-controller_builds), which module a directory of a
|
||||
// repository is: the module of the newest build record naming that repository and path (BuildRecord.Path). Safe for use
|
||||
// from a watch and from the message handler at once.
|
||||
type Builds struct {
|
||||
mu sync.Mutex
|
||||
records map[string]delivery.BuildRecord // by bucket key
|
||||
byDir map[string]delivery.BuildRecord // by repository and path; nil when stale
|
||||
mods map[string]*moduleBuilds // by module; nil when stale
|
||||
}
|
||||
|
||||
// moduleBuilds is what the builds say of one module: its newest record of any kind, and its newest successful trunk
|
||||
// build that recorded its inputs.
|
||||
type moduleBuilds struct {
|
||||
any delivery.BuildRecord
|
||||
trunk *delivery.BuildRecord
|
||||
}
|
||||
|
||||
// NewBuilds is an empty Builds.
|
||||
func NewBuilds() *Builds { return &Builds{records: map[string]delivery.BuildRecord{}} }
|
||||
|
||||
// Put takes a key of the builds bucket and its value. Only a build record of a module (key `<module>.build.<id>`,
|
||||
// delivery.BuildKey) counts; the newest-pointer keys and requests that knew no module are not records of one.
|
||||
func (b *Builds) Put(key string, value []byte) {
|
||||
if !strings.Contains(key, ".build.") {
|
||||
return
|
||||
}
|
||||
var r delivery.BuildRecord
|
||||
if err := json.Unmarshal(value, &r); err != nil || r.Module == "" || r.Repository == "" {
|
||||
return
|
||||
}
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
b.records[key] = r
|
||||
b.byDir, b.mods = nil, nil
|
||||
}
|
||||
|
||||
// Delete takes a key out.
|
||||
func (b *Builds) Delete(key string) {
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
if _, ok := b.records[key]; ok {
|
||||
delete(b.records, key)
|
||||
b.byDir, b.mods = nil, nil
|
||||
}
|
||||
}
|
||||
|
||||
func dirKey(repository, dir string) string { return repository + "\x00" + strings.Trim(dir, "/") }
|
||||
|
||||
// Module is the module a build record of the repository names the directory as its path, if one does.
|
||||
func (b *Builds) Module(repository, dir string) (string, bool) {
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
if b.byDir == nil {
|
||||
b.byDir = map[string]delivery.BuildRecord{}
|
||||
for _, r := range b.records {
|
||||
k := dirKey(r.Repository, r.Path)
|
||||
if old, ok := b.byDir[k]; !ok || r.At.After(old.At) || (r.At.Equal(old.At) && r.Module < old.Module) {
|
||||
b.byDir[k] = r
|
||||
}
|
||||
}
|
||||
}
|
||||
r, ok := b.byDir[dirKey(repository, dir)]
|
||||
return r.Module, ok
|
||||
}
|
||||
|
||||
// ModuleOf is the module of a changed directory: the one a build record of the repository names it the path of; a
|
||||
// directory no build record names is the directory's last element (the catalogue's rule for nearly every manifest).
|
||||
func (b *Builds) ModuleOf(repository, dir string) string {
|
||||
if m, ok := b.Module(repository, dir); ok {
|
||||
return m
|
||||
}
|
||||
return path.Base(strings.Trim(dir, "/"))
|
||||
}
|
||||
|
||||
// Known is a module the builds know of in a repository: its directory there and its newest successful trunk build (ref
|
||||
// main, state built, with source inputs) if any. The newest key of a module is any state and ref, so it is not what
|
||||
// finds this (mesh-sdk go/delivery/buildrecord.go NewestKey); the records are.
|
||||
type Known struct {
|
||||
Module string
|
||||
Repository string
|
||||
Path string
|
||||
Trunk *delivery.BuildRecord
|
||||
}
|
||||
|
||||
func newer(a, b delivery.BuildRecord) bool {
|
||||
return a.At.After(b.At) || (a.At.Equal(b.At) && a.ID > b.ID)
|
||||
}
|
||||
|
||||
// Modules are the modules built from a repository, by name. A module whose newest successful trunk build is of the
|
||||
// repository belongs to it with that record's path; a module with no such build belongs to the repository of its newest
|
||||
// record of any kind.
|
||||
func (b *Builds) Modules(repository string) []Known {
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
if b.mods == nil {
|
||||
b.mods = map[string]*moduleBuilds{}
|
||||
for _, r := range b.records {
|
||||
m := b.mods[r.Module]
|
||||
if m == nil {
|
||||
m = &moduleBuilds{any: r}
|
||||
b.mods[r.Module] = m
|
||||
} else if newer(r, m.any) {
|
||||
m.any = r
|
||||
}
|
||||
if r.Ref == "main" && r.State == "built" && r.SourceInputs != nil && (m.trunk == nil || newer(r, *m.trunk)) {
|
||||
c := r
|
||||
m.trunk = &c
|
||||
}
|
||||
}
|
||||
}
|
||||
var out []Known
|
||||
for name, m := range b.mods {
|
||||
k := Known{Module: name, Repository: m.any.Repository, Path: strings.Trim(m.any.Path, "/"), Trunk: m.trunk}
|
||||
if m.trunk != nil {
|
||||
k.Repository, k.Path = m.trunk.Repository, strings.Trim(m.trunk.Path, "/")
|
||||
}
|
||||
if k.Repository == repository {
|
||||
out = append(out, k)
|
||||
}
|
||||
}
|
||||
sort.Slice(out, func(i, j int) bool { return out[i].Module < out[j].Module })
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,175 @@
|
||||
package index
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"slices"
|
||||
"strings"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// ErrConflict is what a Store says when a write found the key changed or created since it was read.
|
||||
var ErrConflict = errors.New("the key changed since it was read")
|
||||
|
||||
// ErrUnreadable wraps what says a message cannot be indexed, ever: it is never tried again.
|
||||
var ErrUnreadable = errors.New("unreadable message")
|
||||
|
||||
// Store is the candidates bucket: a compare-and-set key-value store.
|
||||
type Store interface {
|
||||
// Get is the key's value and revision; found is false when there is none.
|
||||
Get(ctx context.Context, key string) (value []byte, revision uint64, found bool, err error)
|
||||
// Create writes a key that must not exist (ErrConflict when it does).
|
||||
Create(ctx context.Context, key string, value []byte) error
|
||||
// Update writes the key if its revision is still revision (ErrConflict when not).
|
||||
Update(ctx context.Context, key string, value []byte, revision uint64) error
|
||||
// Delete removes the key if its revision is still revision (ErrConflict when not).
|
||||
Delete(ctx context.Context, key string, revision uint64) error
|
||||
}
|
||||
|
||||
// trunkLike are the branches that are no group: a pull request from one is nobody's delivery group (the rule of
|
||||
// mesh-catalog modules/mesh-delivery/cmd/mesh-delivery/holder.go:321).
|
||||
var trunkLike = map[string]bool{"main": true, "master": true, "trunk": true, "": true}
|
||||
|
||||
// writeTries bounds the reads again after a conflict; a message that still conflicts is left to the server to deliver again.
|
||||
const writeTries = 8
|
||||
|
||||
// Indexer writes candidates.<group> from the forge bridge's messages.
|
||||
type Indexer struct {
|
||||
Store Store // candidates
|
||||
// Modules is the modules bucket; nil writes no fingerprints.
|
||||
Modules Store
|
||||
Builds *Builds
|
||||
Logf func(format string, args ...any)
|
||||
}
|
||||
|
||||
func (ix *Indexer) logf(format string, args ...any) {
|
||||
if ix.Logf != nil {
|
||||
ix.Logf(format, args...)
|
||||
}
|
||||
}
|
||||
|
||||
// Apply takes one message of the bridge's stream FORGE into the group of its branch. A pull message adds or moves the
|
||||
// pull request, a closed one or a merged message takes it out; a group left with none is deleted. A change is written
|
||||
// by compare-and-set on the revision read, and read again on a conflict. A lone pull request is a group of one. A
|
||||
// merged message also writes the fingerprint of each module of its repository (modules.go).
|
||||
func (ix *Indexer) Apply(ctx context.Context, f delivery.Forge) error {
|
||||
if f.Kind != delivery.ForgePull && f.Kind != delivery.ForgeMerged {
|
||||
return nil
|
||||
}
|
||||
if f.Kind == delivery.ForgeMerged && ix.Modules != nil {
|
||||
if err := ix.applyMerged(ctx, f); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if trunkLike[f.Branch] {
|
||||
ix.logf("%s#%d: its branch %q is no delivery group, so no candidate", f.Repository, f.Number, f.Branch)
|
||||
return nil
|
||||
}
|
||||
key, err := delivery.CandidatesKey(f.Branch)
|
||||
if err != nil {
|
||||
return fmt.Errorf("%w: %s#%d: %v", ErrUnreadable, f.Repository, f.Number, err)
|
||||
}
|
||||
return write(ctx, ix.Store, key, func(value []byte, found bool) ([]byte, bool, error) {
|
||||
var cur *delivery.Candidate
|
||||
if found {
|
||||
c, err := delivery.DecodeCandidate(value)
|
||||
if err != nil {
|
||||
return nil, false, fmt.Errorf("candidates.%s holds what is not a candidate: %w", key, err)
|
||||
}
|
||||
cur = &c
|
||||
}
|
||||
next := ix.next(cur, f)
|
||||
switch {
|
||||
case next == nil && cur == nil, next != nil && cur != nil && delivery.SameCandidate(*cur, *next):
|
||||
return nil, false, nil
|
||||
case next == nil:
|
||||
return nil, true, nil
|
||||
}
|
||||
data, err := delivery.EncodeCandidate(*next)
|
||||
return data, false, err
|
||||
})
|
||||
}
|
||||
|
||||
// write changes one key of a store by compare-and-set: compute is given the value read (found says there is one) and
|
||||
// answers the new value, or del to remove the key, or neither for no change. A conflict reads again, up to writeTries.
|
||||
func write(ctx context.Context, st Store, key string, compute func(value []byte, found bool) (next []byte, del bool, err error)) error {
|
||||
for try := 0; try < writeTries; try++ {
|
||||
value, revision, found, err := st.Get(ctx, key)
|
||||
if err != nil {
|
||||
return fmt.Errorf("reading %s: %w", key, err)
|
||||
}
|
||||
next, del, err := compute(value, found)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
switch {
|
||||
case next == nil && !del:
|
||||
return nil
|
||||
case del:
|
||||
err = st.Delete(ctx, key, revision)
|
||||
case !found:
|
||||
err = st.Create(ctx, key, next)
|
||||
default:
|
||||
err = st.Update(ctx, key, next, revision)
|
||||
}
|
||||
if errors.Is(err, ErrConflict) {
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
return fmt.Errorf("writing %s: %w", key, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
return fmt.Errorf("%s changed under every one of %d tries", key, writeTries)
|
||||
}
|
||||
|
||||
// next is the group after the message; nil when the group is left with no pull request.
|
||||
func (ix *Indexer) next(cur *delivery.Candidate, f delivery.Forge) *delivery.Candidate {
|
||||
var pulls []delivery.CandidatePull
|
||||
var base uint64
|
||||
if cur != nil {
|
||||
pulls, base = slices.Clone(cur.Pulls), cur.Base
|
||||
}
|
||||
at := slices.IndexFunc(pulls, func(p delivery.CandidatePull) bool { return p.Repository == f.Repository && p.Number == f.Number })
|
||||
if f.Kind == delivery.ForgeMerged || f.Action == delivery.PullClosed {
|
||||
if at < 0 {
|
||||
return cur
|
||||
}
|
||||
pulls = slices.Delete(pulls, at, at+1)
|
||||
} else {
|
||||
entry := delivery.CandidatePull{Head: delivery.Head{Repository: f.Repository, Number: f.Number, Head: f.Head}, Modules: []string{}}
|
||||
switch {
|
||||
case f.DirsKnown:
|
||||
for _, dir := range f.Dirs {
|
||||
if m := ix.Builds.ModuleOf(f.Repository, dir); !slices.Contains(entry.Modules, m) {
|
||||
entry.Modules = append(entry.Modules, m)
|
||||
}
|
||||
}
|
||||
case at >= 0 && pulls[at].Head.Head == f.Head:
|
||||
// Its directories are not known now, for the head the group already holds: what it knew stands.
|
||||
entry.Modules = pulls[at].Modules
|
||||
}
|
||||
if at >= 0 {
|
||||
pulls[at] = entry
|
||||
} else {
|
||||
pulls = append(pulls, entry)
|
||||
}
|
||||
}
|
||||
if len(pulls) == 0 {
|
||||
return nil
|
||||
}
|
||||
slices.SortFunc(pulls, func(a, b delivery.CandidatePull) int {
|
||||
if c := strings.Compare(a.Repository, b.Repository); c != 0 {
|
||||
return c
|
||||
}
|
||||
return a.Number - b.Number
|
||||
})
|
||||
heads := make([]delivery.Head, len(pulls))
|
||||
for i, p := range pulls {
|
||||
heads[i] = p.Head
|
||||
}
|
||||
hash, _ := delivery.HeadsHash(heads)
|
||||
return &delivery.Candidate{V: delivery.Version, Group: f.Branch, Base: base, HeadsHash: hash, Pulls: pulls}
|
||||
}
|
||||
@@ -0,0 +1,308 @@
|
||||
package index
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
const (
|
||||
h1 = "0123456789abcdef0123456789abcdef01234567"
|
||||
h2 = "89abcdef0123456789abcdef0123456789abcdef"
|
||||
h3 = "fedcba9876543210fedcba9876543210fedcba98"
|
||||
)
|
||||
|
||||
// memStore is the candidates bucket's compare-and-set behaviour in memory: a revision per write, a conflict when the
|
||||
// revision read is no longer the key's. Writes counts the writes that went through; beforeWrite lets a test change the
|
||||
// key under the indexer.
|
||||
type memStore struct {
|
||||
vals map[string][]byte
|
||||
revs map[string]uint64
|
||||
next uint64
|
||||
writes int
|
||||
beforeWrite func()
|
||||
}
|
||||
|
||||
func newMem() *memStore { return &memStore{vals: map[string][]byte{}, revs: map[string]uint64{}} }
|
||||
|
||||
func (m *memStore) Get(_ context.Context, key string) ([]byte, uint64, bool, error) {
|
||||
v, ok := m.vals[key]
|
||||
return v, m.revs[key], ok, nil
|
||||
}
|
||||
func (m *memStore) hook() {
|
||||
if h := m.beforeWrite; h != nil {
|
||||
m.beforeWrite = nil
|
||||
h()
|
||||
}
|
||||
}
|
||||
func (m *memStore) put(key string, value []byte) {
|
||||
m.next++
|
||||
m.vals[key], m.revs[key] = value, m.next
|
||||
}
|
||||
func (m *memStore) Create(_ context.Context, key string, value []byte) error {
|
||||
m.hook()
|
||||
if _, ok := m.vals[key]; ok {
|
||||
return ErrConflict
|
||||
}
|
||||
m.writes++
|
||||
m.put(key, value)
|
||||
return nil
|
||||
}
|
||||
func (m *memStore) Update(_ context.Context, key string, value []byte, rev uint64) error {
|
||||
m.hook()
|
||||
if _, ok := m.vals[key]; !ok || m.revs[key] != rev {
|
||||
return ErrConflict
|
||||
}
|
||||
m.writes++
|
||||
m.put(key, value)
|
||||
return nil
|
||||
}
|
||||
func (m *memStore) Delete(_ context.Context, key string, rev uint64) error {
|
||||
m.hook()
|
||||
if _, ok := m.vals[key]; !ok || m.revs[key] != rev {
|
||||
return ErrConflict
|
||||
}
|
||||
m.writes++
|
||||
delete(m.vals, key)
|
||||
return nil
|
||||
}
|
||||
|
||||
// buildRecord is the controller's record as mesh-sdk go/delivery/testdata/build-record-build-agent.json (captured from
|
||||
// the controller's encoder; copied here, the vendored SDK does not carry testdata) has it, with the repository, module and path changed.
|
||||
func buildRecord(t *testing.T, repo, module, path, at string) []byte {
|
||||
t.Helper()
|
||||
raw := []byte(`{"id":"build-1792395447088936520","state":"built","repository":"novox/mesh-controller","ref":"main","module":"build-agent","commit":"7a74cee83f88e6585a5511c3cc53bdf0f5303d20","path":"modules/build-agent","source-fingerprint":"src1:e846efe73716b78e80860baaac96153de2cb76551325753838ca245d2160a3d7","at":"2026-10-19T09:38:00Z"}`)
|
||||
var m map[string]any
|
||||
if err := json.Unmarshal(raw, &m); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m["repository"], m["module"], m["path"], m["at"] = repo, module, path, at
|
||||
b, _ := json.Marshal(m)
|
||||
return b
|
||||
}
|
||||
|
||||
func pull(repo string, number int, branch, head string, dirs ...string) delivery.Forge {
|
||||
return delivery.Forge{V: delivery.Version, Kind: delivery.ForgePull, Repository: repo, Branch: branch, Number: number, Head: head,
|
||||
Action: delivery.PullUpdated, Dirs: dirs, DirsKnown: dirs != nil}
|
||||
}
|
||||
|
||||
func newIndexer(t *testing.T) (*Indexer, *memStore, *strings.Builder) {
|
||||
t.Helper()
|
||||
st, logs := newMem(), &strings.Builder{}
|
||||
b := NewBuilds()
|
||||
b.Put(delivery.BuildKey("a", "build-1"), buildRecord(t, "novox/mesh-delivery", "a", "modules/a", "2026-10-19T09:00:00Z"))
|
||||
b.Put(delivery.BuildKey("c", "build-2"), buildRecord(t, "novox/mesh-catalog", "c", "modules/c", "2026-10-19T09:00:00Z"))
|
||||
b.Put(delivery.BuildKey("odd", "build-3"), buildRecord(t, "novox/mesh-delivery", "odd-name", "modules/odd", "2026-10-19T09:00:00Z"))
|
||||
return &Indexer{Store: st, Builds: b, Logf: func(f string, a ...any) {
|
||||
logs.WriteString(strings.TrimSpace(strings.ReplaceAll(f, "\n", " ")) + "\n")
|
||||
_ = a
|
||||
}}, st, logs
|
||||
}
|
||||
|
||||
func read(t *testing.T, st *memStore, key string) (delivery.Candidate, bool) {
|
||||
t.Helper()
|
||||
v, ok := st.vals[key]
|
||||
if !ok {
|
||||
return delivery.Candidate{}, false
|
||||
}
|
||||
c, err := delivery.DecodeCandidate(v)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return c, true
|
||||
}
|
||||
|
||||
func TestTwoRepositoriesFormOneGroupByBranch(t *testing.T) { // 1238/AC2 (the grouping; the bus is the integration test)
|
||||
ix, st, _ := newIndexer(t)
|
||||
ctx := context.Background()
|
||||
if err := ix.Apply(ctx, pull("novox/mesh-delivery", 12, "feat/x", h1, "modules/a")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := ix.Apply(ctx, pull("novox/mesh-catalog", 300, "feat/x", h2, "modules/c")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
c, ok := read(t, st, "feat/x")
|
||||
if !ok {
|
||||
t.Fatal("no candidate under feat/x")
|
||||
}
|
||||
hash, _ := delivery.HeadsHash([]delivery.Head{{Repository: "novox/mesh-delivery", Number: 12, Head: h1}, {Repository: "novox/mesh-catalog", Number: 300, Head: h2}})
|
||||
if c.Group != "feat/x" || c.Base != 0 || c.HeadsHash != hash || len(c.Pulls) != 2 {
|
||||
t.Fatalf("candidate = %+v", c)
|
||||
}
|
||||
got := map[string][]string{}
|
||||
for _, p := range c.Pulls {
|
||||
got[p.Repository] = p.Modules
|
||||
}
|
||||
if strings.Join(got["novox/mesh-delivery"], ",") != "a" || strings.Join(got["novox/mesh-catalog"], ",") != "c" {
|
||||
t.Errorf("modules = %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestModuleOfADirectory(t *testing.T) { // 1238/AC3
|
||||
ix, st, _ := newIndexer(t)
|
||||
// modules/odd is named by a build record of the repository (module odd-name); modules/plain is named by none.
|
||||
if err := ix.Apply(context.Background(), pull("novox/mesh-delivery", 12, "feat/x", h1, "modules/odd", "modules/plain", "modules/odd")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
c, _ := read(t, st, "feat/x")
|
||||
if strings.Join(c.Pulls[0].Modules, ",") != "odd-name,plain" {
|
||||
t.Errorf("modules = %v, want the record's module for modules/odd, the last element for modules/plain, no repeats", c.Pulls[0].Modules)
|
||||
}
|
||||
// A record of another repository does not name this repository's directory.
|
||||
if m, ok := ix.Builds.Module("novox/other", "modules/a"); ok {
|
||||
t.Errorf("modules/a of novox/other is %q", m)
|
||||
}
|
||||
// The newest record of a directory wins.
|
||||
ix.Builds.Put(delivery.BuildKey("renamed", "build-9"), buildRecord(t, "novox/mesh-delivery", "renamed", "modules/a", "2026-10-19T10:00:00Z"))
|
||||
if m := ix.Builds.ModuleOf("novox/mesh-delivery", "modules/a"); m != "renamed" {
|
||||
t.Errorf("modules/a = %q, want the newest record's module", m)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDirsNotKnownKeepTheModulesOfTheSameHead(t *testing.T) { // 1238/AC4
|
||||
ix, st, _ := newIndexer(t)
|
||||
ctx := context.Background()
|
||||
_ = ix.Apply(ctx, pull("novox/mesh-delivery", 12, "feat/x", h1, "modules/a"))
|
||||
unknown := pull("novox/mesh-delivery", 12, "feat/x", h1)
|
||||
if err := ix.Apply(ctx, unknown); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
c, _ := read(t, st, "feat/x")
|
||||
if strings.Join(c.Pulls[0].Modules, ",") != "a" {
|
||||
t.Errorf("modules = %v, want a kept", c.Pulls[0].Modules)
|
||||
}
|
||||
// A new head with directories not known is listed with no modules.
|
||||
unknown.Head = h3
|
||||
if err := ix.Apply(ctx, unknown); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
c, _ = read(t, st, "feat/x")
|
||||
if c.Pulls[0].Head.Head != h3 || len(c.Pulls[0].Modules) != 0 {
|
||||
t.Errorf("pull = %+v, want the new head with no modules", c.Pulls[0])
|
||||
}
|
||||
// Known and empty is no modules, not unknown.
|
||||
known := pull("novox/mesh-delivery", 12, "feat/x", h3)
|
||||
known.Dirs, known.DirsKnown = nil, true
|
||||
if err := ix.Apply(ctx, known); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestClosedAndMergedTakeAPullOutAndTheLastDeletesTheKey(t *testing.T) { // 1238/AC5
|
||||
for _, how := range []string{"closed", "merged"} {
|
||||
ix, st, _ := newIndexer(t)
|
||||
ctx := context.Background()
|
||||
_ = ix.Apply(ctx, pull("novox/mesh-delivery", 12, "feat/x", h1, "modules/a"))
|
||||
_ = ix.Apply(ctx, pull("novox/mesh-catalog", 300, "feat/x", h2, "modules/c"))
|
||||
gone := func(repo string, n int, head string) delivery.Forge {
|
||||
if how == "closed" {
|
||||
f := pull(repo, n, "feat/x", head)
|
||||
f.Action = delivery.PullClosed
|
||||
return f
|
||||
}
|
||||
return delivery.Forge{V: delivery.Version, Kind: delivery.ForgeMerged, Repository: repo, Branch: "feat/x", Number: n, Head: head, Merge: h3, Root: "r", Trees: map[string]string{"a": "t"}}
|
||||
}
|
||||
if err := ix.Apply(ctx, gone("novox/mesh-delivery", 12, h1)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
c, ok := read(t, st, "feat/x")
|
||||
if !ok || len(c.Pulls) != 1 || c.Pulls[0].Repository != "novox/mesh-catalog" {
|
||||
t.Fatalf("%s: candidate = %+v (kept %v)", how, c, ok)
|
||||
}
|
||||
hash, _ := delivery.HeadsHash([]delivery.Head{c.Pulls[0].Head})
|
||||
if c.HeadsHash != hash {
|
||||
t.Errorf("%s: heads hash %q is not the remaining pull request's %q", how, c.HeadsHash, hash)
|
||||
}
|
||||
if err := ix.Apply(ctx, gone("novox/mesh-catalog", 300, h2)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, ok := read(t, st, "feat/x"); ok {
|
||||
t.Errorf("%s: the last pull request left and the key is still there", how)
|
||||
}
|
||||
// A pull request the group does not hold changes nothing and writes nothing.
|
||||
writes := st.writes
|
||||
if err := ix.Apply(ctx, gone("novox/mesh-catalog", 300, h2)); err != nil || st.writes != writes {
|
||||
t.Errorf("%s: closing an unheld pull request wrote (%d writes, err %v)", how, st.writes-writes, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestTrunkLikeBranchesMakeNoCandidate(t *testing.T) { // 1238/AC6
|
||||
for _, branch := range []string{"main", "master", "trunk", ""} {
|
||||
ix, st, logs := newIndexer(t)
|
||||
if err := ix.Apply(context.Background(), pull("novox/mesh-delivery", 12, branch, h1, "modules/a")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(st.vals) != 0 || st.writes != 0 {
|
||||
t.Errorf("branch %q: %d keys written", branch, len(st.vals))
|
||||
}
|
||||
if !strings.Contains(logs.String(), "no delivery group") {
|
||||
t.Errorf("branch %q: log %q does not say why", branch, logs.String())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestSameMessageWritesNoNewRevision(t *testing.T) { // 1238/AC7
|
||||
ix, st, _ := newIndexer(t)
|
||||
ctx := context.Background()
|
||||
m := pull("novox/mesh-delivery", 12, "feat/x", h1, "modules/a")
|
||||
_ = ix.Apply(ctx, m)
|
||||
rev, writes := st.revs["feat/x"], st.writes
|
||||
for i := 0; i < 3; i++ {
|
||||
if err := ix.Apply(ctx, m); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if st.revs["feat/x"] != rev || st.writes != writes {
|
||||
t.Errorf("the same message wrote again: revision %d to %d", rev, st.revs["feat/x"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestALonePullRequestIsAGroupOfOne(t *testing.T) { // 1238/AC8
|
||||
ix, st, _ := newIndexer(t)
|
||||
if err := ix.Apply(context.Background(), pull("novox/mesh-delivery", 12, "fix/lone", h1, "modules/a")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
c, ok := read(t, st, "fix/lone")
|
||||
if !ok || c.Group != "fix/lone" || len(c.Pulls) != 1 {
|
||||
t.Errorf("candidate = %+v (%v)", c, ok)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAConflictReadsAgain(t *testing.T) { // 1238 (compare-and-set: Update with the revision read, read again on a conflict)
|
||||
ix, st, _ := newIndexer(t)
|
||||
ctx := context.Background()
|
||||
_ = ix.Apply(ctx, pull("novox/mesh-delivery", 12, "feat/x", h1, "modules/a"))
|
||||
// Between the indexer's read and its write another writer adds #300 to the group.
|
||||
st.beforeWrite = func() {
|
||||
other := delivery.Candidate{V: delivery.Version, Group: "feat/x", Pulls: []delivery.CandidatePull{
|
||||
{Head: delivery.Head{Repository: "novox/mesh-delivery", Number: 12, Head: h1}, Modules: []string{"a"}},
|
||||
{Head: delivery.Head{Repository: "novox/mesh-catalog", Number: 300, Head: h2}, Modules: []string{"c"}}}}
|
||||
other.HeadsHash, _ = delivery.HeadsHash([]delivery.Head{other.Pulls[0].Head, other.Pulls[1].Head})
|
||||
data, _ := delivery.EncodeCandidate(other)
|
||||
st.put("feat/x", data)
|
||||
}
|
||||
if err := ix.Apply(ctx, pull("novox/mesh-delivery", 12, "feat/x", h3, "modules/a")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
c, _ := read(t, st, "feat/x")
|
||||
if len(c.Pulls) != 2 {
|
||||
t.Fatalf("pulls = %+v, want #12 moved and #300 kept (the other writer's)", c.Pulls)
|
||||
}
|
||||
for _, p := range c.Pulls {
|
||||
if p.Number == 12 && p.Head.Head != h3 {
|
||||
t.Errorf("#12 head = %s", p.Head.Head)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnUnusableBranchNameIsUnreadable(t *testing.T) {
|
||||
ix, _, _ := newIndexer(t)
|
||||
err := ix.Apply(context.Background(), pull("novox/mesh-delivery", 12, "feat x?", h1, "modules/a"))
|
||||
if err == nil || !strings.Contains(err.Error(), "unreadable") {
|
||||
t.Errorf("err = %v", err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,74 @@
|
||||
package index
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// The reasons a module has no fingerprint (ADR 0437 §5; issue 1209's decision).
|
||||
const (
|
||||
// NoneNarrow is for a module whose image compiles a Go program from a context: the context is the hash of the build
|
||||
// source the builder handed it, not a git tree.
|
||||
NoneNarrow = "an artifact compiles a Go program from a build source that git trees do not pin: no fingerprint before release 4"
|
||||
// NoneContext is for a context that is a repository's root tree.
|
||||
NoneContext = "an artifact reads the root tree of a repository the indexer does not know the tree of before release 4"
|
||||
// NoneNoBuild is for a module no successful trunk build recorded the inputs of.
|
||||
NoneNoBuild = "no successful trunk build recorded its inputs"
|
||||
)
|
||||
|
||||
// fingerprintOf is the module's fingerprint at the merge, or none and why: its directory's tree from the merged
|
||||
// message and the bases, contexts and toolchains its newest successful trunk build recorded.
|
||||
func fingerprintOf(k Known, trees map[string]string) (fp, none string) {
|
||||
if k.Trunk == nil {
|
||||
return "", NoneNoBuild
|
||||
}
|
||||
in := *k.Trunk.SourceInputs
|
||||
artifacts := make([]string, 0, len(in.Contexts))
|
||||
for a := range in.Contexts {
|
||||
artifacts = append(artifacts, a)
|
||||
}
|
||||
sort.Strings(artifacts)
|
||||
for _, a := range artifacts {
|
||||
if strings.HasPrefix(in.Contexts[a], "narrow:") {
|
||||
return "", NoneNarrow
|
||||
}
|
||||
}
|
||||
if len(artifacts) > 0 {
|
||||
return "", NoneContext
|
||||
}
|
||||
in.Tree = trees[k.Path] // the directory "" is the repository's root
|
||||
return delivery.SourceFingerprint(in)
|
||||
}
|
||||
|
||||
// applyMerged writes, for each module built from the merged repository, its fingerprint at the merge commit (or none and
|
||||
// why). A module of another repository is not written. A value the module already holds is not written again.
|
||||
func (ix *Indexer) applyMerged(ctx context.Context, f delivery.Forge) error {
|
||||
for _, k := range ix.Builds.Modules(f.Repository) {
|
||||
key, err := delivery.ModulesKey(k.Module)
|
||||
if err != nil {
|
||||
ix.logf("%s: the module %q has no usable key, skipped: %v", f.Repository, k.Module, err)
|
||||
continue
|
||||
}
|
||||
fp, none := fingerprintOf(k, f.Trees)
|
||||
next := delivery.Module{Repository: f.Repository, Commit: f.Merge, FP: fp, None: none}
|
||||
err = write(ctx, ix.Modules, key, func(value []byte, found bool) ([]byte, bool, error) {
|
||||
if found {
|
||||
if cur, err := delivery.DecodeModule(value); err == nil && delivery.SameModule(cur, withV(next)) {
|
||||
return nil, false, nil
|
||||
}
|
||||
}
|
||||
data, err := delivery.EncodeModule(next)
|
||||
return data, false, err
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("modules.%s: %w", key, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func withV(m delivery.Module) delivery.Module { m.V = delivery.Version; return m }
|
||||
@@ -0,0 +1,203 @@
|
||||
package index
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
const merge = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
|
||||
|
||||
// record is a build record as the controller writes it (mesh-sdk go/delivery/testdata/build-record-build-agent.json is
|
||||
// the captured shape): SourceInputs under its Go field names, no json keys. inputs is that object's JSON.
|
||||
func record(t *testing.T, id, module, repo, path, ref, state, at, inputs string) (string, []byte) {
|
||||
t.Helper()
|
||||
raw := `{"id":"` + id + `","state":"` + state + `","repository":"` + repo + `","ref":"` + ref + `","module":"` + module + `","commit":"7a74cee83f88e6585a5511c3cc53bdf0f5303d20","path":"` + path + `","at":"` + at + `"`
|
||||
if inputs != "" {
|
||||
raw += `,"source-fingerprint":"src1:ignored","source-inputs":` + inputs
|
||||
}
|
||||
raw += `}`
|
||||
var check delivery.BuildRecord
|
||||
if err := json.Unmarshal([]byte(raw), &check); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return delivery.BuildKey(module, id), []byte(raw)
|
||||
}
|
||||
|
||||
func modulesIndexer(t *testing.T, recs ...[2]any) (*Indexer, *memStore) {
|
||||
t.Helper()
|
||||
mods := newMem()
|
||||
ix := &Indexer{Store: newMem(), Modules: mods, Builds: NewBuilds()}
|
||||
for _, r := range recs {
|
||||
ix.Builds.Put(r[0].(string), r[1].([]byte))
|
||||
}
|
||||
return ix, mods
|
||||
}
|
||||
|
||||
func rec(t *testing.T, id, module, repo, path, ref, state, at, inputs string) [2]any {
|
||||
k, v := record(t, id, module, repo, path, ref, state, at, inputs)
|
||||
return [2]any{k, v}
|
||||
}
|
||||
|
||||
func mergedMsg(repo string, trees map[string]string) delivery.Forge {
|
||||
return delivery.Forge{V: delivery.Version, Kind: delivery.ForgeMerged, Repository: repo, Branch: "feat/x", Number: 12, Head: h1, Merge: merge, Root: "r0", Trees: trees}
|
||||
}
|
||||
|
||||
func moduleOf(t *testing.T, st *memStore, key string) (delivery.Module, bool) {
|
||||
t.Helper()
|
||||
v, ok := st.vals[key]
|
||||
if !ok {
|
||||
return delivery.Module{}, false
|
||||
}
|
||||
m, err := delivery.DecodeModule(v)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return m, true
|
||||
}
|
||||
|
||||
const aInputs = `{"Module":"a","Tree":"oldtree","Bases":["registry.example/base@sha256:aaa"],"Contexts":null,"Toolchains":{"code":"go sha256:ccc 0123456789abcdef"},"Unpinned":""}`
|
||||
|
||||
func TestModuleFingerprintFromTheMergeTreeAndTheRecordedInputs(t *testing.T) { // 1239/AC1
|
||||
ix, mods := modulesIndexer(t, rec(t, "build-1", "a", "novox/mesh-delivery", "modules/a", "main", "built", "2026-10-19T09:00:00Z", aInputs))
|
||||
if err := ix.Apply(context.Background(), mergedMsg("novox/mesh-delivery", map[string]string{"modules/a": "newtree", "modules/b": "other"})); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want, none := delivery.SourceFingerprint(delivery.SourceInputs{Module: "a", Tree: "newtree", Bases: []string{"registry.example/base@sha256:aaa"},
|
||||
Toolchains: map[string]string{"code": "go sha256:ccc 0123456789abcdef"}})
|
||||
if none != "" {
|
||||
t.Fatal(none)
|
||||
}
|
||||
m, ok := moduleOf(t, mods, "a")
|
||||
if !ok || m.FP != want || m.None != "" || m.Repository != "novox/mesh-delivery" || m.Commit != merge || m.Src != 0 {
|
||||
t.Errorf("module = %+v (%v), want fp %s", m, ok, want)
|
||||
}
|
||||
if _, ok := mods.vals["b"]; ok {
|
||||
t.Error("a key was written for a directory no build names")
|
||||
}
|
||||
}
|
||||
|
||||
func TestMergeOfTheRecordedTreeReproducesTheRecordedFingerprint(t *testing.T) { // 1239/AC2
|
||||
// mailu, build-1792391134620131370: mesh-catalog at b4527d9091e972f28cb365623d9280045e7741a3, dir modules/mailu. Its
|
||||
// inputs and recorded fingerprint are the live ones mesh-sdk go/delivery/fingerprint_test.go TestFingerprintLiveBuilds
|
||||
// captured (its tree is `git rev-parse <commit>:modules/mailu`, its bases from the build's [bases] log line).
|
||||
const tree = "cfc6fa8ff5fc1e220132e51ade653b9d3f387b3e"
|
||||
const recorded = "src1:539f2e64000d436c572ef53b40e524012455945f8c5a1c6bb173ee0707e88ef4"
|
||||
inputs := `{"Module":"mailu","Tree":"` + tree + `","Bases":["sha256:25f3cfeaceca14921366af4d1240b56457ef46273bdb508c7b0e8f469f6fd228"],"Contexts":null,` +
|
||||
`"Toolchains":{"code":"typescript sha256:4b81682df5665c1f461d3f720a3a0fd7593f4aa71e4fc272dbfdc4bbd235b489 20125bd590041690"},"Unpinned":""}`
|
||||
ix, mods := modulesIndexer(t, rec(t, "build-1792391134620131370", "mailu", "novox/mesh-catalog", "modules/mailu", "main", "built", "2026-10-19T09:00:00Z", inputs))
|
||||
if err := ix.Apply(context.Background(), mergedMsg("novox/mesh-catalog", map[string]string{"modules/mailu": tree})); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if m, ok := moduleOf(t, mods, "mailu"); !ok || m.FP != recorded {
|
||||
t.Errorf("module = %+v (%v), want fp %s", m, ok, recorded)
|
||||
}
|
||||
}
|
||||
|
||||
func TestANarrowContextHasNoFingerprint(t *testing.T) { // 1239/AC3, 1209/AC1
|
||||
// A Go program's build source as the builder records it (build-agent's record, mesh-sdk go/delivery/testdata):
|
||||
// the context "server" is "narrow:" and a hash.
|
||||
inputs := `{"Module":"build-agent","Tree":"11c0b4f90cae0d6e4ef9cc48cbfdf9f6dbe52bca","Bases":["sha256:d9e853e87e55526f6b2917df91a2115c36dd7c696a35be12163d44e6e2a4b6bc"],"Contexts":{"server":"narrow:d51ca744fd11af59d616c858f76c27b3e58e16e11fbdc0c720eb9d7d19d55fac"},"Toolchains":null,"Unpinned":""}`
|
||||
ix, mods := modulesIndexer(t, rec(t, "build-1", "build-agent", "novox/mesh-controller", "modules/build-agent", "main", "built", "2026-10-19T09:00:00Z", inputs))
|
||||
if err := ix.Apply(context.Background(), mergedMsg("novox/mesh-controller", map[string]string{"modules/build-agent": "newtree"})); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m, ok := moduleOf(t, mods, "build-agent")
|
||||
if !ok || m.FP != "" || !strings.Contains(m.None, "Go program") || !strings.Contains(m.None, "release 4") {
|
||||
t.Errorf("module = %+v (%v), want none naming the Go program's build source and release 4", m, ok)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUnpinnedBuildHasNoFingerprint(t *testing.T) { // 1239/AC4
|
||||
inputs := `{"Module":"a","Tree":"t","Bases":null,"Contexts":null,"Toolchains":null,"Unpinned":"the build resolves packages when built"}`
|
||||
ix, mods := modulesIndexer(t, rec(t, "build-1", "a", "novox/mesh-delivery", "modules/a", "main", "built", "2026-10-19T09:00:00Z", inputs))
|
||||
_ = ix.Apply(context.Background(), mergedMsg("novox/mesh-delivery", map[string]string{"modules/a": "newtree"}))
|
||||
if m, ok := moduleOf(t, mods, "a"); !ok || m.FP != "" || m.None != "the build resolves packages when built" {
|
||||
t.Errorf("module = %+v (%v)", m, ok)
|
||||
}
|
||||
}
|
||||
|
||||
func TestARootTreeContextHasNoFingerprint(t *testing.T) { // 1239/AC5
|
||||
inputs := `{"Module":"a","Tree":"t","Bases":null,"Contexts":{"web":"f00dcafef00dcafef00dcafef00dcafef00dcafe"},"Toolchains":null,"Unpinned":""}`
|
||||
ix, mods := modulesIndexer(t, rec(t, "build-1", "a", "novox/mesh-delivery", "modules/a", "main", "built", "2026-10-19T09:00:00Z", inputs))
|
||||
_ = ix.Apply(context.Background(), mergedMsg("novox/mesh-delivery", map[string]string{"modules/a": "newtree"}))
|
||||
m, ok := moduleOf(t, mods, "a")
|
||||
if !ok || m.FP != "" || !strings.Contains(m.None, "root tree") || !strings.Contains(m.None, "release 4") {
|
||||
t.Errorf("module = %+v (%v)", m, ok)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDirectoryGoneFromTheMergeHasNoTree(t *testing.T) { // 1239/AC6
|
||||
ix, mods := modulesIndexer(t, rec(t, "build-1", "a", "novox/mesh-delivery", "modules/a", "main", "built", "2026-10-19T09:00:00Z", aInputs))
|
||||
_ = ix.Apply(context.Background(), mergedMsg("novox/mesh-delivery", map[string]string{"modules/other": "t"}))
|
||||
if m, ok := moduleOf(t, mods, "a"); !ok || m.FP != "" || m.None != "no tree" {
|
||||
t.Errorf("module = %+v (%v)", m, ok)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAModuleWithNoSuccessfulTrunkBuildHasNone(t *testing.T) { // 1239/AC7
|
||||
ix, mods := modulesIndexer(t,
|
||||
rec(t, "build-1", "failed-only", "novox/mesh-delivery", "modules/f", "main", "failed", "2026-10-19T09:00:00Z", aInputs),
|
||||
rec(t, "build-2", "branch-only", "novox/mesh-delivery", "modules/g", "feat/x", "built", "2026-10-19T09:00:00Z", aInputs),
|
||||
rec(t, "build-3", "no-inputs", "novox/mesh-delivery", "modules/h", "main", "built", "2026-10-19T09:00:00Z", ""),
|
||||
)
|
||||
_ = ix.Apply(context.Background(), mergedMsg("novox/mesh-delivery", map[string]string{"modules/f": "t", "modules/g": "t", "modules/h": "t"}))
|
||||
for _, name := range []string{"failed-only", "branch-only", "no-inputs"} {
|
||||
if m, ok := moduleOf(t, mods, name); !ok || m.FP != "" || !strings.Contains(m.None, "no successful trunk build") {
|
||||
t.Errorf("%s: module = %+v (%v)", name, m, ok)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheNewestSuccessfulTrunkBuildCounts(t *testing.T) { // 1239 (the newest key is only a hint)
|
||||
older := strings.Replace(aInputs, `"Bases":["registry.example/base@sha256:aaa"]`, `"Bases":["registry.example/base@sha256:old"]`, 1)
|
||||
ix, mods := modulesIndexer(t,
|
||||
rec(t, "build-1", "a", "novox/mesh-delivery", "modules/a", "main", "built", "2026-10-19T09:00:00Z", older),
|
||||
rec(t, "build-2", "a", "novox/mesh-delivery", "modules/a", "main", "built", "2026-10-19T10:00:00Z", aInputs),
|
||||
rec(t, "build-3", "a", "novox/mesh-delivery", "modules/a", "main", "failed", "2026-10-19T11:00:00Z", older),
|
||||
rec(t, "build-4", "a", "novox/mesh-delivery", "modules/a", "feat/x", "built", "2026-10-19T12:00:00Z", older),
|
||||
)
|
||||
_ = ix.Apply(context.Background(), mergedMsg("novox/mesh-delivery", map[string]string{"modules/a": "newtree"}))
|
||||
want, _ := delivery.SourceFingerprint(delivery.SourceInputs{Module: "a", Tree: "newtree", Bases: []string{"registry.example/base@sha256:aaa"},
|
||||
Toolchains: map[string]string{"code": "go sha256:ccc 0123456789abcdef"}})
|
||||
if m, _ := moduleOf(t, mods, "a"); m.FP != want {
|
||||
t.Errorf("fp = %s, want the 10:00 trunk build's %s", m.FP, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestModulesOfOtherRepositoriesAreNotWritten(t *testing.T) { // 1239/AC8
|
||||
ix, mods := modulesIndexer(t,
|
||||
rec(t, "build-1", "a", "novox/mesh-delivery", "modules/a", "main", "built", "2026-10-19T09:00:00Z", aInputs),
|
||||
rec(t, "build-2", "c", "novox/mesh-catalog", "modules/c", "main", "built", "2026-10-19T09:00:00Z", strings.Replace(aInputs, `"Module":"a"`, `"Module":"c"`, 1)))
|
||||
_ = ix.Apply(context.Background(), mergedMsg("novox/mesh-delivery", map[string]string{"modules/a": "t", "modules/c": "t"}))
|
||||
if _, ok := mods.vals["c"]; ok {
|
||||
t.Error("the key of a module of another repository was written")
|
||||
}
|
||||
if _, ok := mods.vals["a"]; !ok {
|
||||
t.Error("the key of the merged repository's module was not written")
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnUnchangedModuleWritesNothing(t *testing.T) { // 1239 (unchanged value writes nothing)
|
||||
ix, mods := modulesIndexer(t, rec(t, "build-1", "a", "novox/mesh-delivery", "modules/a", "main", "built", "2026-10-19T09:00:00Z", aInputs))
|
||||
m := mergedMsg("novox/mesh-delivery", map[string]string{"modules/a": "newtree"})
|
||||
_ = ix.Apply(context.Background(), m)
|
||||
rev, writes := mods.revs["a"], mods.writes
|
||||
_ = ix.Apply(context.Background(), m)
|
||||
if mods.revs["a"] != rev || mods.writes != writes {
|
||||
t.Errorf("the same merge wrote again (%d writes, was %d)", mods.writes, writes)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMergedMessageRemovesThePullFromItsGroupToo(t *testing.T) { // 1238/AC5 with 1239 (both effects of one message)
|
||||
ix, _ := modulesIndexer(t, rec(t, "build-1", "a", "novox/mesh-delivery", "modules/a", "main", "built", "2026-10-19T09:00:00Z", aInputs))
|
||||
cands := ix.Store.(*memStore)
|
||||
_ = ix.Apply(context.Background(), pull("novox/mesh-delivery", 12, "feat/x", h1, "modules/a"))
|
||||
_ = ix.Apply(context.Background(), mergedMsg("novox/mesh-delivery", map[string]string{"modules/a": "newtree"}))
|
||||
if _, ok := cands.vals["feat/x"]; ok {
|
||||
t.Error("the merged pull request is still a candidate")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,165 @@
|
||||
package manifest
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// The controller's manifest reader is mesh-controller internal/catalogue (Manifest, manifest.go: Consumes :521,
|
||||
// Invokes :590, Streams :613, Reads :659; StreamDeclaration, streams.go), an internal package of another repository.
|
||||
// These tests decode the manifest strictly into the fields those types name (DisallowUnknownFields) and hold the
|
||||
// rules of issue 1238 (ADR 0437 §4). The merge gate's module check proves the controller takes the manifest.
|
||||
|
||||
type state struct {
|
||||
Name string `json:"name"`
|
||||
History int `json:"history,omitempty"`
|
||||
}
|
||||
|
||||
type artifact struct {
|
||||
Name string `json:"name"`
|
||||
Kind string `json:"kind"`
|
||||
Language string `json:"language"`
|
||||
System string `json:"system"`
|
||||
From string `json:"from"`
|
||||
Binary string `json:"binary"`
|
||||
}
|
||||
|
||||
type manifest struct {
|
||||
Module string `json:"module"`
|
||||
Version string `json:"version"`
|
||||
Slug string `json:"slug"`
|
||||
Consumes []string `json:"consumes"`
|
||||
State []state `json:"state"`
|
||||
Reads []string `json:"reads"`
|
||||
OwnSecrets map[string]string `json:"own-secrets"`
|
||||
SecretsOwner string `json:"secrets-owner"`
|
||||
Resources []map[string]any `json:"resources"`
|
||||
Build struct {
|
||||
Artifacts []artifact `json:"artifacts"`
|
||||
} `json:"build"`
|
||||
}
|
||||
|
||||
func load(t *testing.T) (manifest, map[string]json.RawMessage) {
|
||||
t.Helper()
|
||||
raw, err := os.ReadFile("module.json")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var m manifest
|
||||
dec := json.NewDecoder(bytes.NewReader(raw))
|
||||
dec.DisallowUnknownFields()
|
||||
if err := dec.Decode(&m); err != nil {
|
||||
t.Fatalf("the controller's reader refuses a key this manifest has: %v", err)
|
||||
}
|
||||
var top map[string]json.RawMessage
|
||||
if err := json.Unmarshal(raw, &top); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return m, top
|
||||
}
|
||||
|
||||
func sorted(s []string) []string {
|
||||
s = slices.Clone(s)
|
||||
slices.Sort(s)
|
||||
return s
|
||||
}
|
||||
|
||||
func TestManifestDeclarations(t *testing.T) { // 1238/AC1
|
||||
m, top := load(t)
|
||||
if m.Module != "mesh-delivery-indexer" {
|
||||
t.Errorf("module = %q", m.Module)
|
||||
}
|
||||
if want := []state{{Name: "candidates", History: 10}, {Name: "modules", History: 20}}; !reflect.DeepEqual(m.State, want) {
|
||||
t.Errorf("state = %+v, want %+v", m.State, want)
|
||||
}
|
||||
if want := []string{"mesh-controller.builds", "mesh-delivery-gate.source"}; !reflect.DeepEqual(sorted(m.Reads), want) {
|
||||
t.Errorf("reads = %v, want %v", m.Reads, want)
|
||||
}
|
||||
// "Reads FORGE" is a consumes of the indexer's stream: the broker grants only buckets by reads (mesh-control
|
||||
// internal/broker/state.go:62-93).
|
||||
if want := []string{"mesh-delivery-forge-bridge.forge.**"}; !reflect.DeepEqual(sorted(m.Consumes), want) {
|
||||
t.Errorf("consumes = %v, want %v", m.Consumes, want)
|
||||
}
|
||||
if b, ok := m.OwnSecrets["broker"]; !ok || len(m.OwnSecrets) != 1 || b != "${dir:state}/broker" {
|
||||
t.Errorf("own-secrets = %v, want exactly broker", m.OwnSecrets)
|
||||
}
|
||||
if m.SecretsOwner != "mesh-delivery-indexer" {
|
||||
t.Errorf("secrets-owner = %q", m.SecretsOwner)
|
||||
}
|
||||
// No invokes, no emits, no streams, no bus, no seat, no tool (ADR 0437 §4).
|
||||
for _, k := range []string{"invokes", "streams", "logs", "emits", "claims", "provides", "requires", "uses", "defines-seats", "tools", "seat", "seats", "bus"} {
|
||||
if _, ok := top[k]; ok {
|
||||
t.Errorf("the manifest declares %q: the indexer has none", k)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestManifestProcess(t *testing.T) { // 1238/AC1
|
||||
m, _ := load(t)
|
||||
var procs []map[string]any
|
||||
for _, r := range m.Resources {
|
||||
if r["type"] == "process" {
|
||||
procs = append(procs, r)
|
||||
}
|
||||
if r["type"] == "tool" || r["type"] == "seat" {
|
||||
t.Errorf("resource %v: the indexer serves no tool and holds no seat", r)
|
||||
}
|
||||
}
|
||||
if len(procs) != 1 {
|
||||
t.Fatalf("%d process resources, want one", len(procs))
|
||||
}
|
||||
p := procs[0]
|
||||
if p["artifact"] != "program" || !reflect.DeepEqual(p["run"], []any{"./mesh-delivery-indexer"}) {
|
||||
t.Errorf("process = %v", p)
|
||||
}
|
||||
env, _ := p["env"].(map[string]any)
|
||||
if len(env) != 1 || env["MESH_BROKER_FILE"] != "${dir:state}/broker" {
|
||||
t.Errorf("env = %v", p["env"])
|
||||
}
|
||||
if _, bus := p["bus"]; bus {
|
||||
t.Errorf("the process declares a bus: it runs on its own broker credential")
|
||||
}
|
||||
if len(m.Build.Artifacts) != 1 {
|
||||
t.Fatalf("artifacts = %+v", m.Build.Artifacts)
|
||||
}
|
||||
a := m.Build.Artifacts[0]
|
||||
if st, err := os.Stat(a.From); err != nil || !st.IsDir() || a.Binary != "mesh-delivery-indexer" || !strings.HasSuffix(a.From, a.Binary) {
|
||||
t.Errorf("artifact = %+v", a)
|
||||
}
|
||||
}
|
||||
|
||||
func TestServesNoRequestSubjectAndHoldsNoTicker(t *testing.T) { // 1238/AC9 (ADR 0437 §8)
|
||||
forbidden := []string{"Subscribe(", "QueueSubscribe", "ChanSubscribe", "time.NewTicker", "time.Tick", "time.NewTimer", "time.AfterFunc", "time.After(", "time.Sleep"}
|
||||
var files int
|
||||
for _, root := range []string{"cmd", "internal"} {
|
||||
err := filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error {
|
||||
if err != nil || d.IsDir() || !strings.HasSuffix(path, ".go") || strings.HasSuffix(path, "_test.go") {
|
||||
return err
|
||||
}
|
||||
files++
|
||||
src, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, f := range forbidden {
|
||||
if strings.Contains(string(src), f) {
|
||||
t.Errorf("%s holds %s: the indexer serves no request subject and holds no timer", path, f)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if files < 2 {
|
||||
t.Fatalf("read %d source files, want the program's", files)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,65 @@
|
||||
{
|
||||
"module": "mesh-delivery-indexer",
|
||||
"version": "1",
|
||||
"slug": "indexer",
|
||||
"consumes": [
|
||||
"mesh-delivery-forge-bridge.forge.**"
|
||||
],
|
||||
"state": [
|
||||
{ "name": "candidates", "history": 10 },
|
||||
{ "name": "modules", "history": 20 }
|
||||
],
|
||||
"reads": [
|
||||
"mesh-controller.builds",
|
||||
"mesh-delivery-gate.source"
|
||||
],
|
||||
"own-secrets": {
|
||||
"broker": "${dir:state}/broker"
|
||||
},
|
||||
"secrets-owner": "mesh-delivery-indexer",
|
||||
"resources": [
|
||||
{
|
||||
"id": "account",
|
||||
"type": "user",
|
||||
"name": "mesh-delivery-indexer",
|
||||
"shell": "/usr/bin/nologin",
|
||||
"home": "/var/lib/mesh-delivery-indexer"
|
||||
},
|
||||
{
|
||||
"id": "state",
|
||||
"type": "directory",
|
||||
"mode": "0700",
|
||||
"owner": "mesh-delivery-indexer",
|
||||
"place": ".",
|
||||
"unknown": "program"
|
||||
},
|
||||
{
|
||||
"id": "indexer",
|
||||
"type": "process",
|
||||
"name": "mesh-delivery-indexer",
|
||||
"artifact": "program",
|
||||
"run": [
|
||||
"./mesh-delivery-indexer"
|
||||
],
|
||||
"env": {
|
||||
"MESH_BROKER_FILE": "${dir:state}/broker"
|
||||
},
|
||||
"user": "mesh-delivery-indexer",
|
||||
"health": {
|
||||
"kind": "unit"
|
||||
}
|
||||
}
|
||||
],
|
||||
"build": {
|
||||
"artifacts": [
|
||||
{
|
||||
"name": "program",
|
||||
"kind": "bundle",
|
||||
"language": "go",
|
||||
"system": "arch",
|
||||
"from": "cmd/mesh-delivery-indexer",
|
||||
"binary": "mesh-delivery-indexer"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,123 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats-server/v2/server"
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
)
|
||||
|
||||
// busConf is the shape of the composed accounts file (mesh-controller internal/broker): one account with JetStream, the
|
||||
// controller's user and the teller's own principal <node>.<module>.
|
||||
const busConf = `
|
||||
listen: 127.0.0.1:-1
|
||||
jetstream { store_dir: %q }
|
||||
accounts {
|
||||
MESH {
|
||||
jetstream: enabled
|
||||
users = [
|
||||
{ user: "ctl", password: "pw" }
|
||||
{ user: "n1.mesh-delivery-teller", password: "pw" }
|
||||
]
|
||||
}
|
||||
}
|
||||
`
|
||||
|
||||
type syncBuffer struct {
|
||||
mu sync.Mutex
|
||||
b strings.Builder
|
||||
}
|
||||
|
||||
func (s *syncBuffer) Write(p []byte) (int, error) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
return s.b.Write(p)
|
||||
}
|
||||
func (s *syncBuffer) String() string { s.mu.Lock(); defer s.mu.Unlock(); return s.b.String() }
|
||||
|
||||
// TestTellerWaitsForTheGate starts the program the way the node does (its entry, run, with MESH_BROKER_FILE) against an
|
||||
// embedded bus where mesh-delivery-gate_asks does not exist: it keeps running until stopped, says once that it waits for
|
||||
// the gate's asks, and writes and publishes nothing (the revised 1240/AC2: the watch is release 3d's).
|
||||
func TestTellerWaitsForTheGate(t *testing.T) { // 1240/AC2
|
||||
dir, err := os.MkdirTemp(os.Getenv("TMPDIR"), "t")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { os.RemoveAll(dir) })
|
||||
conf := filepath.Join(dir, "bus.conf")
|
||||
if err := os.WriteFile(conf, []byte(fmt.Sprintf(busConf, filepath.Join(dir, "js"))), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
opts, err := server.ProcessConfigFile(conf)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
opts.NoLog, opts.NoSigs = true, true
|
||||
srv, err := server.NewServer(opts)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
go srv.Start()
|
||||
if !srv.ReadyForConnections(10 * time.Second) {
|
||||
t.Fatal("the bus did not come up")
|
||||
}
|
||||
t.Cleanup(srv.Shutdown)
|
||||
|
||||
ctl, err := nats.Connect(srv.ClientURL(), nats.UserInfo("ctl", "pw"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(ctl.Close)
|
||||
js, err := jetstream.New(ctl)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
|
||||
broker := filepath.Join(t.TempDir(), "broker")
|
||||
cred, _ := json.Marshal(credential{URL: srv.ClientURL(), Node: "n1", Module: "mesh-delivery-teller", User: "n1.mesh-delivery-teller", Password: "pw"})
|
||||
if err := os.WriteFile(broker, cred, 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
env := map[string]string{"MESH_BROKER_FILE": broker}
|
||||
logs := &syncBuffer{}
|
||||
rn := startRunner(ctx, func(c context.Context) error { return run(c, func(k string) string { return env[k] }, logs) })
|
||||
t.Cleanup(func() { rn.finish(t, 10*time.Second) })
|
||||
|
||||
for deadline := time.Now().Add(5 * time.Second); time.Now().Before(deadline) && !strings.Contains(logs.String(), "waiting for the gate's asks"); time.Sleep(20 * time.Millisecond) {
|
||||
}
|
||||
if n := strings.Count(logs.String(), "waiting for the gate's asks"); n != 1 {
|
||||
t.Fatalf("the teller said it waits %d times, want once: %q", n, logs.String())
|
||||
}
|
||||
select {
|
||||
case err := <-rn.done:
|
||||
t.Fatalf("the teller stopped: %v", err)
|
||||
case <-time.After(500 * time.Millisecond):
|
||||
}
|
||||
if n := strings.Count(logs.String(), "waiting for the gate's asks"); n != 1 {
|
||||
t.Errorf("the teller said it waits %d times after a while, want once", n)
|
||||
}
|
||||
// It writes and publishes nothing: no stream, no bucket exists.
|
||||
names := js.StreamNames(ctx)
|
||||
for name := range names.Name() {
|
||||
t.Errorf("a stream %s exists: the teller wrote something", name)
|
||||
}
|
||||
if _, err := js.KeyValue(ctx, "mesh-delivery-gate_asks"); err == nil {
|
||||
t.Error("the gate's asks bucket exists")
|
||||
}
|
||||
if err := rn.finish(t, 10*time.Second); err != nil {
|
||||
t.Errorf("the teller stopped with %v, want a clean stop", err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,112 @@
|
||||
// Command mesh-delivery-teller is delivery's teller (novox/hq ADR 0437 §4, issue 1240). It will carry the gate's asks to
|
||||
// the operator and write the answers back (release 3d). In release 3c nothing writes asks, so it only runs: it connects
|
||||
// as itself, says once that it waits for the gate's asks, and writes and publishes nothing. It does not open the asks
|
||||
// bucket, for waiting for an absent bucket needs a timer (ADR 0437 §8 lets the parts hold none); the watch comes with the
|
||||
// gate. It serves no request subject.
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"crypto/tls"
|
||||
"crypto/x509"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"os"
|
||||
"os/signal"
|
||||
"strings"
|
||||
"syscall"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
)
|
||||
|
||||
func main() {
|
||||
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
|
||||
defer stop()
|
||||
if err := run(ctx, os.Getenv, os.Stderr); err != nil {
|
||||
fmt.Fprintln(os.Stderr, "mesh-delivery-teller:", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
// credential is the broker file the controller issues to the module: JSON of the bus address, the certificate
|
||||
// fingerprint to pin, the principal and its password (as the watchman reads it; issue 1242 moves this into the SDK).
|
||||
type credential struct {
|
||||
URL string `json:"url"`
|
||||
Fingerprint string `json:"fingerprint"`
|
||||
Node string `json:"node"`
|
||||
Module string `json:"module"`
|
||||
User string `json:"user"`
|
||||
Password string `json:"password"`
|
||||
}
|
||||
|
||||
func readCredential(path string) (credential, error) {
|
||||
raw, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return credential{}, err
|
||||
}
|
||||
var c credential
|
||||
if err := json.Unmarshal(raw, &c); err != nil {
|
||||
return credential{}, fmt.Errorf("the broker credential is not one: %w", err)
|
||||
}
|
||||
if c.URL == "" || c.User == "" {
|
||||
return credential{}, errors.New("the broker credential names no bus address or user")
|
||||
}
|
||||
return c, nil
|
||||
}
|
||||
|
||||
// pinned accepts exactly the certificate with this SHA-256 and no other.
|
||||
func pinned(want string) *tls.Config {
|
||||
return &tls.Config{
|
||||
InsecureSkipVerify: true, //nolint:gosec // replaced by the pin below, which is stricter
|
||||
MinVersion: tls.VersionTLS12,
|
||||
VerifyPeerCertificate: func(rawCerts [][]byte, _ [][]*x509.Certificate) error {
|
||||
if len(rawCerts) == 0 {
|
||||
return errors.New("the bus presented no certificate")
|
||||
}
|
||||
sum := sha256.Sum256(rawCerts[0])
|
||||
if got := "sha256:" + hex.EncodeToString(sum[:]); got != want {
|
||||
return fmt.Errorf("the bus presented a certificate this mesh does not know (%.23s…)", got)
|
||||
}
|
||||
return nil
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// run is the program: it returns when ctx ends or the bus cannot be used.
|
||||
func run(ctx context.Context, getenv func(string) string, logOut io.Writer) error {
|
||||
logger := log.New(logOut, "mesh-delivery-teller: ", log.LstdFlags)
|
||||
path := strings.TrimSpace(getenv("MESH_BROKER_FILE"))
|
||||
if path == "" {
|
||||
return errors.New("MESH_BROKER_FILE is not set, so there is no credential to be on the bus with")
|
||||
}
|
||||
cred, err := readCredential(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
opts := []nats.Option{
|
||||
nats.Name(cred.Node + "." + cred.Module),
|
||||
nats.UserInfo(cred.User, cred.Password),
|
||||
nats.CustomInboxPrefix("_INBOX." + cred.User),
|
||||
nats.MaxReconnects(-1),
|
||||
nats.RetryOnFailedConnect(true),
|
||||
}
|
||||
if strings.TrimSpace(cred.Fingerprint) != "" {
|
||||
opts = append(opts, nats.Secure(pinned(cred.Fingerprint)))
|
||||
}
|
||||
conn, err := nats.Connect(cred.URL, opts...)
|
||||
if err != nil {
|
||||
return fmt.Errorf("connecting to the bus: %w", err)
|
||||
}
|
||||
defer conn.Close()
|
||||
logger.Print(waiting)
|
||||
<-ctx.Done()
|
||||
return nil
|
||||
}
|
||||
|
||||
// waiting is said once at start.
|
||||
const waiting = "waiting for the gate's asks, which come with the gate in release 3d; nothing is read, written or published until then"
|
||||
@@ -0,0 +1,75 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// runner runs the program's run function in a goroutine and stops it, bounded: a program that does not return when its
|
||||
// context ends fails the test after a deadline with a message, and the result can be taken more than once (the
|
||||
// integration test takes it in its body and again in its cleanup; a plain buffered channel of one value gave the second
|
||||
// taker nothing, and the cleanup waited for ever: the merge check's 10-minute timeout).
|
||||
type runner struct {
|
||||
stop context.CancelFunc
|
||||
done chan error
|
||||
}
|
||||
|
||||
func startRunner(parent context.Context, run func(context.Context) error) *runner {
|
||||
ctx, stop := context.WithCancel(parent)
|
||||
r := &runner{stop: stop, done: make(chan error, 1)}
|
||||
go func() {
|
||||
r.done <- run(ctx)
|
||||
close(r.done) // a second taker reads the closed channel at once
|
||||
}()
|
||||
return r
|
||||
}
|
||||
|
||||
// finish stops the program and waits at most d for it to return. Called again, it returns at once.
|
||||
func (r *runner) finish(t testing.TB, d time.Duration) error {
|
||||
t.Helper()
|
||||
r.stop()
|
||||
select {
|
||||
case err := <-r.done:
|
||||
return err
|
||||
case <-time.After(d):
|
||||
t.Errorf("the program did not return within %s of its context ending", d)
|
||||
return errors.New("did not return")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunnerCanBeFinishedTwice(t *testing.T) { // 1240 (review round 1: body and cleanup both take the result)
|
||||
r := startRunner(context.Background(), func(ctx context.Context) error { <-ctx.Done(); return nil })
|
||||
if err := r.finish(t, 2*time.Second); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
second := make(chan struct{})
|
||||
go func() { r.finish(t, 2*time.Second); close(second) }()
|
||||
select {
|
||||
case <-second:
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("the second finish waits for ever: the result was taken once")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunnerReportsAProgramThatDoesNotReturn(t *testing.T) { // 1240: a hang fails fast with a message
|
||||
block := make(chan struct{})
|
||||
t.Cleanup(func() { close(block) })
|
||||
r := startRunner(context.Background(), func(context.Context) error { <-block; return nil })
|
||||
probe := &recorder{TB: t}
|
||||
start := time.Now()
|
||||
if err := r.finish(probe, 100*time.Millisecond); err == nil || !probe.failed {
|
||||
t.Fatalf("a program that never returns was not reported (err %v)", err)
|
||||
}
|
||||
if time.Since(start) > 5*time.Second {
|
||||
t.Fatal("the report was not prompt")
|
||||
}
|
||||
}
|
||||
|
||||
type recorder struct {
|
||||
testing.TB
|
||||
failed bool
|
||||
}
|
||||
|
||||
func (r *recorder) Errorf(string, ...any) { r.failed = true }
|
||||
@@ -0,0 +1,21 @@
|
||||
module mesh-delivery-teller
|
||||
|
||||
go 1.26.0
|
||||
|
||||
require (
|
||||
github.com/nats-io/nats-server/v2 v2.11.17
|
||||
github.com/nats-io/nats.go v1.54.0
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op // indirect
|
||||
github.com/google/go-tpm v0.9.8 // indirect
|
||||
github.com/klauspost/compress v1.20.0 // indirect
|
||||
github.com/minio/highwayhash v1.0.4 // indirect
|
||||
github.com/nats-io/jwt/v2 v2.8.1 // indirect
|
||||
github.com/nats-io/nkeys v0.4.16 // indirect
|
||||
github.com/nats-io/nuid v1.0.1 // indirect
|
||||
golang.org/x/crypto v0.57.0 // indirect
|
||||
golang.org/x/sys v0.48.0 // indirect
|
||||
golang.org/x/time v0.15.0 // indirect
|
||||
)
|
||||
@@ -0,0 +1,25 @@
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
||||
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
|
||||
github.com/google/go-tpm v0.9.8/go.mod h1:h9jEsEECg7gtLis0upRBQU+GhYVH6jMjrFxI8u6bVUY=
|
||||
github.com/klauspost/compress v1.20.0 h1:a3C1ke2ohxFymNlb2HWAHjDeKCI90scRskErZkR0ezA=
|
||||
github.com/klauspost/compress v1.20.0/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI=
|
||||
github.com/minio/highwayhash v1.0.4 h1:asJizugGgchQod2ja9NJlGOWq4s7KsAWr5XUc9Clgl4=
|
||||
github.com/minio/highwayhash v1.0.4/go.mod h1:GGYsuwP/fPD6Y9hMiXuapVvlIUEhFhMTh0rxU3ik1LQ=
|
||||
github.com/nats-io/jwt/v2 v2.8.1 h1:V0xpGuD/N8Mi+fQNDynXohVvp7ZztevW5io8CUWlPmU=
|
||||
github.com/nats-io/jwt/v2 v2.8.1/go.mod h1:nWnOEEiVMiKHQpnAy4eXlizVEtSfzacZ1Q43LIRavZg=
|
||||
github.com/nats-io/nats-server/v2 v2.11.17 h1:GKEghcFK6A+aFx11Yf1LjgLC3txAwvyhnYzhBIQZA8I=
|
||||
github.com/nats-io/nats-server/v2 v2.11.17/go.mod h1:B1sFVz4StNosQ903ak4N1G01Fl/9f8e06mXpFIE2K24=
|
||||
github.com/nats-io/nats.go v1.54.0 h1:vsXoOxjHp/GmPUN+EcI7uOf/uB+iAP+kEsAFNQN0yzA=
|
||||
github.com/nats-io/nats.go v1.54.0/go.mod h1:y+DZoD1oBOYfZTU681eTUiUjI0vbqYGixNVFHcjHJ0k=
|
||||
github.com/nats-io/nkeys v0.4.16 h1:rd5oAuLOb8mnAycB0xleuEBNS1pVVnN0fv/FF34Eypg=
|
||||
github.com/nats-io/nkeys v0.4.16/go.mod h1:llLgWoI0o4z/Q57q2R1kHfmocyhGV6VG/U18Glg1Afs=
|
||||
github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw=
|
||||
github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c=
|
||||
golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M=
|
||||
golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA=
|
||||
golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo=
|
||||
golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
|
||||
golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U=
|
||||
golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno=
|
||||
@@ -0,0 +1,166 @@
|
||||
package manifest
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// The controller's manifest reader is mesh-controller internal/catalogue (Manifest, manifest.go: Consumes :521,
|
||||
// Invokes :590, Streams :613, Reads :659; StreamDeclaration, streams.go), an internal package of another repository.
|
||||
// These tests decode the manifest strictly into the fields those types name (DisallowUnknownFields) and hold the
|
||||
// rules of issue 1240 (ADR 0437 §4). The merge gate's module check proves the controller takes the manifest.
|
||||
|
||||
type state struct {
|
||||
Name string `json:"name"`
|
||||
History int `json:"history,omitempty"`
|
||||
}
|
||||
|
||||
type artifact struct {
|
||||
Name string `json:"name"`
|
||||
Kind string `json:"kind"`
|
||||
Language string `json:"language"`
|
||||
System string `json:"system"`
|
||||
From string `json:"from"`
|
||||
Binary string `json:"binary"`
|
||||
}
|
||||
|
||||
type manifest struct {
|
||||
Module string `json:"module"`
|
||||
Version string `json:"version"`
|
||||
Slug string `json:"slug"`
|
||||
Uses []string `json:"uses"`
|
||||
State []state `json:"state"`
|
||||
Reads []string `json:"reads"`
|
||||
OwnSecrets map[string]string `json:"own-secrets"`
|
||||
SecretsOwner string `json:"secrets-owner"`
|
||||
Resources []map[string]any `json:"resources"`
|
||||
Build struct {
|
||||
Artifacts []artifact `json:"artifacts"`
|
||||
} `json:"build"`
|
||||
}
|
||||
|
||||
func load(t *testing.T) (manifest, map[string]json.RawMessage) {
|
||||
t.Helper()
|
||||
raw, err := os.ReadFile("module.json")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var m manifest
|
||||
dec := json.NewDecoder(bytes.NewReader(raw))
|
||||
dec.DisallowUnknownFields()
|
||||
if err := dec.Decode(&m); err != nil {
|
||||
t.Fatalf("the controller's reader refuses a key this manifest has: %v", err)
|
||||
}
|
||||
var top map[string]json.RawMessage
|
||||
if err := json.Unmarshal(raw, &top); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return m, top
|
||||
}
|
||||
|
||||
func sorted(s []string) []string {
|
||||
s = slices.Clone(s)
|
||||
slices.Sort(s)
|
||||
return s
|
||||
}
|
||||
|
||||
func TestManifestDeclarations(t *testing.T) { // 1240/AC1
|
||||
m, top := load(t)
|
||||
if m.Module != "mesh-delivery-teller" {
|
||||
t.Errorf("module = %q", m.Module)
|
||||
}
|
||||
if want := []state{{Name: "answers", History: 5}}; !reflect.DeepEqual(m.State, want) {
|
||||
t.Errorf("state = %+v, want %+v", m.State, want)
|
||||
}
|
||||
if want := []string{"mesh-delivery-gate.asks"}; !reflect.DeepEqual(sorted(m.Reads), want) {
|
||||
t.Errorf("reads = %v, want %v", m.Reads, want)
|
||||
}
|
||||
if want := []string{"operator-channel"}; !reflect.DeepEqual(sorted(m.Uses), want) {
|
||||
t.Errorf("uses = %v, want %v", m.Uses, want)
|
||||
}
|
||||
if b, ok := m.OwnSecrets["broker"]; !ok || len(m.OwnSecrets) != 1 || b != "${dir:state}/broker" {
|
||||
t.Errorf("own-secrets = %v, want exactly broker", m.OwnSecrets)
|
||||
}
|
||||
if m.SecretsOwner != "mesh-delivery-teller" {
|
||||
t.Errorf("secrets-owner = %q", m.SecretsOwner)
|
||||
}
|
||||
// No consumes, invokes, emits, streams, bus, seat or tool (ADR 0437 §4).
|
||||
for _, k := range []string{"consumes", "invokes", "streams", "logs", "emits", "claims", "provides", "requires", "defines-seats", "tools", "seat", "seats", "bus"} {
|
||||
if _, ok := top[k]; ok {
|
||||
t.Errorf("the manifest declares %q: the teller has none", k)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestManifestProcess(t *testing.T) { // 1240/AC1
|
||||
m, _ := load(t)
|
||||
var procs []map[string]any
|
||||
for _, r := range m.Resources {
|
||||
if r["type"] == "process" {
|
||||
procs = append(procs, r)
|
||||
}
|
||||
if r["type"] == "tool" || r["type"] == "seat" {
|
||||
t.Errorf("resource %v: the teller serves no tool and holds no seat", r)
|
||||
}
|
||||
}
|
||||
if len(procs) != 1 {
|
||||
t.Fatalf("%d process resources, want one", len(procs))
|
||||
}
|
||||
p := procs[0]
|
||||
if p["artifact"] != "program" || !reflect.DeepEqual(p["run"], []any{"./mesh-delivery-teller"}) {
|
||||
t.Errorf("process = %v", p)
|
||||
}
|
||||
env, _ := p["env"].(map[string]any)
|
||||
if len(env) != 1 || env["MESH_BROKER_FILE"] != "${dir:state}/broker" {
|
||||
t.Errorf("env = %v", p["env"])
|
||||
}
|
||||
if _, bus := p["bus"]; bus {
|
||||
t.Errorf("the process declares a bus: it runs on its own broker credential")
|
||||
}
|
||||
if len(m.Build.Artifacts) != 1 {
|
||||
t.Fatalf("artifacts = %+v", m.Build.Artifacts)
|
||||
}
|
||||
a := m.Build.Artifacts[0]
|
||||
if st, err := os.Stat(a.From); err != nil || !st.IsDir() || a.Binary != "mesh-delivery-teller" || !strings.HasSuffix(a.From, a.Binary) {
|
||||
t.Errorf("artifact = %+v", a)
|
||||
}
|
||||
}
|
||||
|
||||
func TestServesNoRequestSubjectAndHoldsNoTicker(t *testing.T) { // 1240/AC3 (ADR 0437 §8)
|
||||
forbidden := []string{"Subscribe(", "QueueSubscribe", "ChanSubscribe", "time.NewTicker", "time.Tick", "time.NewTimer", "time.AfterFunc", "time.After(", "time.Sleep"}
|
||||
var files int
|
||||
for _, root := range []string{"cmd", "internal"} {
|
||||
err := filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error {
|
||||
if os.IsNotExist(err) {
|
||||
return fs.SkipDir
|
||||
}
|
||||
if err != nil || d.IsDir() || !strings.HasSuffix(path, ".go") || strings.HasSuffix(path, "_test.go") {
|
||||
return err
|
||||
}
|
||||
files++
|
||||
src, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, f := range forbidden {
|
||||
if strings.Contains(string(src), f) {
|
||||
t.Errorf("%s holds %s: the teller serves no request subject and holds no timer", path, f)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if files < 1 {
|
||||
t.Fatalf("read %d source files, want the program's", files)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,63 @@
|
||||
{
|
||||
"module": "mesh-delivery-teller",
|
||||
"version": "1",
|
||||
"slug": "teller",
|
||||
"state": [
|
||||
{ "name": "answers", "history": 5 }
|
||||
],
|
||||
"reads": [
|
||||
"mesh-delivery-gate.asks"
|
||||
],
|
||||
"uses": [
|
||||
"operator-channel"
|
||||
],
|
||||
"own-secrets": {
|
||||
"broker": "${dir:state}/broker"
|
||||
},
|
||||
"secrets-owner": "mesh-delivery-teller",
|
||||
"resources": [
|
||||
{
|
||||
"id": "account",
|
||||
"type": "user",
|
||||
"name": "mesh-delivery-teller",
|
||||
"shell": "/usr/bin/nologin",
|
||||
"home": "/var/lib/mesh-delivery-teller"
|
||||
},
|
||||
{
|
||||
"id": "state",
|
||||
"type": "directory",
|
||||
"mode": "0700",
|
||||
"owner": "mesh-delivery-teller",
|
||||
"place": ".",
|
||||
"unknown": "program"
|
||||
},
|
||||
{
|
||||
"id": "teller",
|
||||
"type": "process",
|
||||
"name": "mesh-delivery-teller",
|
||||
"artifact": "program",
|
||||
"run": [
|
||||
"./mesh-delivery-teller"
|
||||
],
|
||||
"env": {
|
||||
"MESH_BROKER_FILE": "${dir:state}/broker"
|
||||
},
|
||||
"user": "mesh-delivery-teller",
|
||||
"health": {
|
||||
"kind": "unit"
|
||||
}
|
||||
}
|
||||
],
|
||||
"build": {
|
||||
"artifacts": [
|
||||
{
|
||||
"name": "program",
|
||||
"kind": "bundle",
|
||||
"language": "go",
|
||||
"system": "arch",
|
||||
"from": "cmd/mesh-delivery-teller",
|
||||
"binary": "mesh-delivery-teller"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
@@ -9,6 +9,8 @@ import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
|
||||
"mesh-delivery-view/internal/view"
|
||||
)
|
||||
|
||||
@@ -61,3 +63,49 @@ func TestFailsLoudWhenTheViewIsNotThere(t *testing.T) { // 1134/AC6
|
||||
t.Fatalf("an unknown flag must exit 2, got %d", code)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPrintsTheCandidateGroupsAndTheFingerprintComparison(t *testing.T) { // 1241/AC6
|
||||
m := view.New()
|
||||
const head = "0123456789abcdef0123456789abcdef01234567"
|
||||
cand, err := delivery.EncodeCandidate(delivery.Candidate{Group: "feat/x", Pulls: []delivery.CandidatePull{{Head: delivery.Head{Repository: "novox/mesh-delivery", Number: 12, Head: head}, Modules: []string{"a"}}},
|
||||
HeadsHash: mustHash(t, delivery.Head{Repository: "novox/mesh-delivery", Number: 12, Head: head})})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
mod, err := delivery.EncodeModule(delivery.Module{None: "no tree", Repository: "novox/mesh-delivery", Commit: head})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m.Apply(view.Entry{Bucket: delivery.CandidatesBucket, Key: "feat/x", Value: cand})
|
||||
m.Apply(view.Entry{Bucket: delivery.ModulesBucket, Key: "a", Value: mod})
|
||||
dir, err := os.MkdirTemp(os.Getenv("TMPDIR"), "c")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { os.RemoveAll(dir) })
|
||||
l, err := view.ListenSocket(filepath.Join(dir, "cli.sock"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
go view.ServeSocket(ctx, l, m)
|
||||
t.Cleanup(cancel)
|
||||
var out, errOut bytes.Buffer
|
||||
if code := run(nil, filepath.Join(dir, "cli.sock"), &out, &errOut); code != 0 {
|
||||
t.Fatalf("exit %d: %s", code, errOut.String())
|
||||
}
|
||||
for _, want := range []string{"CANDIDATES", "feat/x", "novox/mesh-delivery#12", head[:12], "none", "no tree"} {
|
||||
if !strings.Contains(out.String(), want) {
|
||||
t.Errorf("the command's table lacks %q:\n%s", want, out.String())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func mustHash(t *testing.T, h delivery.Head) string {
|
||||
t.Helper()
|
||||
s, err := delivery.HeadsHash([]delivery.Head{h})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
@@ -208,3 +208,94 @@ func TestProcessWholeShowsWhatTheWatchmanSays(t *testing.T) { // 1168/AC5
|
||||
stuck(true)
|
||||
wait(func(s view.Snapshot) bool { return len(s.Offline) == 0 && len(s.Stuck) == 0 })
|
||||
}
|
||||
|
||||
// TestProcessWholeShowsTheIndexersBucketsOnceTheyExist starts the program whole before the indexer's buckets exist, then
|
||||
// creates them and puts keys: the view's absent-bucket retry (follow, unchanged) finds them and the snapshot holds them.
|
||||
// The retry is the view's own, every 5 s, so the wait here is 15 s, not 5.
|
||||
func TestProcessWholeShowsTheIndexersBucketsOnceTheyExist(t *testing.T) { // 1241/AC5
|
||||
srv, err := server.NewServer(&server.Options{Host: "127.0.0.1", Port: -1, JetStream: true, StoreDir: t.TempDir(), NoLog: true, NoSigs: true})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
go srv.Start()
|
||||
if !srv.ReadyForConnections(10 * time.Second) {
|
||||
t.Fatal("the bus did not come up")
|
||||
}
|
||||
t.Cleanup(srv.Shutdown)
|
||||
nc, err := nats.Connect(srv.ClientURL())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(nc.Close)
|
||||
js, err := jetstream.New(nc)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
|
||||
defer cancel()
|
||||
for _, b := range []string{delivery.HaveBucket, view.AssignmentsBucket, view.BuildsBucket, delivery.PresenceBucket, delivery.StuckBucket} {
|
||||
if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: b, History: 5}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
dir, err := os.MkdirTemp(os.Getenv("TMPDIR"), "i")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { os.RemoveAll(dir) })
|
||||
broker := filepath.Join(dir, "broker")
|
||||
cred, _ := json.Marshal(credential{URL: srv.ClientURL(), Node: "n", Module: "mesh-delivery-view", User: "mesh-delivery-view", Password: "x"})
|
||||
if err := os.WriteFile(broker, cred, 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
socket := filepath.Join(dir, "run", "cli.sock")
|
||||
env := map[string]string{"MESH_BROKER_FILE": broker, "MESH_DELIVERY_SOCKET": socket, "MESH_DELIVERY_WEB": "127.0.0.1:0"}
|
||||
runCtx, stop := context.WithCancel(ctx)
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- run(runCtx, func(k string) string { return env[k] }, io.Discard) }()
|
||||
t.Cleanup(func() { stop(); <-done })
|
||||
|
||||
// The view is up and the indexer's buckets are not there.
|
||||
for deadline := time.Now().Add(10 * time.Second); time.Now().Before(deadline); time.Sleep(100 * time.Millisecond) {
|
||||
if _, err := view.Ask(socket); err == nil {
|
||||
break
|
||||
}
|
||||
}
|
||||
cands, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: delivery.CandidatesBucket, History: delivery.CandidatesHistory})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
mods, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: delivery.ModulesBucket, History: delivery.ModulesHistory})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
const head = "0123456789abcdef0123456789abcdef01234567"
|
||||
hash, _ := delivery.HeadsHash([]delivery.Head{{Repository: "novox/mesh-delivery", Number: 12, Head: head}})
|
||||
cand, err := delivery.EncodeCandidate(delivery.Candidate{Group: "feat/x", HeadsHash: hash, Pulls: []delivery.CandidatePull{{Head: delivery.Head{Repository: "novox/mesh-delivery", Number: 12, Head: head}, Modules: []string{"a"}}}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
mod, err := delivery.EncodeModule(delivery.Module{None: "no tree", Repository: "novox/mesh-delivery", Commit: head})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := cands.Put(ctx, "feat/x", cand); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := mods.Put(ctx, "a", mod); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var last view.Snapshot
|
||||
for deadline := time.Now().Add(15 * time.Second); time.Now().Before(deadline); time.Sleep(100 * time.Millisecond) {
|
||||
if s, err := view.Ask(socket); err == nil {
|
||||
last = s
|
||||
if len(s.Groups) == 1 && len(s.Fingerprints) == 1 {
|
||||
if s.Groups[0].Group != "feat/x" || s.Fingerprints[0].Module != "a" || s.Fingerprints[0].State != view.FPNone {
|
||||
t.Fatalf("snapshot = %+v", s)
|
||||
}
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
t.Fatalf("the snapshot never held the indexer's keys; last: %+v", last)
|
||||
}
|
||||
|
||||
@@ -136,7 +136,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, bucket := range []string{delivery.HaveBucket, view.AssignmentsBucket, view.BuildsBucket, delivery.PresenceBucket, delivery.StuckBucket} {
|
||||
for _, bucket := range []string{delivery.HaveBucket, view.AssignmentsBucket, view.BuildsBucket, delivery.PresenceBucket, delivery.StuckBucket, delivery.CandidatesBucket, delivery.ModulesBucket} {
|
||||
go follow(ctx, js, bucket, model, logger)
|
||||
}
|
||||
select {
|
||||
|
||||
@@ -3,7 +3,7 @@ module mesh-delivery-view
|
||||
go 1.26.0
|
||||
|
||||
require (
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261018184557-238440d6f8c4
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261019124204-9ade199740e3
|
||||
github.com/nats-io/nats-server/v2 v2.11.17
|
||||
github.com/nats-io/nats.go v1.54.0
|
||||
)
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261018184557-238440d6f8c4 h1:yYDxFRJoH59kiCoEulD+FBwctyf9CrtKIxlEAsVL39Q=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261018184557-238440d6f8c4/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261019124204-9ade199740e3 h1:Mu2QiGbL/sZFU5ya4lGUyQOB5OKXPy/KB40XfXTf2EU=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261019124204-9ade199740e3/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
||||
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
|
||||
|
||||
@@ -0,0 +1,190 @@
|
||||
package view
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
const (
|
||||
hA = "0123456789abcdef0123456789abcdef01234567"
|
||||
hB = "89abcdef0123456789abcdef0123456789abcdef"
|
||||
fF = "src1:1111111111111111111111111111111111111111111111111111111111111111"
|
||||
fG = "src1:2222222222222222222222222222222222222222222222222222222222222222"
|
||||
)
|
||||
|
||||
func candidate(t *testing.T, group string, pulls ...delivery.CandidatePull) Entry {
|
||||
t.Helper()
|
||||
heads := make([]delivery.Head, len(pulls))
|
||||
for i, p := range pulls {
|
||||
heads[i] = p.Head
|
||||
}
|
||||
hash, err := delivery.HeadsHash(heads)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
b, err := delivery.EncodeCandidate(delivery.Candidate{Group: group, HeadsHash: hash, Pulls: pulls})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return Entry{Bucket: delivery.CandidatesBucket, Key: group, Value: b, At: t0}
|
||||
}
|
||||
|
||||
func pullOf(repo string, n int, head string, modules ...string) delivery.CandidatePull {
|
||||
if modules == nil {
|
||||
modules = []string{}
|
||||
}
|
||||
return delivery.CandidatePull{Head: delivery.Head{Repository: repo, Number: n, Head: head}, Modules: modules}
|
||||
}
|
||||
|
||||
func modEntry(t *testing.T, name string, m delivery.Module) Entry {
|
||||
t.Helper()
|
||||
b, err := delivery.EncodeModule(m)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return Entry{Bucket: delivery.ModulesBucket, Key: name, Value: b, At: t0}
|
||||
}
|
||||
|
||||
// trunkBuild is a build record as the controller writes it (mesh-sdk go/delivery/testdata/build-record-build-agent.json is
|
||||
// the captured shape: id, state, repository, ref, module, commit, path, source-fingerprint, source-inputs, at).
|
||||
func trunkBuild(module, id, ref, state, fp, at string) []Entry {
|
||||
rec := `{"id":"` + id + `","state":"` + state + `","repository":"novox/mesh-delivery","ref":"` + ref + `","module":"` + module + `","commit":"7a74cee83f88e6585a5511c3cc53bdf0f5303d20","path":"modules/` + module + `","at":"` + at + `"`
|
||||
if fp != "" {
|
||||
rec += `,"source-fingerprint":"` + fp + `"`
|
||||
}
|
||||
rec += `}`
|
||||
return []Entry{
|
||||
{Bucket: BuildsBucket, Key: module + ".build." + id, Value: []byte(rec), At: t0},
|
||||
{Bucket: BuildsBucket, Key: module + ".newest", Value: []byte(id), At: t0},
|
||||
}
|
||||
}
|
||||
|
||||
func fpRow(t *testing.T, s Snapshot, module string) FingerprintRow {
|
||||
t.Helper()
|
||||
for _, f := range s.Fingerprints {
|
||||
if f.Module == module {
|
||||
return f
|
||||
}
|
||||
}
|
||||
t.Fatalf("no fingerprint row for %s in %+v", module, s.Fingerprints)
|
||||
return FingerprintRow{}
|
||||
}
|
||||
|
||||
func TestSnapshotListsTheCandidateGroups(t *testing.T) { // 1241/AC1
|
||||
m := New()
|
||||
fill(m,
|
||||
candidate(t, "feat/x", pullOf("novox/mesh-delivery", 12, hA, "a", "b"), pullOf("novox/mesh-catalog", 300, hB, "c")),
|
||||
candidate(t, "feat/y", pullOf("novox/mesh-delivery", 13, hB)))
|
||||
g := m.Snapshot().Groups
|
||||
if len(g) != 2 || g[0].Group != "feat/x" || g[1].Group != "feat/y" {
|
||||
t.Fatalf("groups = %+v", g)
|
||||
}
|
||||
// sorted by repository: mesh-catalog before mesh-delivery
|
||||
if len(g[0].Pulls) != 2 || g[0].Pulls[0].Repository != "novox/mesh-catalog" || g[0].Pulls[0].Number != 300 {
|
||||
t.Fatalf("pulls = %+v", g[0].Pulls)
|
||||
}
|
||||
p := g[0].Pulls[1]
|
||||
if p.Repository != "novox/mesh-delivery" || p.Number != 12 || p.Head != hA[:12] || strings.Join(p.Modules, ",") != "a,b" {
|
||||
t.Errorf("pull = %+v, want #12 with head %s and modules a,b", p, hA[:12])
|
||||
}
|
||||
if len(g[1].Pulls) != 1 || len(g[1].Pulls[0].Modules) != 0 {
|
||||
t.Errorf("feat/y = %+v", g[1])
|
||||
}
|
||||
m.Apply(Entry{Bucket: delivery.CandidatesBucket, Key: "feat/x", Deleted: true})
|
||||
if g := m.Snapshot().Groups; len(g) != 1 || g[0].Group != "feat/y" {
|
||||
t.Errorf("after the delete: %+v", g)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFingerprintAgainstTheNewestTrunkBuild(t *testing.T) { // 1241/AC2
|
||||
m := New()
|
||||
fill(m, trunkBuild("a", "build-1", "main", "built", fF, "2026-10-19T10:00:00Z")...)
|
||||
fill(m, trunkBuild("d", "build-2", "main", "built", fF, "2026-10-19T10:00:00Z")...)
|
||||
fill(m,
|
||||
modEntry(t, "a", delivery.Module{FP: fF, Repository: "novox/mesh-delivery", Commit: hA}),
|
||||
modEntry(t, "d", delivery.Module{FP: fG, Repository: "novox/mesh-delivery", Commit: hA}),
|
||||
modEntry(t, "b", delivery.Module{None: "an artifact compiles a Go program from a build source that git trees do not pin: no fingerprint before release 4", Repository: "novox/mesh-delivery", Commit: hA}),
|
||||
modEntry(t, "e", delivery.Module{FP: fG, Repository: "novox/mesh-delivery", Commit: hA}))
|
||||
s := m.Snapshot()
|
||||
if r := fpRow(t, s, "a"); r.State != FPSame || r.Indexer != fF || r.Build != fF {
|
||||
t.Errorf("a = %+v, want same", r)
|
||||
}
|
||||
if r := fpRow(t, s, "d"); r.State != FPDiffers || r.Indexer != fG || r.Build != fF {
|
||||
t.Errorf("d = %+v, want differs with both", r)
|
||||
}
|
||||
if r := fpRow(t, s, "b"); r.State != FPNone || !strings.Contains(r.Reason, "release 4") || r.Indexer != "" {
|
||||
t.Errorf("b = %+v, want none with the reason", r)
|
||||
}
|
||||
if r := fpRow(t, s, "e"); r.State != FPNoBuild || r.Indexer != fG || r.Build != "" {
|
||||
t.Errorf("e = %+v, want no build with the indexer's alone", r)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOnlyTheNewestSuccessfulTrunkBuildCounts(t *testing.T) { // 1241/AC2 (the newest key is only a hint)
|
||||
m := New()
|
||||
fill(m, trunkBuild("a", "build-1", "main", "built", fG, "2026-10-19T09:00:00Z")...)
|
||||
fill(m, trunkBuild("a", "build-2", "main", "built", fF, "2026-10-19T10:00:00Z")...)
|
||||
fill(m, trunkBuild("a", "build-3", "main", "failed", fG, "2026-10-19T11:00:00Z")...) // newer, not built
|
||||
fill(m, trunkBuild("a", "build-4", "feat/x", "built", fG, "2026-10-19T12:00:00Z")...) // newer, not trunk; also the newest key
|
||||
fill(m, modEntry(t, "a", delivery.Module{FP: fF, Repository: "novox/mesh-delivery", Commit: hA}))
|
||||
if r := fpRow(t, m.Snapshot(), "a"); r.State != FPSame || r.Build != fF {
|
||||
t.Errorf("a = %+v, want same with the 10:00 trunk build's fingerprint", r)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildFingerprintIsReadWithTheSDKRecord(t *testing.T) { // 1241/AC3
|
||||
// The record as the controller's encoder wrote it (mesh-sdk go/delivery/testdata/build-record-build-agent.json), whole,
|
||||
// with its source-inputs under their Go field names: the view reads source-fingerprint from it.
|
||||
const rec = `{"id":"build-1792395447088936520","state":"built","repository":"novox/mesh-controller","ref":"main","module":"build-agent","commit":"7a74cee83f88e6585a5511c3cc53bdf0f5303d20","path":"modules/build-agent","manifest-digest":"sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef","source-fingerprint":"src1:e846efe73716b78e80860baaac96153de2cb76551325753838ca245d2160a3d7","source-inputs":{"Module":"build-agent","Tree":"11c0b4f90cae0d6e4ef9cc48cbfdf9f6dbe52bca","Bases":["sha256:d9e853e87e55526f6b2917df91a2115c36dd7c696a35be12163d44e6e2a4b6bc"],"Contexts":{"server":"narrow:d51ca744fd11af59d616c858f76c27b3e58e16e11fbdc0c720eb9d7d19d55fac"},"Toolchains":null,"Unpinned":""},"made":[{"name":"server","kind":"image","reference":"registry.example/build-agent@sha256:d9e853e87e55526f6b2917df91a2115c36dd7c696a35be12163d44e6e2a4b6bc"}],"at":"2026-10-19T09:38:00Z"}`
|
||||
const fp = "src1:e846efe73716b78e80860baaac96153de2cb76551325753838ca245d2160a3d7"
|
||||
m := New()
|
||||
fill(m, Entry{Bucket: BuildsBucket, Key: delivery.BuildKey("build-agent", "build-1792395447088936520"), Value: []byte(rec), At: t0},
|
||||
Entry{Bucket: BuildsBucket, Key: delivery.NewestKey("build-agent"), Value: []byte("build-1792395447088936520"), At: t0},
|
||||
modEntry(t, "build-agent", delivery.Module{FP: fp, Repository: "novox/mesh-controller", Commit: hA}))
|
||||
if r := fpRow(t, m.Snapshot(), "build-agent"); r.State != FPSame || r.Build != fp {
|
||||
t.Errorf("build-agent = %+v, want same with %s", r, fp)
|
||||
}
|
||||
}
|
||||
|
||||
func shown(t *testing.T) *Model {
|
||||
m := New()
|
||||
fill(m, trunkBuild("a", "build-1", "main", "built", fF, "2026-10-19T10:00:00Z")...)
|
||||
fill(m,
|
||||
candidate(t, "feat/x", pullOf("novox/mesh-delivery", 12, hA, "a"), pullOf("novox/mesh-catalog", 300, hB, "c")),
|
||||
modEntry(t, "a", delivery.Module{FP: fG, Repository: "novox/mesh-delivery", Commit: hA}),
|
||||
modEntry(t, "b", delivery.Module{None: "no tree", Repository: "novox/mesh-delivery", Commit: hA}))
|
||||
return m
|
||||
}
|
||||
|
||||
func TestTableAndPagePrintTheGroupsAndTheComparison(t *testing.T) { // 1241/AC6
|
||||
m := shown(t)
|
||||
var out bytes.Buffer
|
||||
Table(&out, m.Snapshot())
|
||||
for _, want := range []string{"CANDIDATES", "feat/x", "novox/mesh-delivery#12", hA[:12], "novox/mesh-catalog#300", "FINGERPRINT", "differs", "indexer " + fG, "build " + fF, "no tree"} {
|
||||
if !strings.Contains(out.String(), want) {
|
||||
t.Errorf("the table lacks %q:\n%s", want, out.String())
|
||||
}
|
||||
}
|
||||
rec := httptest.NewRecorder()
|
||||
Web(m).ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/", nil))
|
||||
for _, want := range []string{"CANDIDATES", "feat/x", "novox/mesh-delivery#12", hA[:12], "FINGERPRINTS", "differs", fG, fF, "no tree"} {
|
||||
if !strings.Contains(rec.Body.String(), want) {
|
||||
t.Errorf("the page lacks %q:\n%s", want, rec.Body.String())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestUnreadableIndexerValuesAreNamed(t *testing.T) { // never dropped silently
|
||||
m := New()
|
||||
fill(m, Entry{Bucket: delivery.CandidatesBucket, Key: "feat/x", Value: []byte("{"), At: t0},
|
||||
Entry{Bucket: delivery.ModulesBucket, Key: "a", Value: []byte("{"), At: t0})
|
||||
s := m.Snapshot()
|
||||
if len(s.Unreadable) != 2 || len(s.Groups) != 0 || len(s.Fingerprints) != 0 {
|
||||
t.Errorf("snapshot = %+v", s)
|
||||
}
|
||||
}
|
||||
@@ -12,6 +12,7 @@ import (
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"text/tabwriter"
|
||||
"time"
|
||||
)
|
||||
@@ -118,6 +119,39 @@ func rowText(r Row) []string {
|
||||
return []string{r.Node, r.Module, r.Assigned, r.Have, r.Phase, r.Health, since(r.Since), r.Detail, what}
|
||||
}
|
||||
|
||||
// groupText is one line per pull request of a group: the branch, `<repository>#<number>`, its head, its modules.
|
||||
func groupText(s Snapshot) [][]string {
|
||||
var out [][]string
|
||||
for _, g := range s.Groups {
|
||||
for i, p := range g.Pulls {
|
||||
group := ""
|
||||
if i == 0 {
|
||||
group = g.Group
|
||||
}
|
||||
out = append(out, []string{group, fmt.Sprintf("%s#%d", p.Repository, p.Number), p.Head, strings.Join(p.Modules, ", ")})
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// fingerprintText is one line per module: the indexer's fingerprint beside the build's, or the reason it has none.
|
||||
func fingerprintText(s Snapshot) [][]string {
|
||||
var out [][]string
|
||||
for _, f := range s.Fingerprints {
|
||||
what := f.Indexer
|
||||
switch f.State {
|
||||
case FPDiffers:
|
||||
what = "indexer " + f.Indexer + "; build " + f.Build
|
||||
case FPNone:
|
||||
what = f.Reason
|
||||
case FPNoBuild:
|
||||
what = "indexer " + f.Indexer + "; no successful trunk build recorded a fingerprint"
|
||||
}
|
||||
out = append(out, []string{f.Module, f.State, what})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
var head = []string{"NODE", "MODULE", "ASSIGNED", "HAVE", "PHASE", "HEALTH", "SINCE", "DETAIL", "WAITS"}
|
||||
|
||||
// Table prints the snapshot as a table.
|
||||
@@ -143,6 +177,20 @@ func Table(w io.Writer, s Snapshot) {
|
||||
row([]string{n.Node, fmt.Sprint(n.Windows), fmt.Sprint(n.Busy), fmt.Sprint(n.Strays), since(n.Since)})
|
||||
}
|
||||
}
|
||||
if g := groupText(s); len(g) > 0 {
|
||||
fmt.Fprintln(tw)
|
||||
row([]string{"CANDIDATES", "PULL REQUEST", "HEAD", "MODULES"})
|
||||
for _, cells := range g {
|
||||
row(cells)
|
||||
}
|
||||
}
|
||||
if f := fingerprintText(s); len(f) > 0 {
|
||||
fmt.Fprintln(tw)
|
||||
row([]string{"MODULE", "FINGERPRINT", "INDEXER AGAINST BUILD"})
|
||||
for _, cells := range f {
|
||||
row(cells)
|
||||
}
|
||||
}
|
||||
if len(s.Offline) > 0 {
|
||||
fmt.Fprintln(tw)
|
||||
row([]string{"OFFLINE", "NAME", "STATE", "SINCE"})
|
||||
@@ -178,6 +226,8 @@ var page = template.Must(template.New("p").Parse(`<!doctype html><html><head><me
|
||||
<title>Delivery</title><style>body{font:14px sans-serif;margin:1em}table{border-collapse:collapse}td,th{padding:2px 10px;text-align:left}</style></head>
|
||||
<body><h1>What each node runs</h1><table><tr>{{range .Head}}<th>{{.}}</th>{{end}}</tr>
|
||||
{{range .Rows}}<tr>{{range .}}<td>{{.}}</td>{{end}}</tr>{{end}}</table>
|
||||
{{if .Groups}}<h2>CANDIDATES</h2><table><tr><th>GROUP</th><th>PULL REQUEST</th><th>HEAD</th><th>MODULES</th></tr>{{range .Groups}}<tr>{{range .}}<td>{{.}}</td>{{end}}</tr>{{end}}</table>{{end}}
|
||||
{{if .Fingerprints}}<h2>FINGERPRINTS</h2><table><tr><th>MODULE</th><th>STATE</th><th>INDEXER AGAINST BUILD</th></tr>{{range .Fingerprints}}<tr>{{range .}}<td>{{.}}</td>{{end}}</tr>{{end}}</table>{{end}}
|
||||
{{if .Offline}}<h2>OFFLINE</h2><table>{{range .Offline}}<tr>{{range .}}<td>{{.}}</td>{{end}}</tr>{{end}}</table>{{end}}
|
||||
{{if .Stuck}}<h2>STUCK</h2><table><tr><th>BUCKET</th><th>KEY</th><th>WAITED FOR</th><th>SINCE</th></tr>{{range .Stuck}}<tr>{{range .}}<td>{{.}}</td>{{end}}</tr>{{end}}</table>{{end}}
|
||||
{{if .Unreadable}}<h2>UNREADABLE</h2><table>{{range .Unreadable}}<tr>{{range .}}<td>{{.}}</td>{{end}}</tr>{{end}}</table>{{end}}</body></html>`))
|
||||
@@ -210,6 +260,6 @@ func Web(m *Model) http.Handler {
|
||||
for _, u := range snap.Unreadable {
|
||||
bad = append(bad, []string{u.Bucket, u.Key, u.Error})
|
||||
}
|
||||
page.Execute(w, map[string]any{"Head": head, "Rows": rows, "Offline": off, "Stuck": stuck, "Unreadable": bad})
|
||||
page.Execute(w, map[string]any{"Head": head, "Rows": rows, "Groups": groupText(snap), "Fingerprints": fingerprintText(snap), "Offline": off, "Stuck": stuck, "Unreadable": bad})
|
||||
})
|
||||
}
|
||||
|
||||
@@ -57,6 +57,45 @@ type Snapshot struct {
|
||||
Stuck []StuckRow `json:"stuck"`
|
||||
// Unreadable is every watchman value the view could not read, named by its key: never dropped silently.
|
||||
Unreadable []UnreadableRow `json:"unreadable"`
|
||||
// Groups is every candidate group the indexer holds: the pull requests that share a branch, with the modules each touches.
|
||||
Groups []GroupRow `json:"groups"`
|
||||
// Fingerprints is every module the indexer fingerprinted at a trunk merge, beside its newest successful trunk build's.
|
||||
Fingerprints []FingerprintRow `json:"fingerprints"`
|
||||
}
|
||||
|
||||
// GroupRow is a candidate group (novox/hq ADR 0437 §5): the branch its pull requests share and the pull requests.
|
||||
type GroupRow struct {
|
||||
Group string `json:"group"`
|
||||
Pulls []GroupPull `json:"pulls"`
|
||||
}
|
||||
|
||||
// GroupPull is one pull request of a group, its head shortened to 12 digits, and the modules it touches.
|
||||
type GroupPull struct {
|
||||
Repository string `json:"repository"`
|
||||
Number int `json:"number"`
|
||||
Head string `json:"head"`
|
||||
Modules []string `json:"modules"`
|
||||
}
|
||||
|
||||
// The states of a module's fingerprint against its newest successful trunk build's.
|
||||
const (
|
||||
FPSame = "same"
|
||||
FPDiffers = "differs"
|
||||
FPNone = "none"
|
||||
FPNoBuild = "no build"
|
||||
)
|
||||
|
||||
// FingerprintRow compares the indexer's fingerprint of a module at a trunk merge with the source fingerprint its newest
|
||||
// successful trunk build recorded. State is same, differs (both are given), none (the indexer could not fingerprint it:
|
||||
// Reason says why) or no build (the indexer's alone is given, no successful trunk build recorded one).
|
||||
type FingerprintRow struct {
|
||||
Module string `json:"module"`
|
||||
Repository string `json:"repository,omitempty"`
|
||||
Commit string `json:"commit,omitempty"`
|
||||
State string `json:"state"`
|
||||
Indexer string `json:"indexer,omitempty"`
|
||||
Build string `json:"build,omitempty"`
|
||||
Reason string `json:"reason,omitempty"`
|
||||
}
|
||||
|
||||
// OfflineRow is a node ("node") or a part's instance ("part") that is offline, since the presence entry's time.
|
||||
@@ -87,8 +126,10 @@ type UnreadableRow struct {
|
||||
type Model struct {
|
||||
mu sync.Mutex
|
||||
assigns map[string]assignment
|
||||
builds map[string]build // by <module>.build.<id>
|
||||
newest map[string]string // module -> id
|
||||
builds map[string]buildEntry // by <module>.build.<id>
|
||||
groups map[string]groupEntry // the indexer's candidates, by group
|
||||
modules map[string]moduleEntry // the indexer's modules, by module
|
||||
newest map[string]string // module -> id
|
||||
haves map[string]haveEntry
|
||||
nodes map[string]nodeEntry
|
||||
// the watchman's buckets, by key
|
||||
@@ -136,6 +177,23 @@ type build struct {
|
||||
|
||||
func (b build) done() bool { return b.State == "" && b.Failed == "" }
|
||||
|
||||
// buildEntry is a build as the view reads it: its own fields (state, failure) and the SDK's record (the source
|
||||
// fingerprint, the ref, the time), decoded as the controller writes them.
|
||||
type buildEntry struct {
|
||||
build
|
||||
rec delivery.BuildRecord
|
||||
}
|
||||
|
||||
type groupEntry struct {
|
||||
value delivery.Candidate
|
||||
err string
|
||||
}
|
||||
|
||||
type moduleEntry struct {
|
||||
value delivery.Module
|
||||
err string
|
||||
}
|
||||
|
||||
type haveEntry struct {
|
||||
have delivery.Have
|
||||
err string
|
||||
@@ -150,7 +208,7 @@ type nodeEntry struct {
|
||||
|
||||
// New is an empty model.
|
||||
func New() *Model {
|
||||
return &Model{assigns: map[string]assignment{}, builds: map[string]build{}, newest: map[string]string{},
|
||||
return &Model{assigns: map[string]assignment{}, builds: map[string]buildEntry{}, groups: map[string]groupEntry{}, modules: map[string]moduleEntry{}, newest: map[string]string{},
|
||||
haves: map[string]haveEntry{}, nodes: map[string]nodeEntry{},
|
||||
presence: map[string]presenceEntry{}, stuck: map[string]stuckEntry{}}
|
||||
}
|
||||
@@ -205,6 +263,20 @@ func (m *Model) Apply(e Entry) {
|
||||
return
|
||||
}
|
||||
m.assigns[a.Node+"."+a.Module] = a
|
||||
case delivery.CandidatesBucket:
|
||||
if e.Deleted {
|
||||
delete(m.groups, e.Key)
|
||||
return
|
||||
}
|
||||
c, err := delivery.DecodeCandidate(e.Value)
|
||||
m.groups[e.Key] = groupEntry{value: c, err: errText(err)}
|
||||
case delivery.ModulesBucket:
|
||||
if e.Deleted {
|
||||
delete(m.modules, e.Key)
|
||||
return
|
||||
}
|
||||
mod, err := delivery.DecodeModule(e.Value)
|
||||
m.modules[e.Key] = moduleEntry{value: mod, err: errText(err)}
|
||||
case BuildsBucket:
|
||||
module, rest, ok := strings.Cut(e.Key, ".")
|
||||
if !ok {
|
||||
@@ -226,7 +298,12 @@ func (m *Model) Apply(e Entry) {
|
||||
if json.Unmarshal(e.Value, &b) != nil {
|
||||
return
|
||||
}
|
||||
m.builds[e.Key] = b
|
||||
// The source fingerprint is read with the SDK's record, not with the view's own struct.
|
||||
var rec delivery.BuildRecord
|
||||
if json.Unmarshal(e.Value, &rec) != nil {
|
||||
return
|
||||
}
|
||||
m.builds[e.Key] = buildEntry{build: b, rec: rec}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -327,6 +404,16 @@ func (m *Model) Snapshot() Snapshot {
|
||||
}
|
||||
stuck = append(stuck, StuckRow{Bucket: st.value.Bucket, Key: st.value.Key, Waited: st.value.Waited, Since: st.value.Since})
|
||||
}
|
||||
for key, g := range m.groups {
|
||||
if g.err != "" {
|
||||
unreadable = append(unreadable, UnreadableRow{Bucket: delivery.CandidatesBucket, Key: key, Error: g.err})
|
||||
}
|
||||
}
|
||||
for key, mod := range m.modules {
|
||||
if mod.err != "" {
|
||||
unreadable = append(unreadable, UnreadableRow{Bucket: delivery.ModulesBucket, Key: key, Error: mod.err})
|
||||
}
|
||||
}
|
||||
sort.Slice(offline, func(i, j int) bool {
|
||||
a, b := offline[i], offline[j]
|
||||
return a.Kind+a.Node+a.Part+a.Instance < b.Kind+b.Node+b.Part+b.Instance
|
||||
@@ -337,7 +424,69 @@ func (m *Model) Snapshot() Snapshot {
|
||||
sort.Slice(unreadable, func(i, j int) bool {
|
||||
return unreadable[i].Bucket+"."+unreadable[i].Key < unreadable[j].Bucket+"."+unreadable[j].Key
|
||||
})
|
||||
return Snapshot{Rows: rows, Nodes: nodes, Offline: offline, Stuck: stuck, Unreadable: unreadable}
|
||||
return Snapshot{Rows: rows, Nodes: nodes, Offline: offline, Stuck: stuck, Unreadable: unreadable,
|
||||
Groups: m.groupRows(), Fingerprints: m.fingerprintRows()}
|
||||
}
|
||||
|
||||
// groupRows are the readable candidate groups, ordered by group, their pull requests by repository and number.
|
||||
func (m *Model) groupRows() []GroupRow {
|
||||
out := []GroupRow{}
|
||||
for key, g := range m.groups {
|
||||
if g.err != "" {
|
||||
continue
|
||||
}
|
||||
row := GroupRow{Group: key, Pulls: []GroupPull{}}
|
||||
for _, p := range g.value.Pulls {
|
||||
head := p.Head.Head
|
||||
if len(head) > 12 {
|
||||
head = head[:12]
|
||||
}
|
||||
row.Pulls = append(row.Pulls, GroupPull{Repository: p.Repository, Number: p.Number, Head: head, Modules: append([]string{}, p.Modules...)})
|
||||
}
|
||||
sort.Slice(row.Pulls, func(i, j int) bool {
|
||||
a, b := row.Pulls[i], row.Pulls[j]
|
||||
return a.Repository < b.Repository || (a.Repository == b.Repository && a.Number < b.Number)
|
||||
})
|
||||
out = append(out, row)
|
||||
}
|
||||
sort.Slice(out, func(i, j int) bool { return out[i].Group < out[j].Group })
|
||||
return out
|
||||
}
|
||||
|
||||
// fingerprintRows compare each module the indexer fingerprinted with the source fingerprint of the module's newest
|
||||
// successful trunk build (ref main, state built) that recorded one. The newest key is any state and ref, so it is not
|
||||
// used: the records are.
|
||||
func (m *Model) fingerprintRows() []FingerprintRow {
|
||||
newest := map[string]buildEntry{}
|
||||
for _, b := range m.builds {
|
||||
if b.rec.Module == "" || b.rec.Ref != "main" || b.rec.State != "built" || b.rec.SourceFingerprint == "" {
|
||||
continue
|
||||
}
|
||||
if old, ok := newest[b.rec.Module]; !ok || b.rec.At.After(old.rec.At) || (b.rec.At.Equal(old.rec.At) && b.rec.ID > old.rec.ID) {
|
||||
newest[b.rec.Module] = b
|
||||
}
|
||||
}
|
||||
out := []FingerprintRow{}
|
||||
for name, mod := range m.modules {
|
||||
if mod.err != "" {
|
||||
continue
|
||||
}
|
||||
row := FingerprintRow{Module: name, Repository: mod.value.Repository, Commit: mod.value.Commit}
|
||||
b, hasBuild := newest[name]
|
||||
switch {
|
||||
case mod.value.None != "":
|
||||
row.State, row.Reason = FPNone, mod.value.None
|
||||
case !hasBuild:
|
||||
row.State, row.Indexer = FPNoBuild, mod.value.FP
|
||||
case b.rec.SourceFingerprint == mod.value.FP:
|
||||
row.State, row.Indexer, row.Build = FPSame, mod.value.FP, b.rec.SourceFingerprint
|
||||
default:
|
||||
row.State, row.Indexer, row.Build = FPDiffers, mod.value.FP, b.rec.SourceFingerprint
|
||||
}
|
||||
out = append(out, row)
|
||||
}
|
||||
sort.Slice(out, func(i, j int) bool { return out[i].Module < out[j].Module })
|
||||
return out
|
||||
}
|
||||
|
||||
// count is the number of items of a carried JSON array or object, and 0 for anything else.
|
||||
|
||||
@@ -74,7 +74,7 @@ func TestManifestIsTakenAndReadsOnly(t *testing.T) { // 1134/AC7
|
||||
if m.Module != "mesh-delivery-view" {
|
||||
t.Errorf("module = %q", m.Module)
|
||||
}
|
||||
want := []string{"mesh-controller.assignments", "mesh-controller.builds", "mesh-delivery-watchman.presence", "mesh-delivery-watchman.stuck", "node-engine.have"} // 1168/AC1
|
||||
want := []string{"mesh-controller.assignments", "mesh-controller.builds", "mesh-delivery-indexer.candidates", "mesh-delivery-indexer.modules", "mesh-delivery-watchman.presence", "mesh-delivery-watchman.stuck", "node-engine.have"} // 1168/AC1, 1241/AC4
|
||||
got := slices.Clone(m.Reads)
|
||||
slices.Sort(got)
|
||||
if !reflect.DeepEqual(got, want) {
|
||||
|
||||
@@ -6,6 +6,8 @@
|
||||
"reads": [
|
||||
"mesh-controller.assignments",
|
||||
"mesh-controller.builds",
|
||||
"mesh-delivery-indexer.candidates",
|
||||
"mesh-delivery-indexer.modules",
|
||||
"mesh-delivery-watchman.presence",
|
||||
"mesh-delivery-watchman.stuck",
|
||||
"node-engine.have"
|
||||
|
||||
-378
@@ -1,378 +0,0 @@
|
||||
// Package delivery is the one home of the keys and values the parts of delivery read and write on the bus
|
||||
// (novox/hq ADR 0429 §2: the key layouts another repository reads live in mesh-sdk; research 063 section 01). The
|
||||
// node-engine writes the have keys, the watchman of the repository mesh-delivery writes presence and pulse, the
|
||||
// view reads all of them, and the controller grants them; every one imports this package, and nothing else defines
|
||||
// these names.
|
||||
//
|
||||
// **Buckets** are named `<owner module>_<name>` as the broker names a module's state (provider.BucketOf): the
|
||||
// node-engine's `have`, and the watchman's `presence` and `pulse`.
|
||||
//
|
||||
// **Keys.** A have key is `<node>.<module>`, and a node's own facts are `<node>.node`, so a module may not be named
|
||||
// `node`. A presence key is `<node>` for a node and `part.<part>.<instance>` for a part's instance, so a node may
|
||||
// not be named `part`. A pulse key is `<bucket>.<key>`: the full name of the bucket watched, then that bucket's key,
|
||||
// which may hold dots, so it splits at the first dot. A name is refused when it is empty or holds a dot, a wildcard,
|
||||
// whitespace or a character a bus key refuses, and the error names it.
|
||||
//
|
||||
// **Values** are JSON of under about 500 bytes, each carrying its version in `v`. A reader refuses a value whose
|
||||
// version is missing or one it does not know, with an error that is ErrUnknownVersion (research 063 section 05
|
||||
// question 9 item 7): a writer writes a new version only once every reader's have shows a version that reads it. A
|
||||
// writer writes only when the new value differs from the one before (the Same functions).
|
||||
//
|
||||
// Standard library only.
|
||||
package delivery
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// The buckets of delivery's parts. Each is written by one owner: every node's node-engine writes the have keys of
|
||||
// its own node only, and the watchman writes presence and pulse.
|
||||
const (
|
||||
HaveBucket = "node-engine_have"
|
||||
PresenceBucket = WatchmanModule + "_presence"
|
||||
PulseBucket = WatchmanModule + "_pulse"
|
||||
)
|
||||
|
||||
// HaveHistory and PresenceHistory are the revisions the have and presence buckets keep per key (research 063
|
||||
// section 01).
|
||||
const (
|
||||
HaveHistory = 20
|
||||
PresenceHistory = 20
|
||||
)
|
||||
|
||||
// PulseMoveBound is how long a move may take: a pulse key lives for this time minus the move's age, and the bucket
|
||||
// expires it on its own (the server writes a delete marker), so a move that overruns it is seen to stall.
|
||||
const PulseMoveBound = 120 * time.Second
|
||||
|
||||
// Version is the version of every value this package writes and the only one it reads.
|
||||
const Version = 1
|
||||
|
||||
// ErrUnknownVersion is what a decode returns for a value whose version is missing or not Version. Test it with
|
||||
// errors.Is; the message names the version seen and the one read.
|
||||
var ErrUnknownVersion = errors.New("delivery: unknown value version")
|
||||
|
||||
// Phase is how far a module's change has come on a node.
|
||||
type Phase string
|
||||
|
||||
// The phases of a module's change on a node.
|
||||
const (
|
||||
PhaseBackingUp Phase = "backing-up"
|
||||
PhaseApplying Phase = "applying"
|
||||
PhaseChecking Phase = "checking"
|
||||
PhaseRunning Phase = "running"
|
||||
PhaseFailed Phase = "failed"
|
||||
)
|
||||
|
||||
// Health is what the module's own check says.
|
||||
type Health string
|
||||
|
||||
// The healths a have carries.
|
||||
const (
|
||||
HealthOK Health = "ok"
|
||||
HealthFailed Health = "failed"
|
||||
HealthWaits Health = "waits"
|
||||
HealthUnknown Health = "unknown"
|
||||
)
|
||||
|
||||
var nameOK = regexp.MustCompile(`^[-/_=a-zA-Z0-9]+$`)
|
||||
|
||||
// checkName refuses a name that is empty or holds a dot, a wildcard, whitespace or a character a bus key refuses.
|
||||
func checkName(kind, name string) error {
|
||||
if name == "" {
|
||||
return fmt.Errorf("delivery: the %s name is empty", kind)
|
||||
}
|
||||
if !nameOK.MatchString(name) {
|
||||
return fmt.Errorf("delivery: the %s name %q holds a dot, a wildcard, whitespace or a character a key refuses", kind, name)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// HaveKey is the key, in HaveBucket, of a module's have on a node: `<node>.<module>`. A module named `node` is
|
||||
// refused, since `<node>.node` is the node's own facts.
|
||||
func HaveKey(node, module string) (string, error) {
|
||||
if err := checkName("node", node); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if err := checkName("module", module); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if module == "node" {
|
||||
return "", fmt.Errorf(`delivery: the module name "node" is refused: %s.node is the node's own key`, node)
|
||||
}
|
||||
return node + "." + module, nil
|
||||
}
|
||||
|
||||
// HaveNodeKey is the key, in HaveBucket, of a node's own facts: `<node>.node`.
|
||||
func HaveNodeKey(node string) (string, error) {
|
||||
if err := checkName("node", node); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return node + ".node", nil
|
||||
}
|
||||
|
||||
// ParseHaveKey is the inverse of HaveKey and HaveNodeKey: isNode is true for `<node>.node`, with module empty.
|
||||
func ParseHaveKey(key string) (node, module string, isNode bool, err error) {
|
||||
node, module, ok := strings.Cut(key, ".")
|
||||
if !ok {
|
||||
return "", "", false, fmt.Errorf("delivery: %q is not a have key, <node>.<module>", key)
|
||||
}
|
||||
if err := checkName("node", node); err != nil {
|
||||
return "", "", false, fmt.Errorf("delivery: %q is not a have key: %w", key, err)
|
||||
}
|
||||
if err := checkName("module", module); err != nil {
|
||||
return "", "", false, fmt.Errorf("delivery: %q is not a have key: %w", key, err)
|
||||
}
|
||||
if module == "node" {
|
||||
return node, "", true, nil
|
||||
}
|
||||
return node, module, false, nil
|
||||
}
|
||||
|
||||
// PresenceNodeKey is the key, in PresenceBucket, of a node: `<node>`. A node named `part` is refused.
|
||||
func PresenceNodeKey(node string) (string, error) {
|
||||
if err := checkName("node", node); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if node == "part" {
|
||||
return "", errors.New(`delivery: the node name "part" is refused: part.<part>.<instance> is a part's key`)
|
||||
}
|
||||
return node, nil
|
||||
}
|
||||
|
||||
// PresencePartKey is the key, in PresenceBucket, of a part's instance: `part.<part>.<instance>`.
|
||||
func PresencePartKey(part, instance string) (string, error) {
|
||||
if err := checkName("part", part); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if err := checkName("instance", instance); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return "part." + part + "." + instance, nil
|
||||
}
|
||||
|
||||
// ParsePresenceKey is the inverse of PresenceNodeKey and PresencePartKey: for a node's key part and instance are
|
||||
// empty, for a part's key node is.
|
||||
func ParsePresenceKey(key string) (node, part, instance string, err error) {
|
||||
if !strings.Contains(key, ".") {
|
||||
node, err = PresenceNodeKey(key)
|
||||
if err != nil {
|
||||
return "", "", "", fmt.Errorf("delivery: %q is not a presence key: %w", key, err)
|
||||
}
|
||||
return node, "", "", nil
|
||||
}
|
||||
fields := strings.Split(key, ".")
|
||||
if len(fields) != 3 || fields[0] != "part" {
|
||||
return "", "", "", fmt.Errorf("delivery: %q is not a presence key, <node> or part.<part>.<instance>", key)
|
||||
}
|
||||
if _, err := PresencePartKey(fields[1], fields[2]); err != nil {
|
||||
return "", "", "", fmt.Errorf("delivery: %q is not a presence key: %w", key, err)
|
||||
}
|
||||
return "", fields[1], fields[2], nil
|
||||
}
|
||||
|
||||
// PulseKey is the key, in PulseBucket, of the move watched at key of bucket: `<bucket>.<key>`. The bucket is the
|
||||
// full bucket name and holds no dot; the key may.
|
||||
func PulseKey(bucket, key string) (string, error) {
|
||||
if err := checkName("bucket", bucket); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if key == "" || strings.HasPrefix(key, ".") || strings.HasSuffix(key, ".") || strings.Contains(key, "..") {
|
||||
return "", fmt.Errorf("delivery: the pulse key %q is empty or has an empty part", key)
|
||||
}
|
||||
for _, part := range strings.Split(key, ".") {
|
||||
if err := checkName("key", part); err != nil {
|
||||
return "", err
|
||||
}
|
||||
}
|
||||
return bucket + "." + key, nil
|
||||
}
|
||||
|
||||
// ParsePulseKey is the inverse of PulseKey, splitting at the first dot.
|
||||
func ParsePulseKey(pulseKey string) (bucket, key string, err error) {
|
||||
bucket, key, ok := strings.Cut(pulseKey, ".")
|
||||
if !ok {
|
||||
return "", "", fmt.Errorf("delivery: %q is not a pulse key, <bucket>.<key>", pulseKey)
|
||||
}
|
||||
if _, err := PulseKey(bucket, key); err != nil {
|
||||
return "", "", fmt.Errorf("delivery: %q is not a pulse key: %w", pulseKey, err)
|
||||
}
|
||||
return bucket, key, nil
|
||||
}
|
||||
|
||||
// PulseTTL is the time to live of a pulse key for a move of the given age: PulseMoveBound minus the age. At or past
|
||||
// the bound the move is stalled, with no time left and nothing to write: the watchman raises it at once.
|
||||
func PulseTTL(moveAge time.Duration) (ttl time.Duration, stalled bool) {
|
||||
if moveAge >= PulseMoveBound {
|
||||
return 0, true
|
||||
}
|
||||
return PulseMoveBound - moveAge, false
|
||||
}
|
||||
|
||||
// Have is a module's state on a node, written by that node's node-engine under HaveKey. FP is the module's
|
||||
// fingerprint once the indexer exists (design 47 release 3); until then Build is the build the node was sent, and
|
||||
// the view compares by build.
|
||||
type Have struct {
|
||||
V int `json:"v"`
|
||||
FP string `json:"fp,omitempty"`
|
||||
Build string `json:"build,omitempty"`
|
||||
Phase Phase `json:"phase"`
|
||||
Health Health `json:"health"`
|
||||
Detail string `json:"detail,omitempty"`
|
||||
}
|
||||
|
||||
// HaveNode is a node's own facts, written under HaveNodeKey. Each field is the node-engine's own JSON, carried whole:
|
||||
// this package names the facts and does not type them.
|
||||
type HaveNode struct {
|
||||
V int `json:"v"`
|
||||
Windows json.RawMessage `json:"windows,omitempty"`
|
||||
Busy json.RawMessage `json:"busy,omitempty"`
|
||||
Strays json.RawMessage `json:"strays,omitempty"`
|
||||
Filters json.RawMessage `json:"filters,omitempty"`
|
||||
Profile json.RawMessage `json:"profile,omitempty"`
|
||||
}
|
||||
|
||||
// Presence is whether a node or a part's instance is online; the entry's own time says since when. The watchman
|
||||
// writes it.
|
||||
type Presence struct {
|
||||
V int `json:"v"`
|
||||
Online bool `json:"online"`
|
||||
}
|
||||
|
||||
// Pulse is the value of a pulse key: the watched key mirrored, its revision, and who owes the next move (a node or
|
||||
// a part instance). The watchman writes it with the time to live PulseTTL gives.
|
||||
type Pulse struct {
|
||||
V int `json:"v"`
|
||||
Bucket string `json:"bucket"`
|
||||
Key string `json:"key"`
|
||||
Revision uint64 `json:"revision"`
|
||||
Owes string `json:"owes,omitempty"`
|
||||
}
|
||||
|
||||
func checkVersion(data []byte) error {
|
||||
var probe struct {
|
||||
V *int `json:"v"`
|
||||
}
|
||||
if err := json.Unmarshal(data, &probe); err != nil {
|
||||
return fmt.Errorf("delivery: value is not JSON: %w", err)
|
||||
}
|
||||
if probe.V == nil {
|
||||
return fmt.Errorf("%w: no version in the value, this reader reads %d", ErrUnknownVersion, Version)
|
||||
}
|
||||
if *probe.V != Version {
|
||||
return fmt.Errorf("%w: the value is version %d, this reader reads %d", ErrUnknownVersion, *probe.V, Version)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func decode(data []byte, into any) error {
|
||||
if err := checkVersion(data); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := json.Unmarshal(data, into); err != nil {
|
||||
return fmt.Errorf("delivery: value does not fit its shape: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (h Have) check() error {
|
||||
switch h.Phase {
|
||||
case PhaseBackingUp, PhaseApplying, PhaseChecking, PhaseRunning, PhaseFailed:
|
||||
default:
|
||||
return fmt.Errorf("delivery: phase %q is not backing-up, applying, checking, running or failed", h.Phase)
|
||||
}
|
||||
switch h.Health {
|
||||
case HealthOK, HealthFailed, HealthWaits, HealthUnknown:
|
||||
default:
|
||||
return fmt.Errorf("delivery: health %q is not ok, failed, waits or unknown", h.Health)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// EncodeHave is h as JSON at Version; it refuses a phase or health outside the lists.
|
||||
func EncodeHave(h Have) ([]byte, error) {
|
||||
if err := h.check(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
h.V = Version
|
||||
return json.Marshal(h)
|
||||
}
|
||||
|
||||
// DecodeHave reads a have; it refuses an unknown version (ErrUnknownVersion) and a phase or health outside the lists.
|
||||
func DecodeHave(data []byte) (Have, error) {
|
||||
var h Have
|
||||
if err := decode(data, &h); err != nil {
|
||||
return Have{}, err
|
||||
}
|
||||
if err := h.check(); err != nil {
|
||||
return Have{}, err
|
||||
}
|
||||
return h, nil
|
||||
}
|
||||
|
||||
// EncodeHaveNode is n as JSON at Version.
|
||||
func EncodeHaveNode(n HaveNode) ([]byte, error) {
|
||||
n.V = Version
|
||||
return json.Marshal(n)
|
||||
}
|
||||
|
||||
// DecodeHaveNode reads a node's facts; it refuses an unknown version (ErrUnknownVersion).
|
||||
func DecodeHaveNode(data []byte) (HaveNode, error) {
|
||||
var n HaveNode
|
||||
err := decode(data, &n)
|
||||
return n, err
|
||||
}
|
||||
|
||||
// EncodePresence is p as JSON at Version.
|
||||
func EncodePresence(p Presence) ([]byte, error) {
|
||||
p.V = Version
|
||||
return json.Marshal(p)
|
||||
}
|
||||
|
||||
// DecodePresence reads a presence; it refuses an unknown version (ErrUnknownVersion).
|
||||
func DecodePresence(data []byte) (Presence, error) {
|
||||
var p Presence
|
||||
err := decode(data, &p)
|
||||
return p, err
|
||||
}
|
||||
|
||||
// EncodePulse is p as JSON at Version.
|
||||
func EncodePulse(p Pulse) ([]byte, error) {
|
||||
p.V = Version
|
||||
return json.Marshal(p)
|
||||
}
|
||||
|
||||
// DecodePulse reads a pulse; it refuses an unknown version (ErrUnknownVersion).
|
||||
func DecodePulse(data []byte) (Pulse, error) {
|
||||
var p Pulse
|
||||
err := decode(data, &p)
|
||||
return p, err
|
||||
}
|
||||
|
||||
// SameHave says whether two haves are the same value, so a writer writes only when they differ.
|
||||
func SameHave(a, b Have) bool { return a == b }
|
||||
|
||||
// SameHaveNode says whether two node facts are the same value, comparing each carried JSON without its spacing.
|
||||
func SameHaveNode(a, b HaveNode) bool {
|
||||
return a.V == b.V && sameJSON(a.Windows, b.Windows) && sameJSON(a.Busy, b.Busy) &&
|
||||
sameJSON(a.Strays, b.Strays) && sameJSON(a.Filters, b.Filters) && sameJSON(a.Profile, b.Profile)
|
||||
}
|
||||
|
||||
// SamePresence says whether two presences are the same value.
|
||||
func SamePresence(a, b Presence) bool { return a == b }
|
||||
|
||||
// SamePulse says whether two pulses are the same value.
|
||||
func SamePulse(a, b Pulse) bool { return a == b }
|
||||
|
||||
func sameJSON(a, b json.RawMessage) bool {
|
||||
var ca, cb bytes.Buffer
|
||||
if json.Compact(&ca, a) != nil || json.Compact(&cb, b) != nil {
|
||||
return bytes.Equal(a, b)
|
||||
}
|
||||
return bytes.Equal(ca.Bytes(), cb.Bytes())
|
||||
}
|
||||
-65
@@ -1,65 +0,0 @@
|
||||
package delivery
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"time"
|
||||
)
|
||||
|
||||
// WatchmanModule is the watchman's module: its buckets are its own state, `<module>_<name>`, and the controller's
|
||||
// broker grants it alone the bus's connect and disconnect events (novox/hq ADR 0431).
|
||||
const WatchmanModule = "mesh-delivery-watchman"
|
||||
|
||||
// StuckBucket holds what stopped moving (novox/hq ADR 0433 §2): one key per key whose pulse expired, written by the
|
||||
// watchman, read by the controller's self-check, which raises each as a condition until it says moved. No key of it
|
||||
// is removed: a key that moves again is written as moved.
|
||||
const StuckBucket = WatchmanModule + "_stuck"
|
||||
|
||||
// StuckKey is the key, in StuckBucket, of the key that stopped moving: `<bucket>.<key>`, formed as PulseKey forms a
|
||||
// pulse's, so a node or part gone offline is `mesh-delivery-watchman_presence.<presence key>`.
|
||||
func StuckKey(bucket, key string) (string, error) {
|
||||
stuck, err := PulseKey(bucket, key)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("delivery: no stuck key: %w", err)
|
||||
}
|
||||
return stuck, nil
|
||||
}
|
||||
|
||||
// ParseStuckKey is the inverse of StuckKey, splitting at the first dot.
|
||||
func ParseStuckKey(stuckKey string) (bucket, key string, err error) {
|
||||
bucket, key, err = ParsePulseKey(stuckKey)
|
||||
if err != nil {
|
||||
return "", "", fmt.Errorf("delivery: %q is not a stuck key: %w", stuckKey, err)
|
||||
}
|
||||
return bucket, key, nil
|
||||
}
|
||||
|
||||
// Stuck is the value of a stuck key: the key that stopped moving, what it waited for in words, since when, and
|
||||
// whether it has moved again since. The watchman writes it; the controller reads it.
|
||||
type Stuck struct {
|
||||
V int `json:"v"`
|
||||
Bucket string `json:"bucket"`
|
||||
Key string `json:"key"`
|
||||
Waited string `json:"waited,omitempty"`
|
||||
Since time.Time `json:"since"`
|
||||
Moved bool `json:"moved,omitempty"`
|
||||
}
|
||||
|
||||
// EncodeStuck is s as JSON at Version.
|
||||
func EncodeStuck(s Stuck) ([]byte, error) {
|
||||
s.V = Version
|
||||
return json.Marshal(s)
|
||||
}
|
||||
|
||||
// DecodeStuck reads a stuck value; it refuses an unknown version (ErrUnknownVersion).
|
||||
func DecodeStuck(data []byte) (Stuck, error) {
|
||||
var s Stuck
|
||||
err := decode(data, &s)
|
||||
return s, err
|
||||
}
|
||||
|
||||
// SameStuck says whether two stuck values are the same, so a writer writes only when they differ.
|
||||
func SameStuck(a, b Stuck) bool {
|
||||
return a.V == b.V && a.Bucket == b.Bucket && a.Key == b.Key && a.Waited == b.Waited && a.Since.Equal(b.Since) &&
|
||||
a.Moved == b.Moved
|
||||
}
|
||||
Generated
Vendored
-21
@@ -1,21 +0,0 @@
|
||||
MIT License
|
||||
|
||||
Copyright (c) 2024 Antithesis
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all
|
||||
copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
SOFTWARE.
|
||||
Generated
Vendored
-214
@@ -1,214 +0,0 @@
|
||||
//go:build enable_antithesis_sdk
|
||||
|
||||
// Package assert enables defining [test properties] about your program or [workload]. It is part of the [Antithesis Go SDK], which enables Go applications to integrate with the [Antithesis platform].
|
||||
//
|
||||
// Code that uses this package should be instrumented with the [antithesis-go-generator] utility. This step is required for the Always, Sometime, and Reachable methods. It is not required for the Unreachable and AlwaysOrUnreachable methods, but it will improve the experience of using them.
|
||||
//
|
||||
// These functions are no-ops with minimal performance overhead when called outside of the Antithesis environment. However, if the environment variable ANTITHESIS_SDK_LOCAL_OUTPUT is set, these functions will log to the file pointed to by that variable using a structured JSON format defined [here]. This allows you to make use of the Antithesis assertions package in your regular testing, or even in production. In particular, very few assertions frameworks offer a convenient way to define [Sometimes assertions], but they can be quite useful even outside Antithesis.
|
||||
//
|
||||
// Each function in this package takes a parameter called message, which is a human readable identifier used to aggregate assertions. Antithesis generates one test property per unique message and this test property will be named "<message>" in the [triage report].
|
||||
//
|
||||
// This test property either passes or fails, which depends upon the evaluation of every assertion that shares its message. Different assertions in different parts of the code should have different message, but the same assertion should always have the same message even if it is moved to a different file.
|
||||
//
|
||||
// Each function also takes a parameter called details, which is a key-value map of optional additional information provided by the user to add context for assertion failures. The information that is logged will appear in the [triage report], under the details section of the corresponding property. Normally the values passed to details are evaluated at runtime.
|
||||
//
|
||||
// [Antithesis Go SDK]: https://antithesis.com/docs/using_antithesis/sdk/go/
|
||||
// [Antithesis platform]: https://antithesis.com
|
||||
// [test properties]: https://antithesis.com/docs/properties_assertions/properties/
|
||||
// [workload]: https://antithesis.com/docs/test_templates/first_test/
|
||||
// [antithesis-go-generator]: https://antithesis.com/docs/using_antithesis/sdk/go/instrumentor/
|
||||
// [triage report]: https://antithesis.com/docs/reports/
|
||||
// [here]: https://antithesis.com/docs/using_antithesis/sdk/fallback/
|
||||
// [Sometimes assertions]: https://antithesis.com/docs/best_practices/sometimes_assertions/
|
||||
//
|
||||
// [details]: https://antithesis.com/docs/reports/
|
||||
package assert
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
)
|
||||
|
||||
type assertInfo struct {
|
||||
Location *locationInfo `json:"location"`
|
||||
Details map[string]any `json:"details"`
|
||||
AssertType string `json:"assert_type"`
|
||||
DisplayType string `json:"display_type"`
|
||||
Message string `json:"message"`
|
||||
Id string `json:"id"`
|
||||
Hit bool `json:"hit"`
|
||||
MustHit bool `json:"must_hit"`
|
||||
Condition bool `json:"condition"`
|
||||
}
|
||||
|
||||
// Create a custom json marshaler for assertInfo so that we can force Errors to be marshaled with their error details.
|
||||
// Without this, custom errors are marshaled as an empty object because the default json marshaler doesn't include the error
|
||||
// (because it's a method - not an exported struct field).
|
||||
func (f assertInfo) MarshalJSON() ([]byte, error) {
|
||||
type alias assertInfo // prevent infinite recursion
|
||||
a := alias(f)
|
||||
if a.Details != nil {
|
||||
a.Details = normalizeMap(a.Details)
|
||||
}
|
||||
return json.Marshal(a)
|
||||
}
|
||||
|
||||
type jsonError struct {
|
||||
innerError error
|
||||
}
|
||||
|
||||
func (e jsonError) MarshalJSON() ([]byte, error) {
|
||||
// Marshal this as the debug output string instead of e.Error(). These should be equivalent, but Sprintf correctly
|
||||
// handles nil values for us (which otherwise are annoying to defend against due to this - https://go.dev/doc/faq#nil_error)
|
||||
return json.Marshal(fmt.Sprintf("%+v", e.innerError))
|
||||
}
|
||||
|
||||
// Recursively replace any `error` with jsonError while doing a deep copy.
|
||||
// Most of the logic is in the normalize method below. This method exists to localize the type assertions
|
||||
// and provide a function that takes in/out a map instead of any.
|
||||
func normalizeMap(v map[string]any) map[string]any {
|
||||
return normalize(v).(map[string]any)
|
||||
}
|
||||
|
||||
func normalize(input any) any {
|
||||
// This switch will miss some cases (pointers, structs, non-any types), but should catch a very large proportion of real error interfaces
|
||||
// in real details objects. We can augment this if we find other cases common enough to support.
|
||||
switch inputTyped := input.(type) {
|
||||
case error:
|
||||
// Check if the underlying error implements json.Marshaler, so that if the error
|
||||
// already knows who to marshal itself, we don't override that.
|
||||
if _, ok := inputTyped.(json.Marshaler); ok {
|
||||
return inputTyped
|
||||
} else {
|
||||
return jsonError{inputTyped}
|
||||
}
|
||||
case map[string]any:
|
||||
out := make(map[string]any, len(inputTyped))
|
||||
for k, v := range inputTyped {
|
||||
out[k] = normalize(v)
|
||||
}
|
||||
return out
|
||||
case []any:
|
||||
out := make([]any, len(inputTyped))
|
||||
for i := range inputTyped {
|
||||
out[i] = normalize(inputTyped[i])
|
||||
}
|
||||
return out
|
||||
default:
|
||||
return input
|
||||
}
|
||||
}
|
||||
|
||||
type wrappedAssertInfo struct {
|
||||
A *assertInfo `json:"antithesis_assert"`
|
||||
}
|
||||
|
||||
// --------------------------------------------------------------------------------
|
||||
// Assertions
|
||||
// --------------------------------------------------------------------------------
|
||||
const (
|
||||
wasHit = true
|
||||
mustBeHit = true
|
||||
optionallyHit = false
|
||||
expectingTrue = true
|
||||
)
|
||||
|
||||
const (
|
||||
universalTest = "always"
|
||||
existentialTest = "sometimes"
|
||||
reachabilityTest = "reachability"
|
||||
)
|
||||
|
||||
const (
|
||||
alwaysDisplay = "Always"
|
||||
alwaysOrUnreachableDisplay = "AlwaysOrUnreachable"
|
||||
sometimesDisplay = "Sometimes"
|
||||
reachableDisplay = "Reachable"
|
||||
unreachableDisplay = "Unreachable"
|
||||
)
|
||||
|
||||
// Always asserts that condition is true every time this function is called, and that it is called at least once. The corresponding test property will be viewable in the Antithesis SDK: Always group of your triage report.
|
||||
func Always(condition bool, message string, details map[string]any) {
|
||||
locationInfo := newLocationInfo(offsetAPICaller)
|
||||
id := makeKey(message, locationInfo)
|
||||
assertImpl(condition, message, details, locationInfo, wasHit, mustBeHit, universalTest, alwaysDisplay, id)
|
||||
}
|
||||
|
||||
// AlwaysOrUnreachable asserts that condition is true every time this function is called. The corresponding test property will pass if the assertion is never encountered (unlike Always assertion types). This test property will be viewable in the “Antithesis SDK: Always” group of your triage report.
|
||||
func AlwaysOrUnreachable(condition bool, message string, details map[string]any) {
|
||||
locationInfo := newLocationInfo(offsetAPICaller)
|
||||
id := makeKey(message, locationInfo)
|
||||
assertImpl(condition, message, details, locationInfo, wasHit, optionallyHit, universalTest, alwaysOrUnreachableDisplay, id)
|
||||
}
|
||||
|
||||
// Sometimes asserts that condition is true at least one time that this function was called. (If the assertion is never encountered, the test property will therefore fail.) This test property will be viewable in the “Antithesis SDK: Sometimes” group.
|
||||
func Sometimes(condition bool, message string, details map[string]any) {
|
||||
locationInfo := newLocationInfo(offsetAPICaller)
|
||||
id := makeKey(message, locationInfo)
|
||||
assertImpl(condition, message, details, locationInfo, wasHit, mustBeHit, existentialTest, sometimesDisplay, id)
|
||||
}
|
||||
|
||||
// Unreachable asserts that a line of code is never reached. The corresponding test property will fail if this function is ever called. (If it is never called the test property will therefore pass.) This test property will be viewable in the “Antithesis SDK: Reachablity assertions” group.
|
||||
func Unreachable(message string, details map[string]any) {
|
||||
locationInfo := newLocationInfo(offsetAPICaller)
|
||||
id := makeKey(message, locationInfo)
|
||||
assertImpl(false, message, details, locationInfo, wasHit, optionallyHit, reachabilityTest, unreachableDisplay, id)
|
||||
}
|
||||
|
||||
// Reachable asserts that a line of code is reached at least once. The corresponding test property will pass if this function is ever called. (If it is never called the test property will therefore fail.) This test property will be viewable in the “Antithesis SDK: Reachablity assertions” group.
|
||||
func Reachable(message string, details map[string]any) {
|
||||
locationInfo := newLocationInfo(offsetAPICaller)
|
||||
id := makeKey(message, locationInfo)
|
||||
assertImpl(true, message, details, locationInfo, wasHit, mustBeHit, reachabilityTest, reachableDisplay, id)
|
||||
}
|
||||
|
||||
// AssertRaw is a low-level method designed to be used by third-party frameworks. Regular users of the assert package should not call it.
|
||||
func AssertRaw(cond bool, message string, details map[string]any,
|
||||
classname, funcname, filename string, line int,
|
||||
hit bool, mustHit bool,
|
||||
assertType string, displayType string,
|
||||
id string,
|
||||
) {
|
||||
assertImpl(cond, message, details,
|
||||
&locationInfo{classname, funcname, filename, line, columnUnknown},
|
||||
hit, mustHit,
|
||||
assertType, displayType,
|
||||
id)
|
||||
}
|
||||
|
||||
func assertImpl(cond bool, message string, details map[string]any,
|
||||
loc *locationInfo,
|
||||
hit bool, mustHit bool,
|
||||
assertType string, displayType string,
|
||||
id string,
|
||||
) {
|
||||
trackerEntry := assertTracker.getTrackerEntry(id, loc.Filename, loc.Classname)
|
||||
|
||||
// Always grab the Filename and Classname captured when the trackerEntry was established
|
||||
// This provides the consistency needed between instrumentation-time and runtime
|
||||
if loc.Filename != trackerEntry.Filename {
|
||||
loc.Filename = trackerEntry.Filename
|
||||
}
|
||||
|
||||
if loc.Classname != trackerEntry.Classname {
|
||||
loc.Classname = trackerEntry.Classname
|
||||
}
|
||||
|
||||
aI := &assertInfo{
|
||||
Hit: hit,
|
||||
MustHit: mustHit,
|
||||
AssertType: assertType,
|
||||
DisplayType: displayType,
|
||||
Message: message,
|
||||
Condition: cond,
|
||||
Id: id,
|
||||
Location: loc,
|
||||
Details: details,
|
||||
}
|
||||
|
||||
trackerEntry.emit(aI)
|
||||
}
|
||||
|
||||
func makeKey(message string, _ *locationInfo) string {
|
||||
return message
|
||||
}
|
||||
Generated
Vendored
-16
@@ -1,16 +0,0 @@
|
||||
//go:build !enable_antithesis_sdk
|
||||
|
||||
package assert
|
||||
|
||||
func Always(condition bool, message string, details map[string]any) {}
|
||||
func AlwaysOrUnreachable(condition bool, message string, details map[string]any) {}
|
||||
func Sometimes(condition bool, message string, details map[string]any) {}
|
||||
func Unreachable(message string, details map[string]any) {}
|
||||
func Reachable(message string, details map[string]any) {}
|
||||
func AssertRaw(cond bool, message string, details map[string]any,
|
||||
classname, funcname, filename string, line int,
|
||||
hit bool, mustHit bool,
|
||||
assertType string, displayType string,
|
||||
id string,
|
||||
) {
|
||||
}
|
||||
Generated
Vendored
-30
@@ -1,30 +0,0 @@
|
||||
package assert
|
||||
|
||||
// Allowable numeric types of comparison parameters
|
||||
type Number interface {
|
||||
~int | ~int8 | ~int16 | ~int32 | ~int64 | ~uint8 | ~uint16 | ~uint32 | ~float32 | ~float64 | ~uint64 | ~uint | ~uintptr
|
||||
}
|
||||
|
||||
// Internally, numeric guidanceFn Operands only use these
|
||||
type operandConstraint interface {
|
||||
int32 | int64 | uint64 | float64
|
||||
}
|
||||
|
||||
type numConstraint interface {
|
||||
uint64 | float64
|
||||
}
|
||||
|
||||
// Used for boolean assertions
|
||||
type NamedBool struct {
|
||||
First string `json:"first"`
|
||||
Second bool `json:"second"`
|
||||
}
|
||||
|
||||
// Convenience function to construct a NamedBool used for boolean assertions
|
||||
func NewNamedBool(first string, second bool) *NamedBool {
|
||||
p := NamedBool{
|
||||
First: first,
|
||||
Second: second,
|
||||
}
|
||||
return &p
|
||||
}
|
||||
Generated
Vendored
-66
@@ -1,66 +0,0 @@
|
||||
//go:build enable_antithesis_sdk
|
||||
|
||||
package assert
|
||||
|
||||
import (
|
||||
"sync"
|
||||
|
||||
"github.com/antithesishq/antithesis-sdk-go/internal"
|
||||
)
|
||||
|
||||
// TODO: Tracker is intended to prevent sending the same guidance
|
||||
// more than once. In this case, we always send, so the tracker
|
||||
// is not presently used.
|
||||
type booleanGuidance struct {
|
||||
n int
|
||||
}
|
||||
|
||||
type booleanGuidanceTracker map[string]*booleanGuidance
|
||||
|
||||
var (
|
||||
boolean_guidance_tracker booleanGuidanceTracker = make(booleanGuidanceTracker)
|
||||
boolean_guidance_tracker_mutex sync.Mutex
|
||||
boolean_guidance_info_mutex sync.Mutex
|
||||
)
|
||||
|
||||
func (tracker booleanGuidanceTracker) getTrackerEntry(messageKey string) *booleanGuidance {
|
||||
var trackerEntry *booleanGuidance
|
||||
var ok bool
|
||||
|
||||
if tracker == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
boolean_guidance_tracker_mutex.Lock()
|
||||
defer boolean_guidance_tracker_mutex.Unlock()
|
||||
if trackerEntry, ok = boolean_guidance_tracker[messageKey]; !ok {
|
||||
trackerEntry = newBooleanGuidance()
|
||||
tracker[messageKey] = trackerEntry
|
||||
}
|
||||
|
||||
return trackerEntry
|
||||
}
|
||||
|
||||
// Create a boolean guidance tracker
|
||||
func newBooleanGuidance() *booleanGuidance {
|
||||
trackerInfo := booleanGuidance{}
|
||||
return &trackerInfo
|
||||
}
|
||||
|
||||
func (tI *booleanGuidance) send_value(bgI *booleanGuidanceInfo) {
|
||||
if tI == nil {
|
||||
return
|
||||
}
|
||||
|
||||
boolean_guidance_info_mutex.Lock()
|
||||
defer boolean_guidance_info_mutex.Unlock()
|
||||
|
||||
// The tracker entry should be consulted to determine
|
||||
// if this Guidance info has already been sent, or not.
|
||||
|
||||
emitBooleanGuidance(bgI)
|
||||
}
|
||||
|
||||
func emitBooleanGuidance(bgI *booleanGuidanceInfo) error {
|
||||
return internal.Json_data(map[string]any{"antithesis_guidance": bgI})
|
||||
}
|
||||
Generated
Vendored
-58
@@ -1,58 +0,0 @@
|
||||
//go:build enable_antithesis_sdk
|
||||
|
||||
package assert
|
||||
|
||||
import (
|
||||
"path"
|
||||
"runtime"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// stackFrameOffset indicates how many frames to go up in the
|
||||
// call stack to find the filename/location/line info. As
|
||||
// this work is always done in NewLocationInfo(), the offset is
|
||||
// specified from the perspective of NewLocationInfo
|
||||
type stackFrameOffset int
|
||||
|
||||
// Order is important here since iota is being used
|
||||
const (
|
||||
offsetNewLocationInfo stackFrameOffset = iota
|
||||
offsetHere
|
||||
offsetAPICaller
|
||||
offsetAPICallersCaller
|
||||
)
|
||||
|
||||
// locationInfo represents the attributes known at instrumentation time
|
||||
// for each Antithesis assertion discovered
|
||||
type locationInfo struct {
|
||||
Classname string `json:"class"`
|
||||
Funcname string `json:"function"`
|
||||
Filename string `json:"file"`
|
||||
Line int `json:"begin_line"`
|
||||
Column int `json:"begin_column"`
|
||||
}
|
||||
|
||||
// columnUnknown is used when the column associated with
|
||||
// a locationInfo is not available
|
||||
const columnUnknown = 0
|
||||
|
||||
// NewLocationInfo creates a locationInfo directly from
|
||||
// the current execution context
|
||||
func newLocationInfo(nframes stackFrameOffset) *locationInfo {
|
||||
// Get location info and add to details
|
||||
funcname := "*function*"
|
||||
classname := "*class*"
|
||||
pc, filename, line, ok := runtime.Caller(int(nframes))
|
||||
if !ok {
|
||||
filename = "*file*"
|
||||
line = 0
|
||||
} else {
|
||||
if this_func := runtime.FuncForPC(pc); this_func != nil {
|
||||
fullname := this_func.Name()
|
||||
funcname = path.Ext(fullname)
|
||||
classname, _ = strings.CutSuffix(fullname, funcname)
|
||||
funcname = funcname[1:]
|
||||
}
|
||||
}
|
||||
return &locationInfo{classname, funcname, filename, line, columnUnknown}
|
||||
}
|
||||
Generated
Vendored
-323
@@ -1,323 +0,0 @@
|
||||
//go:build enable_antithesis_sdk
|
||||
|
||||
package assert
|
||||
|
||||
import (
|
||||
"math"
|
||||
"sync"
|
||||
|
||||
"github.com/antithesishq/antithesis-sdk-go/internal"
|
||||
)
|
||||
|
||||
// --------------------------------------------------------------------------------
|
||||
// IntegerGap is used for:
|
||||
// - int, int8, int16, int32, int64:
|
||||
// - uint, uint8, uint16, uint32, uint64, uintptr:
|
||||
//
|
||||
// FloatGap is used for:
|
||||
// - float32, float64
|
||||
// --------------------------------------------------------------------------------
|
||||
type numericGapType int
|
||||
|
||||
const (
|
||||
integerGapType numericGapType = iota
|
||||
floatGapType
|
||||
)
|
||||
|
||||
func gapTypeForOperand[T Number](num T) numericGapType {
|
||||
gapType := integerGapType
|
||||
|
||||
switch any(num).(type) {
|
||||
case float32, float64:
|
||||
gapType = floatGapType
|
||||
}
|
||||
return gapType
|
||||
}
|
||||
|
||||
// --------------------------------------------------------------------------------
|
||||
// numericGuidanceTracker - Tracking Info for Numeric Guidance
|
||||
//
|
||||
// For GuidanceFnMaximize:
|
||||
// - gap is the largest value sent so far
|
||||
//
|
||||
// For GuidanceFnMinimize:
|
||||
// - gap is the most negative value sent so far
|
||||
//
|
||||
// --------------------------------------------------------------------------------
|
||||
type numericGuidanceInfo struct {
|
||||
gap any
|
||||
descriminator numericGapType
|
||||
maximize bool
|
||||
}
|
||||
|
||||
type numericGuidanceTracker map[string]*numericGuidanceInfo
|
||||
|
||||
var (
|
||||
numeric_guidance_tracker numericGuidanceTracker = make(numericGuidanceTracker)
|
||||
numeric_guidance_tracker_mutex sync.Mutex
|
||||
numeric_guidance_info_mutex sync.Mutex
|
||||
)
|
||||
|
||||
func (tracker numericGuidanceTracker) getTrackerEntry(messageKey string, trackerType numericGapType, maximize bool) *numericGuidanceInfo {
|
||||
var trackerEntry *numericGuidanceInfo
|
||||
var ok bool
|
||||
|
||||
if tracker == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
numeric_guidance_tracker_mutex.Lock()
|
||||
defer numeric_guidance_tracker_mutex.Unlock()
|
||||
if trackerEntry, ok = numeric_guidance_tracker[messageKey]; !ok {
|
||||
trackerEntry = newNumericGuidanceInfo(trackerType, maximize)
|
||||
tracker[messageKey] = trackerEntry
|
||||
}
|
||||
|
||||
return trackerEntry
|
||||
}
|
||||
|
||||
// Create an numeric guidance entry
|
||||
func newNumericGuidanceInfo(trackerType numericGapType, maximize bool) *numericGuidanceInfo {
|
||||
|
||||
var gap any
|
||||
if trackerType == integerGapType {
|
||||
gap = newGapValue(uint64(math.MaxUint64), maximize)
|
||||
} else {
|
||||
gap = newGapValue(float64(math.MaxFloat64), maximize)
|
||||
}
|
||||
trackerInfo := numericGuidanceInfo{
|
||||
maximize: maximize,
|
||||
descriminator: trackerType,
|
||||
gap: gap,
|
||||
}
|
||||
return &trackerInfo
|
||||
}
|
||||
|
||||
func (tI *numericGuidanceInfo) should_maximize() bool {
|
||||
return tI.maximize
|
||||
}
|
||||
|
||||
func (tI *numericGuidanceInfo) is_integer_gap() bool {
|
||||
return tI.descriminator == integerGapType
|
||||
}
|
||||
|
||||
// --------------------------------------------------------------------------------
|
||||
// Represents integral and floating point extremes
|
||||
// --------------------------------------------------------------------------------
|
||||
type gapValue[T numConstraint] struct {
|
||||
gap_size T
|
||||
gap_is_negative bool
|
||||
}
|
||||
|
||||
func newGapValue[T numConstraint](sz T, is_neg bool) any {
|
||||
switch any(sz).(type) {
|
||||
case uint64:
|
||||
return gapValue[uint64]{gap_size: uint64(sz), gap_is_negative: is_neg}
|
||||
|
||||
case float64:
|
||||
return gapValue[float64]{gap_size: float64(sz), gap_is_negative: is_neg}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func is_same_sign(left_val int64, right_val int64) bool {
|
||||
same_sign := false
|
||||
if left_val < 0 {
|
||||
// left is negative
|
||||
if right_val < 0 {
|
||||
same_sign = true
|
||||
}
|
||||
} else {
|
||||
// left is non-negative
|
||||
if right_val >= 0 {
|
||||
same_sign = true
|
||||
}
|
||||
}
|
||||
return same_sign
|
||||
}
|
||||
|
||||
func abs_int64(val int64) uint64 {
|
||||
if val >= 0 {
|
||||
return uint64(val)
|
||||
}
|
||||
return uint64(0 - val)
|
||||
}
|
||||
|
||||
func is_greater_than[T numConstraint](left gapValue[T], right gapValue[T]) bool {
|
||||
if !left.gap_is_negative && !right.gap_is_negative {
|
||||
return left.gap_size > right.gap_size
|
||||
}
|
||||
if !left.gap_is_negative && right.gap_is_negative {
|
||||
return true // any positive is greater than a negative
|
||||
}
|
||||
if left.gap_is_negative && right.gap_is_negative {
|
||||
return right.gap_size > left.gap_size
|
||||
}
|
||||
if left.gap_is_negative && !right.gap_is_negative {
|
||||
return false // any negative is less than a positive
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func is_less_than[T numConstraint](left gapValue[T], right gapValue[T]) bool {
|
||||
if !left.gap_is_negative && !right.gap_is_negative {
|
||||
return left.gap_size < right.gap_size
|
||||
}
|
||||
if !left.gap_is_negative && right.gap_is_negative {
|
||||
return false // any positive is greater than a negative
|
||||
}
|
||||
if left.gap_is_negative && right.gap_is_negative {
|
||||
return right.gap_size < left.gap_size
|
||||
}
|
||||
if left.gap_is_negative && !right.gap_is_negative {
|
||||
return true // any negative is less than a positive
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func send_value_if_needed(tI *numericGuidanceInfo, gI *guidanceInfo) {
|
||||
if tI == nil {
|
||||
return
|
||||
}
|
||||
|
||||
numeric_guidance_info_mutex.Lock()
|
||||
defer numeric_guidance_info_mutex.Unlock()
|
||||
|
||||
// if this is a catalog entry (gI.hit is false)
|
||||
// do not update the reference gap in the tracker (tI *numericGuidanceInfo)
|
||||
if !gI.Hit {
|
||||
emitGuidance(gI)
|
||||
return
|
||||
}
|
||||
|
||||
should_send := false
|
||||
maximize := tI.should_maximize()
|
||||
|
||||
var gap gapValue[uint64]
|
||||
var float_gap gapValue[float64]
|
||||
|
||||
// Needs to have individual case statements to assist
|
||||
// the compiler to infer the actual type of the var named 'operands'
|
||||
switch operands := (gI.Data).(type) {
|
||||
case numericOperands[int32]:
|
||||
gap = makeGap(operands)
|
||||
case numericOperands[int64]:
|
||||
gap = makeGap(operands)
|
||||
case numericOperands[uint64]:
|
||||
gap = makeGap(operands)
|
||||
case numericOperands[float64]:
|
||||
float_gap = makeFloatGap(operands)
|
||||
}
|
||||
|
||||
var prev_gap gapValue[uint64]
|
||||
var prev_float_gap gapValue[float64]
|
||||
has_prev_gap := false
|
||||
has_prev_float_gap := false
|
||||
|
||||
prev_gap, has_prev_gap = tI.gap.(gapValue[uint64])
|
||||
if !has_prev_gap {
|
||||
prev_float_gap, has_prev_float_gap = tI.gap.(gapValue[float64])
|
||||
}
|
||||
|
||||
if has_prev_gap {
|
||||
if maximize {
|
||||
should_send = is_greater_than(gap, prev_gap)
|
||||
} else {
|
||||
should_send = is_less_than(gap, prev_gap)
|
||||
}
|
||||
}
|
||||
|
||||
if has_prev_float_gap {
|
||||
if maximize {
|
||||
should_send = is_greater_than(float_gap, prev_float_gap)
|
||||
} else {
|
||||
should_send = is_less_than(float_gap, prev_float_gap)
|
||||
}
|
||||
}
|
||||
|
||||
if should_send {
|
||||
if tI.is_integer_gap() {
|
||||
tI.gap = gap
|
||||
} else {
|
||||
tI.gap = float_gap
|
||||
}
|
||||
emitGuidance(gI)
|
||||
}
|
||||
}
|
||||
|
||||
func emitGuidance(gI *guidanceInfo) error {
|
||||
return internal.Json_data(map[string]any{"antithesis_guidance": gI})
|
||||
}
|
||||
|
||||
// When left and right are the same sign (both negative, or both non-negative)
|
||||
// Calculate: <result> = (left - right). The gap_size is abs(<result>) and
|
||||
// gap_is_negative is (right > left)
|
||||
func makeGap[Op operandConstraint](operand numericOperands[Op]) gapValue[uint64] {
|
||||
|
||||
var gap_size uint64
|
||||
var gap_is_negative bool
|
||||
|
||||
switch this_op := any(operand).(type) {
|
||||
case numericOperands[int32]:
|
||||
result := int64(this_op.Left) - int64(this_op.Right)
|
||||
gap_size = abs_int64(result)
|
||||
gap_is_negative = result < 0
|
||||
|
||||
case numericOperands[int64]:
|
||||
if is_same_sign(this_op.Left, this_op.Right) {
|
||||
result := int64(this_op.Left) - int64(this_op.Right)
|
||||
gap_size = abs_int64(result)
|
||||
gap_is_negative = result < 0
|
||||
break
|
||||
}
|
||||
|
||||
// Otherwise left and right are opposite signs
|
||||
// gap = abs(left) + abs(right)
|
||||
// gap_is_negative = left < right
|
||||
left_gap_size := abs_int64(this_op.Left)
|
||||
right_gap_size := abs_int64(this_op.Right)
|
||||
gap_size = left_gap_size + right_gap_size
|
||||
gap_is_negative = this_op.Left < this_op.Right
|
||||
|
||||
case numericOperands[uint64]:
|
||||
left_val := this_op.Left
|
||||
right_val := this_op.Right
|
||||
gap_is_negative = false
|
||||
if left_val < right_val {
|
||||
gap_is_negative = true
|
||||
gap_size = right_val - left_val
|
||||
} else {
|
||||
gap_size = left_val - right_val
|
||||
}
|
||||
|
||||
default:
|
||||
zero_gap, _ := newGapValue(uint64(0), false).(gapValue[uint64])
|
||||
return zero_gap
|
||||
}
|
||||
|
||||
this_gap, _ := newGapValue(gap_size, gap_is_negative).(gapValue[uint64])
|
||||
return this_gap
|
||||
} // MakeGap
|
||||
|
||||
func makeFloatGap[Op operandConstraint](operand numericOperands[Op]) gapValue[float64] {
|
||||
switch this_op := any(operand).(type) {
|
||||
case numericOperands[float64]:
|
||||
left_val := this_op.Left
|
||||
right_val := this_op.Right
|
||||
gap_is_negative := false
|
||||
var gap_size float64
|
||||
if left_val < right_val {
|
||||
gap_is_negative = true
|
||||
gap_size = right_val - left_val
|
||||
} else {
|
||||
gap_size = left_val - right_val
|
||||
}
|
||||
|
||||
this_gap, _ := newGapValue(gap_size, gap_is_negative).(gapValue[float64])
|
||||
return this_gap
|
||||
|
||||
default:
|
||||
zero_gap, _ := newGapValue(float64(0.0), false).(gapValue[float64])
|
||||
return zero_gap
|
||||
}
|
||||
} // MakeFloatGap
|
||||
Generated
Vendored
-330
@@ -1,330 +0,0 @@
|
||||
//go:build enable_antithesis_sdk
|
||||
|
||||
package assert
|
||||
|
||||
// A type for writing raw assertions.
|
||||
// guidanceFnType allows the assertion to provide guidance to
|
||||
// the Antithesis platform when testing in Antithesis.
|
||||
// Regular users of the assert package should not use it.
|
||||
type guidanceFnType int
|
||||
|
||||
const (
|
||||
guidanceFnMaximize guidanceFnType = iota // Maximize (left - right) values
|
||||
guidanceFnMinimize // Minimize (left - right) values
|
||||
guidanceFnWantAll // Encourages fuzzing explorations where boolean values are true
|
||||
guidanceFnWantNone // Encourages fuzzing explorations where boolean values are false
|
||||
guidanceFnExplore
|
||||
)
|
||||
|
||||
// guidanceFnExplore
|
||||
|
||||
func get_guidance_type_string(gt guidanceFnType) string {
|
||||
switch gt {
|
||||
case guidanceFnMaximize, guidanceFnMinimize:
|
||||
return "numeric"
|
||||
case guidanceFnWantAll, guidanceFnWantNone:
|
||||
return "boolean"
|
||||
case guidanceFnExplore:
|
||||
return "json"
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
type numericOperands[T operandConstraint] struct {
|
||||
Left T `json:"left"`
|
||||
Right T `json:"right"`
|
||||
}
|
||||
|
||||
type guidanceInfo struct {
|
||||
Data any `json:"guidance_data,omitempty"`
|
||||
Location *locationInfo `json:"location"`
|
||||
GuidanceType string `json:"guidance_type"`
|
||||
Message string `json:"message"`
|
||||
Id string `json:"id"`
|
||||
Maximize bool `json:"maximize"`
|
||||
Hit bool `json:"hit"`
|
||||
}
|
||||
|
||||
type booleanGuidanceInfo struct {
|
||||
Data any `json:"guidance_data,omitempty"`
|
||||
Location *locationInfo `json:"location"`
|
||||
GuidanceType string `json:"guidance_type"`
|
||||
Message string `json:"message"`
|
||||
Id string `json:"id"`
|
||||
Maximize bool `json:"maximize"`
|
||||
Hit bool `json:"hit"`
|
||||
}
|
||||
|
||||
func uses_maximize(gt guidanceFnType) bool {
|
||||
return gt == guidanceFnMaximize || gt == guidanceFnWantAll
|
||||
}
|
||||
|
||||
func newOperands[T Number](left, right T) any {
|
||||
switch any(left).(type) {
|
||||
case int8, int16, int32:
|
||||
return numericOperands[int32]{int32(left), int32(right)}
|
||||
case int, int64:
|
||||
return numericOperands[int64]{int64(left), int64(right)}
|
||||
case uint8, uint16, uint32, uint, uint64, uintptr:
|
||||
return numericOperands[uint64]{uint64(left), uint64(right)}
|
||||
case float32, float64:
|
||||
return numericOperands[float64]{float64(left), float64(right)}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func build_numeric_guidance[T Number](gt guidanceFnType, message string, left, right T, loc *locationInfo, id string, hit bool) *guidanceInfo {
|
||||
|
||||
operands := newOperands(left, right)
|
||||
if !hit {
|
||||
operands = nil
|
||||
}
|
||||
|
||||
gI := guidanceInfo{
|
||||
GuidanceType: get_guidance_type_string(gt),
|
||||
Message: message,
|
||||
Id: id,
|
||||
Location: loc,
|
||||
Maximize: uses_maximize(gt),
|
||||
Data: operands,
|
||||
Hit: hit,
|
||||
}
|
||||
|
||||
return &gI
|
||||
}
|
||||
|
||||
type namedBoolDictionary map[string]bool
|
||||
|
||||
func build_boolean_guidance(gt guidanceFnType, message string, named_bools []NamedBool,
|
||||
loc *locationInfo,
|
||||
id string, hit bool) *booleanGuidanceInfo {
|
||||
|
||||
var guidance_data any
|
||||
|
||||
// To ensure the sequence and naming for the named_bool values
|
||||
if hit {
|
||||
named_bool_dictionary := namedBoolDictionary{}
|
||||
for _, named_bool := range named_bools {
|
||||
named_bool_dictionary[named_bool.First] = named_bool.Second
|
||||
}
|
||||
guidance_data = named_bool_dictionary
|
||||
}
|
||||
|
||||
bgI := booleanGuidanceInfo{
|
||||
GuidanceType: get_guidance_type_string(gt),
|
||||
Message: message,
|
||||
Id: id,
|
||||
Location: loc,
|
||||
Maximize: uses_maximize(gt),
|
||||
Data: guidance_data,
|
||||
Hit: hit,
|
||||
}
|
||||
|
||||
return &bgI
|
||||
}
|
||||
|
||||
func behavior_to_guidance(behavior string) guidanceFnType {
|
||||
guidance := guidanceFnExplore
|
||||
switch behavior {
|
||||
case "maximize":
|
||||
guidance = guidanceFnMaximize
|
||||
case "minimize":
|
||||
guidance = guidanceFnMinimize
|
||||
case "all":
|
||||
guidance = guidanceFnWantAll
|
||||
case "none":
|
||||
guidance = guidanceFnWantNone
|
||||
}
|
||||
return guidance
|
||||
}
|
||||
|
||||
func numericGuidanceImpl[T Number](left, right T, message, id string, loc *locationInfo, guidanceFn guidanceFnType, hit bool) {
|
||||
tI := numeric_guidance_tracker.getTrackerEntry(id, gapTypeForOperand(left), uses_maximize(guidanceFn))
|
||||
gI := build_numeric_guidance(guidanceFn, message, left, right, loc, id, hit)
|
||||
send_value_if_needed(tI, gI)
|
||||
}
|
||||
|
||||
func booleanGuidanceImpl(named_bools []NamedBool, message, id string, loc *locationInfo, guidanceFn guidanceFnType, hit bool) {
|
||||
tI := boolean_guidance_tracker.getTrackerEntry(id)
|
||||
bgI := build_boolean_guidance(guidanceFn, message, named_bools, loc, id, hit)
|
||||
tI.send_value(bgI)
|
||||
}
|
||||
|
||||
// NumericGuidanceRaw is a low-level method designed to be used by third-party frameworks. Regular users of the assert package should not call it.
|
||||
func NumericGuidanceRaw[T Number](
|
||||
left, right T,
|
||||
message, id string,
|
||||
classname, funcname, filename string,
|
||||
line int,
|
||||
behavior string,
|
||||
hit bool,
|
||||
) {
|
||||
loc := &locationInfo{classname, funcname, filename, line, columnUnknown}
|
||||
guidanceFn := behavior_to_guidance(behavior)
|
||||
numericGuidanceImpl(left, right, message, id, loc, guidanceFn, hit)
|
||||
}
|
||||
|
||||
// BooleanGuidanceRaw is a low-level method designed to be used by third-party frameworks. Regular users of the assert package should not call it.
|
||||
func BooleanGuidanceRaw(
|
||||
named_bools []NamedBool,
|
||||
message, id string,
|
||||
classname, funcname, filename string,
|
||||
line int,
|
||||
behavior string,
|
||||
hit bool,
|
||||
) {
|
||||
loc := &locationInfo{classname, funcname, filename, line, columnUnknown}
|
||||
guidanceFn := behavior_to_guidance(behavior)
|
||||
booleanGuidanceImpl(named_bools, message, id, loc, guidanceFn, hit)
|
||||
}
|
||||
|
||||
func add_numeric_details[T Number](details map[string]any, left, right T) map[string]any {
|
||||
// ----------------------------------------------------
|
||||
// Can not use maps.Clone() until go 1.21.0 or above
|
||||
// enhancedDetails := maps.Clone(details)
|
||||
// ----------------------------------------------------
|
||||
enhancedDetails := map[string]any{}
|
||||
for k, v := range details {
|
||||
enhancedDetails[k] = v
|
||||
}
|
||||
enhancedDetails["left"] = left
|
||||
enhancedDetails["right"] = right
|
||||
return enhancedDetails
|
||||
}
|
||||
|
||||
func add_boolean_details(details map[string]any, named_bools []NamedBool) map[string]any {
|
||||
// ----------------------------------------------------
|
||||
// Can not use maps.Clone() until go 1.21.0 or above
|
||||
// enhancedDetails := maps.Clone(details)
|
||||
// ----------------------------------------------------
|
||||
enhancedDetails := map[string]any{}
|
||||
for k, v := range details {
|
||||
enhancedDetails[k] = v
|
||||
}
|
||||
for _, named_bool := range named_bools {
|
||||
enhancedDetails[named_bool.First] = named_bool.Second
|
||||
}
|
||||
return enhancedDetails
|
||||
}
|
||||
|
||||
// Equivalent to asserting Always(left > right, message, details). Information about left and right will automatically be added to the details parameter, with keys left and right. If you use this function for assertions that compare numeric quantities, you may help Antithesis find more bugs.
|
||||
func AlwaysGreaterThan[T Number](left, right T, message string, details map[string]any) {
|
||||
loc := newLocationInfo(offsetAPICaller)
|
||||
id := makeKey(message, loc)
|
||||
condition := left > right
|
||||
all_details := add_numeric_details(details, left, right)
|
||||
assertImpl(condition, message, all_details, loc, wasHit, mustBeHit, universalTest, alwaysDisplay, id)
|
||||
|
||||
numericGuidanceImpl(left, right, message, id, loc, guidanceFnMinimize, wasHit)
|
||||
}
|
||||
|
||||
// Equivalent to asserting Always(left >= right, message, details). Information about left and right will automatically be added to the details parameter, with keys left and right. If you use this function for assertions that compare numeric quantities, you may help Antithesis find more bugs.
|
||||
func AlwaysGreaterThanOrEqualTo[T Number](left, right T, message string, details map[string]any) {
|
||||
loc := newLocationInfo(offsetAPICaller)
|
||||
id := makeKey(message, loc)
|
||||
condition := left >= right
|
||||
all_details := add_numeric_details(details, left, right)
|
||||
assertImpl(condition, message, all_details, loc, wasHit, mustBeHit, universalTest, alwaysDisplay, id)
|
||||
|
||||
numericGuidanceImpl(left, right, message, id, loc, guidanceFnMinimize, wasHit)
|
||||
}
|
||||
|
||||
// Equivalent to asserting Sometimes(T left > T right, message, details). Information about left and right will automatically be added to the details parameter, with keys left and right. If you use this function for assertions that compare numeric quantities, you may help Antithesis find more bugs.
|
||||
func SometimesGreaterThan[T Number](left, right T, message string, details map[string]any) {
|
||||
loc := newLocationInfo(offsetAPICaller)
|
||||
id := makeKey(message, loc)
|
||||
condition := left > right
|
||||
all_details := add_numeric_details(details, left, right)
|
||||
assertImpl(condition, message, all_details, loc, wasHit, mustBeHit, existentialTest, sometimesDisplay, id)
|
||||
|
||||
numericGuidanceImpl(left, right, message, id, loc, guidanceFnMaximize, wasHit)
|
||||
}
|
||||
|
||||
// Equivalent to asserting Sometimes(T left >= T right, message, details). Information about left and right will automatically be added to the details parameter, with keys left and right. If you use this function for assertions that compare numeric quantities, you may help Antithesis find more bugs.
|
||||
func SometimesGreaterThanOrEqualTo[T Number](left, right T, message string, details map[string]any) {
|
||||
loc := newLocationInfo(offsetAPICaller)
|
||||
id := makeKey(message, loc)
|
||||
condition := left >= right
|
||||
all_details := add_numeric_details(details, left, right)
|
||||
assertImpl(condition, message, all_details, loc, wasHit, mustBeHit, existentialTest, sometimesDisplay, id)
|
||||
|
||||
numericGuidanceImpl(left, right, message, id, loc, guidanceFnMaximize, wasHit)
|
||||
}
|
||||
|
||||
// Equivalent to asserting Always(left < right, message, details). Information about left and right will automatically be added to the details parameter, with keys left and right. If you use this function for assertions that compare numeric quantities, you may help Antithesis find more bugs.
|
||||
func AlwaysLessThan[T Number](left, right T, message string, details map[string]any) {
|
||||
loc := newLocationInfo(offsetAPICaller)
|
||||
id := makeKey(message, loc)
|
||||
condition := left < right
|
||||
all_details := add_numeric_details(details, left, right)
|
||||
assertImpl(condition, message, all_details, loc, wasHit, mustBeHit, universalTest, alwaysDisplay, id)
|
||||
|
||||
numericGuidanceImpl(left, right, message, id, loc, guidanceFnMaximize, wasHit)
|
||||
}
|
||||
|
||||
// Equivalent to asserting Always(left <= right, message, details). Information about left and right will automatically be added to the details parameter, with keys left and right. If you use this function for assertions that compare numeric quantities, you may help Antithesis find more bugs.
|
||||
func AlwaysLessThanOrEqualTo[T Number](left, right T, message string, details map[string]any) {
|
||||
loc := newLocationInfo(offsetAPICaller)
|
||||
id := makeKey(message, loc)
|
||||
condition := left <= right
|
||||
all_details := add_numeric_details(details, left, right)
|
||||
assertImpl(condition, message, all_details, loc, wasHit, mustBeHit, universalTest, alwaysDisplay, id)
|
||||
|
||||
numericGuidanceImpl(left, right, message, id, loc, guidanceFnMaximize, wasHit)
|
||||
}
|
||||
|
||||
// Equivalent to asserting Sometimes(T left < T right, message, details). Information about left and right will automatically be added to the details parameter, with keys left and right. If you use this function for assertions that compare numeric quantities, you may help Antithesis find more bugs.
|
||||
func SometimesLessThan[T Number](left, right T, message string, details map[string]any) {
|
||||
loc := newLocationInfo(offsetAPICaller)
|
||||
id := makeKey(message, loc)
|
||||
condition := left < right
|
||||
all_details := add_numeric_details(details, left, right)
|
||||
assertImpl(condition, message, all_details, loc, wasHit, mustBeHit, existentialTest, sometimesDisplay, id)
|
||||
|
||||
numericGuidanceImpl(left, right, message, id, loc, guidanceFnMinimize, wasHit)
|
||||
}
|
||||
|
||||
// Equivalent to asserting Sometimes(T left <= T right, message, details). Information about left and right will automatically be added to the details parameter, with keys left and right. If you use this function for assertions that compare numeric quantities, you may help Antithesis find more bugs.
|
||||
func SometimesLessThanOrEqualTo[T Number](left, right T, message string, details map[string]any) {
|
||||
loc := newLocationInfo(offsetAPICaller)
|
||||
id := makeKey(message, loc)
|
||||
condition := left <= right
|
||||
all_details := add_numeric_details(details, left, right)
|
||||
assertImpl(condition, message, all_details, loc, wasHit, mustBeHit, existentialTest, sometimesDisplay, id)
|
||||
|
||||
numericGuidanceImpl(left, right, message, id, loc, guidanceFnMinimize, wasHit)
|
||||
}
|
||||
|
||||
// Asserts that every time this is called, at least one bool in named_bools is true. Equivalent to Always(named_bools[0].second || named_bools[1].second || ..., message, details). If you use this for assertions about the behavior of booleans, you may help Antithesis find more bugs. Information about named_bools will automatically be added to the details parameter, and the keys will be the names of the bools.
|
||||
func AlwaysSome(named_bools []NamedBool, message string, details map[string]any) {
|
||||
loc := newLocationInfo(offsetAPICaller)
|
||||
id := makeKey(message, loc)
|
||||
disjunction := false
|
||||
for _, named_bool := range named_bools {
|
||||
if named_bool.Second {
|
||||
disjunction = true
|
||||
break
|
||||
}
|
||||
}
|
||||
all_details := add_boolean_details(details, named_bools)
|
||||
assertImpl(disjunction, message, all_details, loc, wasHit, mustBeHit, universalTest, alwaysDisplay, id)
|
||||
|
||||
booleanGuidanceImpl(named_bools, message, id, loc, guidanceFnWantNone, wasHit)
|
||||
}
|
||||
|
||||
// Asserts that at least one time this is called, every bool in named_bools is true. Equivalent to Sometimes(named_bools[0].second && named_bools[1].second && ..., message, details). If you use this for assertions about the behavior of booleans, you may help Antithesis find more bugs. Information about named_bools will automatically be added to the details parameter, and the keys will be the names of the bools.
|
||||
func SometimesAll(named_bools []NamedBool, message string, details map[string]any) {
|
||||
loc := newLocationInfo(offsetAPICaller)
|
||||
id := makeKey(message, loc)
|
||||
conjunction := true
|
||||
for _, named_bool := range named_bools {
|
||||
if !named_bool.Second {
|
||||
conjunction = false
|
||||
break
|
||||
}
|
||||
}
|
||||
all_details := add_boolean_details(details, named_bools)
|
||||
assertImpl(conjunction, message, all_details, loc, wasHit, mustBeHit, existentialTest, sometimesDisplay, id)
|
||||
|
||||
booleanGuidanceImpl(named_bools, message, id, loc, guidanceFnWantAll, wasHit)
|
||||
}
|
||||
Generated
Vendored
-34
@@ -1,34 +0,0 @@
|
||||
//go:build !enable_antithesis_sdk
|
||||
|
||||
package assert
|
||||
|
||||
func AlwaysGreaterThan[T Number](left, right T, message string, details map[string]any) {}
|
||||
func AlwaysGreaterThanOrEqualTo[T Number](left, right T, message string, details map[string]any) {}
|
||||
func SometimesGreaterThan[T Number](left, right T, message string, details map[string]any) {}
|
||||
func SometimesGreaterThanOrEqualTo[T Number](left, right T, message string, details map[string]any) {}
|
||||
func AlwaysLessThan[T Number](left, right T, message string, details map[string]any) {}
|
||||
func AlwaysLessThanOrEqualTo[T Number](left, right T, message string, details map[string]any) {}
|
||||
func SometimesLessThan[T Number](left, right T, message string, details map[string]any) {}
|
||||
func SometimesLessThanOrEqualTo[T Number](left, right T, message string, details map[string]any) {}
|
||||
|
||||
func AlwaysSome(named_bool []NamedBool, message string, details map[string]any) {}
|
||||
func SometimesAll(named_bool []NamedBool, message string, details map[string]any) {}
|
||||
|
||||
func NumericGuidanceRaw[T Number](left, right T,
|
||||
message, id string,
|
||||
classname, funcname, filename string,
|
||||
line int,
|
||||
behavior string,
|
||||
hit bool,
|
||||
) {
|
||||
}
|
||||
|
||||
func BooleanGuidanceRaw(
|
||||
named_bools []NamedBool,
|
||||
message, id string,
|
||||
classname, funcname, filename string,
|
||||
line int,
|
||||
behavior string,
|
||||
hit bool,
|
||||
) {
|
||||
}
|
||||
Generated
Vendored
-111
@@ -1,111 +0,0 @@
|
||||
//go:build enable_antithesis_sdk
|
||||
|
||||
package assert
|
||||
|
||||
import (
|
||||
"runtime"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
|
||||
"github.com/antithesishq/antithesis-sdk-go/internal"
|
||||
)
|
||||
|
||||
type trackerInfo struct {
|
||||
Filename string
|
||||
Classname string
|
||||
PassCount int
|
||||
FailCount int
|
||||
}
|
||||
|
||||
type emitTracker map[string]*trackerInfo
|
||||
|
||||
// assert_tracker (global) keeps track of the unique asserts evaluated
|
||||
var (
|
||||
assertTracker emitTracker = make(emitTracker)
|
||||
trackerMutex sync.Mutex
|
||||
trackerInfoMutex sync.Mutex
|
||||
)
|
||||
|
||||
func (tracker emitTracker) getTrackerEntry(messageKey string, filename, classname string) *trackerInfo {
|
||||
var trackerEntry *trackerInfo
|
||||
var ok bool
|
||||
|
||||
if tracker == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
trackerMutex.Lock()
|
||||
defer trackerMutex.Unlock()
|
||||
if trackerEntry, ok = tracker[messageKey]; !ok {
|
||||
trackerEntry = newTrackerInfo(filename, classname)
|
||||
tracker[messageKey] = trackerEntry
|
||||
}
|
||||
return trackerEntry
|
||||
}
|
||||
|
||||
func newTrackerInfo(filename, classname string) *trackerInfo {
|
||||
trackerInfo := trackerInfo{
|
||||
PassCount: 0,
|
||||
FailCount: 0,
|
||||
Filename: filename,
|
||||
Classname: classname,
|
||||
}
|
||||
return &trackerInfo
|
||||
}
|
||||
|
||||
func (ti *trackerInfo) emit(ai *assertInfo) {
|
||||
if ti == nil || ai == nil {
|
||||
return
|
||||
}
|
||||
|
||||
// Registrations are just sent to voidstar
|
||||
hit := ai.Hit
|
||||
if !hit {
|
||||
emitAssert(ai)
|
||||
return
|
||||
}
|
||||
|
||||
var err error
|
||||
cond := ai.Condition
|
||||
|
||||
trackerInfoMutex.Lock()
|
||||
defer trackerInfoMutex.Unlock()
|
||||
if cond {
|
||||
if ti.PassCount == 0 {
|
||||
err = emitAssert(ai)
|
||||
}
|
||||
if err == nil {
|
||||
ti.PassCount++
|
||||
}
|
||||
return
|
||||
}
|
||||
if ti.FailCount == 0 {
|
||||
err = emitAssert(ai)
|
||||
}
|
||||
if err == nil {
|
||||
ti.FailCount++
|
||||
}
|
||||
}
|
||||
|
||||
func versionMessage() {
|
||||
languageBlock := map[string]any{
|
||||
"name": "Go",
|
||||
"version": runtime.Version(),
|
||||
}
|
||||
versionBlock := map[string]any{
|
||||
"language": languageBlock,
|
||||
"sdk_version": internal.SDK_Version,
|
||||
"protocol_version": internal.Protocol_Version,
|
||||
}
|
||||
internal.Json_data(map[string]any{"antithesis_sdk": versionBlock})
|
||||
}
|
||||
|
||||
// package-level flag
|
||||
var hasEmitted atomic.Bool // initialzed to false
|
||||
|
||||
func emitAssert(ai *assertInfo) error {
|
||||
if hasEmitted.CompareAndSwap(false, true) {
|
||||
versionMessage()
|
||||
}
|
||||
return internal.Json_data(wrappedAssertInfo{ai})
|
||||
}
|
||||
Generated
Vendored
-100
@@ -1,100 +0,0 @@
|
||||
//go:build enable_antithesis_sdk
|
||||
|
||||
package internal
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"log"
|
||||
"math/rand"
|
||||
"os"
|
||||
)
|
||||
|
||||
func Json_data(v any) error {
|
||||
if data, err := json.Marshal(v); err != nil {
|
||||
return err
|
||||
} else {
|
||||
handler.output(string(data))
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
func Get_random() uint64 {
|
||||
return handler.random()
|
||||
}
|
||||
|
||||
func Notify(edge uint64) bool {
|
||||
return handler.notify(edge)
|
||||
}
|
||||
|
||||
func InitCoverage(num_edges uint64, symbols string) uint64 {
|
||||
return handler.init_coverage(num_edges, symbols)
|
||||
}
|
||||
|
||||
type libHandler interface {
|
||||
output(message string)
|
||||
random() uint64
|
||||
notify(edge uint64) bool
|
||||
init_coverage(num_edges uint64, symbols string) uint64
|
||||
}
|
||||
|
||||
const (
|
||||
errorLogLinePrefix = "[* antithesis-sdk-go *]"
|
||||
)
|
||||
|
||||
var handler libHandler
|
||||
|
||||
type localHandler struct {
|
||||
outputFile *os.File // can be nil
|
||||
}
|
||||
|
||||
func (h *localHandler) output(message string) {
|
||||
msg_len := len(message)
|
||||
if msg_len == 0 {
|
||||
return
|
||||
}
|
||||
if h.outputFile != nil {
|
||||
h.outputFile.WriteString(message + "\n")
|
||||
}
|
||||
}
|
||||
|
||||
func (h *localHandler) random() uint64 {
|
||||
return rand.Uint64()
|
||||
}
|
||||
|
||||
func (h *localHandler) notify(edge uint64) bool {
|
||||
return false
|
||||
}
|
||||
|
||||
func (h *localHandler) init_coverage(num_edges uint64, symbols string) uint64 {
|
||||
return 0
|
||||
}
|
||||
|
||||
func init() {
|
||||
handler = init_in_antithesis()
|
||||
if handler == nil {
|
||||
// Otherwise fallback to the local handler.
|
||||
handler = openLocalHandler()
|
||||
}
|
||||
}
|
||||
|
||||
// If `localOutputEnvVar` is set to a non-empty path, attempt to open that path and truncate the file
|
||||
// to serve as the log file of the local handler.
|
||||
// Otherwise, we don't have a log file, and logging is a no-op in the local handler.
|
||||
func openLocalHandler() *localHandler {
|
||||
path, is_set := os.LookupEnv(localOutputEnvVar)
|
||||
if !is_set || len(path) == 0 {
|
||||
return &localHandler{nil}
|
||||
}
|
||||
|
||||
// Open the file R/W (create if needed and possible)
|
||||
file, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE, 0644)
|
||||
if err != nil {
|
||||
log.Printf("%s Failed to open path %s: %v", errorLogLinePrefix, path, err)
|
||||
file = nil
|
||||
} else if err = file.Truncate(0); err != nil {
|
||||
log.Printf("%s Failed to truncate file at %s: %v", errorLogLinePrefix, path, err)
|
||||
file = nil
|
||||
}
|
||||
|
||||
return &localHandler{file}
|
||||
}
|
||||
Generated
Vendored
-12
@@ -1,12 +0,0 @@
|
||||
package internal
|
||||
|
||||
// --------------------------------------------------------------------------------
|
||||
// Versions
|
||||
// --------------------------------------------------------------------------------
|
||||
const SDK_Version = "0.7.0"
|
||||
const Protocol_Version = "1.1.0"
|
||||
|
||||
// --------------------------------------------------------------------------------
|
||||
// Environment Vars
|
||||
// --------------------------------------------------------------------------------
|
||||
const localOutputEnvVar = "ANTITHESIS_SDK_LOCAL_OUTPUT"
|
||||
Generated
Vendored
-160
@@ -1,160 +0,0 @@
|
||||
//go:build enable_antithesis_sdk && linux && amd64 && cgo
|
||||
|
||||
package internal
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"unsafe"
|
||||
"os"
|
||||
)
|
||||
|
||||
// --------------------------------------------------------------------------------
|
||||
// To build and run an executable with this package
|
||||
//
|
||||
// CC=clang CGO_ENABLED=1 go run ./main.go
|
||||
// --------------------------------------------------------------------------------
|
||||
|
||||
// \/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/
|
||||
//
|
||||
// The commented lines below, and the `import "C"` line which must directly follow
|
||||
// the commented lines are used by CGO. They are load-bearing, and should not be
|
||||
// changed without first understanding how CGO uses them.
|
||||
//
|
||||
// \/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/
|
||||
|
||||
// #cgo LDFLAGS: -ldl
|
||||
//
|
||||
// #include <dlfcn.h>
|
||||
// #include <stdbool.h>
|
||||
// #include <stdint.h>
|
||||
// #include <stdlib.h>
|
||||
//
|
||||
// typedef void (*go_fuzz_json_data_fn)(const char *data, size_t size);
|
||||
// void
|
||||
// go_fuzz_json_data(void *f, const char *data, size_t size) {
|
||||
// ((go_fuzz_json_data_fn)f)(data, size);
|
||||
// }
|
||||
//
|
||||
// typedef void (*go_fuzz_flush_fn)(void);
|
||||
// void
|
||||
// go_fuzz_flush(void *f) {
|
||||
// ((go_fuzz_flush_fn)f)();
|
||||
// }
|
||||
//
|
||||
// typedef uint64_t (*go_fuzz_get_random_fn)(void);
|
||||
// uint64_t
|
||||
// go_fuzz_get_random(void *f) {
|
||||
// return ((go_fuzz_get_random_fn)f)();
|
||||
// }
|
||||
//
|
||||
// typedef bool (*go_notify_coverage_fn)(size_t);
|
||||
// int
|
||||
// go_notify_coverage(void *f, size_t edges) {
|
||||
// bool b = ((go_notify_coverage_fn)f)(edges);
|
||||
// return b ? 1 : 0;
|
||||
// }
|
||||
//
|
||||
// typedef uint64_t (*go_init_coverage_fn)(size_t num_edges, const char *symbols);
|
||||
// uint64_t
|
||||
// go_init_coverage(void *f, size_t num_edges, const char *symbols) {
|
||||
// return ((go_init_coverage_fn)f)(num_edges, symbols);
|
||||
// }
|
||||
//
|
||||
import "C"
|
||||
|
||||
const (
|
||||
defaultNativeLibraryPath = "/usr/lib/libvoidstar.so"
|
||||
)
|
||||
|
||||
type voidstarHandler struct {
|
||||
fuzzJsonData unsafe.Pointer
|
||||
fuzzFlush unsafe.Pointer
|
||||
fuzzGetRandom unsafe.Pointer
|
||||
initCoverage unsafe.Pointer
|
||||
notifyCoverage unsafe.Pointer
|
||||
}
|
||||
|
||||
func (h *voidstarHandler) output(message string) {
|
||||
msg_len := len(message)
|
||||
if msg_len == 0 {
|
||||
return
|
||||
}
|
||||
cstrMessage := C.CString(message)
|
||||
defer C.free(unsafe.Pointer(cstrMessage))
|
||||
C.go_fuzz_json_data(h.fuzzJsonData, cstrMessage, C.ulong(msg_len))
|
||||
C.go_fuzz_flush(h.fuzzFlush)
|
||||
}
|
||||
|
||||
func (h *voidstarHandler) random() uint64 {
|
||||
return uint64(C.go_fuzz_get_random(h.fuzzGetRandom))
|
||||
}
|
||||
|
||||
func (h *voidstarHandler) init_coverage(num_edge uint64, symbols string) uint64 {
|
||||
cstrSymbols := C.CString(symbols)
|
||||
defer C.free(unsafe.Pointer(cstrSymbols))
|
||||
return uint64(C.go_init_coverage(h.initCoverage, C.ulong(num_edge), cstrSymbols))
|
||||
}
|
||||
|
||||
func (h *voidstarHandler) notify(edge uint64) bool {
|
||||
ival := int(C.go_notify_coverage(h.notifyCoverage, C.ulong(edge)))
|
||||
return ival == 1
|
||||
}
|
||||
|
||||
// Attempt to load libvoidstar and some symbols from `path`
|
||||
func openSharedLib(path string) (*voidstarHandler, error) {
|
||||
cstrPath := C.CString(path)
|
||||
defer C.free(unsafe.Pointer(cstrPath))
|
||||
|
||||
dlError := func(message string) error {
|
||||
return fmt.Errorf("%s: (%s)", message, C.GoString(C.dlerror()))
|
||||
}
|
||||
|
||||
sharedLib := C.dlopen(cstrPath, C.int(C.RTLD_NOW))
|
||||
if sharedLib == nil {
|
||||
return nil, dlError("Can not load the Antithesis native library")
|
||||
}
|
||||
|
||||
loadFunc := func(name string) (symbol unsafe.Pointer, err error) {
|
||||
cstrName := C.CString(name)
|
||||
defer C.free(unsafe.Pointer(cstrName))
|
||||
if symbol = C.dlsym(sharedLib, cstrName); symbol == nil {
|
||||
err = dlError(fmt.Sprintf("Can not access symbol %s", name))
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
fuzzJsonData, err := loadFunc("fuzz_json_data")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
fuzzFlush, err := loadFunc("fuzz_flush")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
fuzzGetRandom, err := loadFunc("fuzz_get_random")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
notifyCoverage, err := loadFunc("notify_coverage")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
initCoverage, err := loadFunc("init_coverage_module")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &voidstarHandler{fuzzJsonData, fuzzFlush, fuzzGetRandom, initCoverage, notifyCoverage}, nil
|
||||
}
|
||||
|
||||
// If we have a file at `defaultNativeLibraryPath`, we load the shared library
|
||||
// (and panic on any error encountered during load).
|
||||
func init_in_antithesis() libHandler {
|
||||
if _, err := os.Stat(defaultNativeLibraryPath); err == nil {
|
||||
handler, err := openSharedLib(defaultNativeLibraryPath)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return handler
|
||||
}
|
||||
return nil
|
||||
}
|
||||
Generated
Vendored
-7
@@ -1,7 +0,0 @@
|
||||
//go:build enable_antithesis_sdk && (!linux || !amd64 || !cgo)
|
||||
|
||||
package internal
|
||||
|
||||
func init_in_antithesis() libHandler {
|
||||
return nil
|
||||
}
|
||||
-202
@@ -1,202 +0,0 @@
|
||||
|
||||
Apache License
|
||||
Version 2.0, January 2004
|
||||
http://www.apache.org/licenses/
|
||||
|
||||
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
||||
|
||||
1. Definitions.
|
||||
|
||||
"License" shall mean the terms and conditions for use, reproduction,
|
||||
and distribution as defined by Sections 1 through 9 of this document.
|
||||
|
||||
"Licensor" shall mean the copyright owner or entity authorized by
|
||||
the copyright owner that is granting the License.
|
||||
|
||||
"Legal Entity" shall mean the union of the acting entity and all
|
||||
other entities that control, are controlled by, or are under common
|
||||
control with that entity. For the purposes of this definition,
|
||||
"control" means (i) the power, direct or indirect, to cause the
|
||||
direction or management of such entity, whether by contract or
|
||||
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
||||
outstanding shares, or (iii) beneficial ownership of such entity.
|
||||
|
||||
"You" (or "Your") shall mean an individual or Legal Entity
|
||||
exercising permissions granted by this License.
|
||||
|
||||
"Source" form shall mean the preferred form for making modifications,
|
||||
including but not limited to software source code, documentation
|
||||
source, and configuration files.
|
||||
|
||||
"Object" form shall mean any form resulting from mechanical
|
||||
transformation or translation of a Source form, including but
|
||||
not limited to compiled object code, generated documentation,
|
||||
and conversions to other media types.
|
||||
|
||||
"Work" shall mean the work of authorship, whether in Source or
|
||||
Object form, made available under the License, as indicated by a
|
||||
copyright notice that is included in or attached to the work
|
||||
(an example is provided in the Appendix below).
|
||||
|
||||
"Derivative Works" shall mean any work, whether in Source or Object
|
||||
form, that is based on (or derived from) the Work and for which the
|
||||
editorial revisions, annotations, elaborations, or other modifications
|
||||
represent, as a whole, an original work of authorship. For the purposes
|
||||
of this License, Derivative Works shall not include works that remain
|
||||
separable from, or merely link (or bind by name) to the interfaces of,
|
||||
the Work and Derivative Works thereof.
|
||||
|
||||
"Contribution" shall mean any work of authorship, including
|
||||
the original version of the Work and any modifications or additions
|
||||
to that Work or Derivative Works thereof, that is intentionally
|
||||
submitted to Licensor for inclusion in the Work by the copyright owner
|
||||
or by an individual or Legal Entity authorized to submit on behalf of
|
||||
the copyright owner. For the purposes of this definition, "submitted"
|
||||
means any form of electronic, verbal, or written communication sent
|
||||
to the Licensor or its representatives, including but not limited to
|
||||
communication on electronic mailing lists, source code control systems,
|
||||
and issue tracking systems that are managed by, or on behalf of, the
|
||||
Licensor for the purpose of discussing and improving the Work, but
|
||||
excluding communication that is conspicuously marked or otherwise
|
||||
designated in writing by the copyright owner as "Not a Contribution."
|
||||
|
||||
"Contributor" shall mean Licensor and any individual or Legal Entity
|
||||
on behalf of whom a Contribution has been received by Licensor and
|
||||
subsequently incorporated within the Work.
|
||||
|
||||
2. Grant of Copyright License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
copyright license to reproduce, prepare Derivative Works of,
|
||||
publicly display, publicly perform, sublicense, and distribute the
|
||||
Work and such Derivative Works in Source or Object form.
|
||||
|
||||
3. Grant of Patent License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
(except as stated in this section) patent license to make, have made,
|
||||
use, offer to sell, sell, import, and otherwise transfer the Work,
|
||||
where such license applies only to those patent claims licensable
|
||||
by such Contributor that are necessarily infringed by their
|
||||
Contribution(s) alone or by combination of their Contribution(s)
|
||||
with the Work to which such Contribution(s) was submitted. If You
|
||||
institute patent litigation against any entity (including a
|
||||
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
||||
or a Contribution incorporated within the Work constitutes direct
|
||||
or contributory patent infringement, then any patent licenses
|
||||
granted to You under this License for that Work shall terminate
|
||||
as of the date such litigation is filed.
|
||||
|
||||
4. Redistribution. You may reproduce and distribute copies of the
|
||||
Work or Derivative Works thereof in any medium, with or without
|
||||
modifications, and in Source or Object form, provided that You
|
||||
meet the following conditions:
|
||||
|
||||
(a) You must give any other recipients of the Work or
|
||||
Derivative Works a copy of this License; and
|
||||
|
||||
(b) You must cause any modified files to carry prominent notices
|
||||
stating that You changed the files; and
|
||||
|
||||
(c) You must retain, in the Source form of any Derivative Works
|
||||
that You distribute, all copyright, patent, trademark, and
|
||||
attribution notices from the Source form of the Work,
|
||||
excluding those notices that do not pertain to any part of
|
||||
the Derivative Works; and
|
||||
|
||||
(d) If the Work includes a "NOTICE" text file as part of its
|
||||
distribution, then any Derivative Works that You distribute must
|
||||
include a readable copy of the attribution notices contained
|
||||
within such NOTICE file, excluding those notices that do not
|
||||
pertain to any part of the Derivative Works, in at least one
|
||||
of the following places: within a NOTICE text file distributed
|
||||
as part of the Derivative Works; within the Source form or
|
||||
documentation, if provided along with the Derivative Works; or,
|
||||
within a display generated by the Derivative Works, if and
|
||||
wherever such third-party notices normally appear. The contents
|
||||
of the NOTICE file are for informational purposes only and
|
||||
do not modify the License. You may add Your own attribution
|
||||
notices within Derivative Works that You distribute, alongside
|
||||
or as an addendum to the NOTICE text from the Work, provided
|
||||
that such additional attribution notices cannot be construed
|
||||
as modifying the License.
|
||||
|
||||
You may add Your own copyright statement to Your modifications and
|
||||
may provide additional or different license terms and conditions
|
||||
for use, reproduction, or distribution of Your modifications, or
|
||||
for any such Derivative Works as a whole, provided Your use,
|
||||
reproduction, and distribution of the Work otherwise complies with
|
||||
the conditions stated in this License.
|
||||
|
||||
5. Submission of Contributions. Unless You explicitly state otherwise,
|
||||
any Contribution intentionally submitted for inclusion in the Work
|
||||
by You to the Licensor shall be under the terms and conditions of
|
||||
this License, without any additional terms or conditions.
|
||||
Notwithstanding the above, nothing herein shall supersede or modify
|
||||
the terms of any separate license agreement you may have executed
|
||||
with Licensor regarding such Contributions.
|
||||
|
||||
6. Trademarks. This License does not grant permission to use the trade
|
||||
names, trademarks, service marks, or product names of the Licensor,
|
||||
except as required for reasonable and customary use in describing the
|
||||
origin of the Work and reproducing the content of the NOTICE file.
|
||||
|
||||
7. Disclaimer of Warranty. Unless required by applicable law or
|
||||
agreed to in writing, Licensor provides the Work (and each
|
||||
Contributor provides its Contributions) on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
implied, including, without limitation, any warranties or conditions
|
||||
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
||||
PARTICULAR PURPOSE. You are solely responsible for determining the
|
||||
appropriateness of using or redistributing the Work and assume any
|
||||
risks associated with Your exercise of permissions under this License.
|
||||
|
||||
8. Limitation of Liability. In no event and under no legal theory,
|
||||
whether in tort (including negligence), contract, or otherwise,
|
||||
unless required by applicable law (such as deliberate and grossly
|
||||
negligent acts) or agreed to in writing, shall any Contributor be
|
||||
liable to You for damages, including any direct, indirect, special,
|
||||
incidental, or consequential damages of any character arising as a
|
||||
result of this License or out of the use or inability to use the
|
||||
Work (including but not limited to damages for loss of goodwill,
|
||||
work stoppage, computer failure or malfunction, or any and all
|
||||
other commercial damages or losses), even if such Contributor
|
||||
has been advised of the possibility of such damages.
|
||||
|
||||
9. Accepting Warranty or Additional Liability. While redistributing
|
||||
the Work or Derivative Works thereof, You may choose to offer,
|
||||
and charge a fee for, acceptance of support, warranty, indemnity,
|
||||
or other liability obligations and/or rights consistent with this
|
||||
License. However, in accepting such obligations, You may act only
|
||||
on Your own behalf and on Your sole responsibility, not on behalf
|
||||
of any other Contributor, and only if You agree to indemnify,
|
||||
defend, and hold each Contributor harmless for any liability
|
||||
incurred by, or claims asserted against, such Contributor by reason
|
||||
of your accepting any such warranty or additional liability.
|
||||
|
||||
END OF TERMS AND CONDITIONS
|
||||
|
||||
APPENDIX: How to apply the Apache License to your work.
|
||||
|
||||
To apply the Apache License to your work, attach the following
|
||||
boilerplate notice, with the fields enclosed by brackets "[]"
|
||||
replaced with your own identifying information. (Don't include
|
||||
the brackets!) The text should be enclosed in the appropriate
|
||||
comment syntax for the file format. We also recommend that a
|
||||
file or class name and description of purpose be included on the
|
||||
same "printed page" as the copyright notice for easier
|
||||
identification within third-party archives.
|
||||
|
||||
Copyright [yyyy] [name of copyright owner]
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
-35
@@ -1,35 +0,0 @@
|
||||
# TPM 2.0 client library
|
||||
|
||||
## Tests
|
||||
|
||||
This library contains unit tests in `github.com/google/go-tpm/tpm2`, which just
|
||||
tests that various encoding and error checking functions work correctly. It also
|
||||
contains more comprehensive integration tests in
|
||||
`github.com/google/go-tpm/tpm2/test`, which run actual commands on a TPM.
|
||||
|
||||
By default, these integration tests are run against the
|
||||
[`go-tpm-tools`](https://github.com/google/go-tpm-tools)
|
||||
simulator, which is baesed on the
|
||||
[Microsoft Reference TPM2 code](https://github.com/microsoft/ms-tpm-20-ref). To
|
||||
run both the unit and integration tests, run (in this directory)
|
||||
```bash
|
||||
go test . ./test
|
||||
```
|
||||
|
||||
These integration tests can also be run against a real TPM device. This is
|
||||
slightly more complex as the tests often need to be built as a normal user and
|
||||
then executed as root. For example,
|
||||
```bash
|
||||
# Build the test binary without running it
|
||||
go test -c github.com/google/go-tpm/tpm2/test
|
||||
# Execute the test binary as root
|
||||
sudo ./test.test --tpm-path=/dev/tpmrm0
|
||||
```
|
||||
On Linux, The `--tpm-path` causes the integration tests to be run against a
|
||||
real TPM located at that path (usually `/dev/tpmrm0` or `/dev/tpm0`). On Windows, the story is similar, execept that
|
||||
the `--use-tbs` flag is used instead.
|
||||
|
||||
Tip: if your TPM host is remote and you don't want to install Go on it, this
|
||||
same two-step process can be used. The test binary can be copied to a remote
|
||||
host and run without extra installation (as the test binary has very few
|
||||
*runtime* dependancies).
|
||||
Generated
Vendored
-576
@@ -1,576 +0,0 @@
|
||||
// Copyright (c) 2018, Google LLC All rights reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package tpm2
|
||||
|
||||
import (
|
||||
"crypto"
|
||||
"crypto/elliptic"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
// Register the relevant hash implementations to prevent a runtime failure.
|
||||
_ "crypto/sha1"
|
||||
_ "crypto/sha256"
|
||||
_ "crypto/sha512"
|
||||
|
||||
"github.com/google/go-tpm/tpmutil"
|
||||
)
|
||||
|
||||
var hashInfo = []struct {
|
||||
alg Algorithm
|
||||
hash crypto.Hash
|
||||
}{
|
||||
{AlgSHA1, crypto.SHA1},
|
||||
{AlgSHA256, crypto.SHA256},
|
||||
{AlgSHA384, crypto.SHA384},
|
||||
{AlgSHA512, crypto.SHA512},
|
||||
{AlgSHA3_256, crypto.SHA3_256},
|
||||
{AlgSHA3_384, crypto.SHA3_384},
|
||||
{AlgSHA3_512, crypto.SHA3_512},
|
||||
}
|
||||
|
||||
// MAX_DIGEST_BUFFER is the maximum size of []byte request or response fields.
|
||||
// Typically used for chunking of big blobs of data (such as for hashing or
|
||||
// encryption).
|
||||
const maxDigestBuffer = 1024
|
||||
|
||||
// Algorithm represents a TPM_ALG_ID value.
|
||||
type Algorithm uint16
|
||||
|
||||
// HashToAlgorithm looks up the TPM2 algorithm corresponding to the provided crypto.Hash
|
||||
func HashToAlgorithm(hash crypto.Hash) (Algorithm, error) {
|
||||
for _, info := range hashInfo {
|
||||
if info.hash == hash {
|
||||
return info.alg, nil
|
||||
}
|
||||
}
|
||||
return AlgUnknown, fmt.Errorf("go hash algorithm #%d has no TPM2 algorithm", hash)
|
||||
}
|
||||
|
||||
// IsNull returns true if a is AlgNull or zero (unset).
|
||||
func (a Algorithm) IsNull() bool {
|
||||
return a == AlgNull || a == AlgUnknown
|
||||
}
|
||||
|
||||
// UsesCount returns true if a signature algorithm uses count value.
|
||||
func (a Algorithm) UsesCount() bool {
|
||||
return a == AlgECDAA
|
||||
}
|
||||
|
||||
// UsesHash returns true if the algorithm requires the use of a hash.
|
||||
func (a Algorithm) UsesHash() bool {
|
||||
return a == AlgOAEP
|
||||
}
|
||||
|
||||
// Hash returns a crypto.Hash based on the given TPM_ALG_ID.
|
||||
// An error is returned if the given algorithm is not a hash algorithm or is not available.
|
||||
func (a Algorithm) Hash() (crypto.Hash, error) {
|
||||
for _, info := range hashInfo {
|
||||
if info.alg == a {
|
||||
if !info.hash.Available() {
|
||||
return crypto.Hash(0), fmt.Errorf("go hash algorithm #%d not available", info.hash)
|
||||
}
|
||||
return info.hash, nil
|
||||
}
|
||||
}
|
||||
return crypto.Hash(0), fmt.Errorf("hash algorithm not supported: 0x%x", a)
|
||||
}
|
||||
|
||||
func (a Algorithm) String() string {
|
||||
var s strings.Builder
|
||||
var err error
|
||||
switch a {
|
||||
case AlgUnknown:
|
||||
_, err = s.WriteString("AlgUnknown")
|
||||
case AlgRSA:
|
||||
_, err = s.WriteString("RSA")
|
||||
case AlgSHA1:
|
||||
_, err = s.WriteString("SHA1")
|
||||
case AlgHMAC:
|
||||
_, err = s.WriteString("HMAC")
|
||||
case AlgAES:
|
||||
_, err = s.WriteString("AES")
|
||||
case AlgKeyedHash:
|
||||
_, err = s.WriteString("KeyedHash")
|
||||
case AlgXOR:
|
||||
_, err = s.WriteString("XOR")
|
||||
case AlgSHA256:
|
||||
_, err = s.WriteString("SHA256")
|
||||
case AlgSHA384:
|
||||
_, err = s.WriteString("SHA384")
|
||||
case AlgSHA512:
|
||||
_, err = s.WriteString("SHA512")
|
||||
case AlgNull:
|
||||
_, err = s.WriteString("AlgNull")
|
||||
case AlgRSASSA:
|
||||
_, err = s.WriteString("RSASSA")
|
||||
case AlgRSAES:
|
||||
_, err = s.WriteString("RSAES")
|
||||
case AlgRSAPSS:
|
||||
_, err = s.WriteString("RSAPSS")
|
||||
case AlgOAEP:
|
||||
_, err = s.WriteString("OAEP")
|
||||
case AlgECDSA:
|
||||
_, err = s.WriteString("ECDSA")
|
||||
case AlgECDH:
|
||||
_, err = s.WriteString("ECDH")
|
||||
case AlgECDAA:
|
||||
_, err = s.WriteString("ECDAA")
|
||||
case AlgKDF2:
|
||||
_, err = s.WriteString("KDF2")
|
||||
case AlgECC:
|
||||
_, err = s.WriteString("ECC")
|
||||
case AlgSymCipher:
|
||||
_, err = s.WriteString("SymCipher")
|
||||
case AlgSHA3_256:
|
||||
_, err = s.WriteString("SHA3_256")
|
||||
case AlgSHA3_384:
|
||||
_, err = s.WriteString("SHA3_384")
|
||||
case AlgSHA3_512:
|
||||
_, err = s.WriteString("SHA3_512")
|
||||
case AlgCTR:
|
||||
_, err = s.WriteString("CTR")
|
||||
case AlgOFB:
|
||||
_, err = s.WriteString("OFB")
|
||||
case AlgCBC:
|
||||
_, err = s.WriteString("CBC")
|
||||
case AlgCFB:
|
||||
_, err = s.WriteString("CFB")
|
||||
case AlgECB:
|
||||
_, err = s.WriteString("ECB")
|
||||
default:
|
||||
return fmt.Sprintf("Alg?<%d>", int(a))
|
||||
}
|
||||
if err != nil {
|
||||
return fmt.Sprintf("Writing to string builder failed: %v", err)
|
||||
}
|
||||
return s.String()
|
||||
}
|
||||
|
||||
// Supported Algorithms.
|
||||
const (
|
||||
AlgUnknown Algorithm = 0x0000
|
||||
AlgRSA Algorithm = 0x0001
|
||||
AlgSHA1 Algorithm = 0x0004
|
||||
AlgHMAC Algorithm = 0x0005
|
||||
AlgAES Algorithm = 0x0006
|
||||
AlgKeyedHash Algorithm = 0x0008
|
||||
AlgXOR Algorithm = 0x000A
|
||||
AlgSHA256 Algorithm = 0x000B
|
||||
AlgSHA384 Algorithm = 0x000C
|
||||
AlgSHA512 Algorithm = 0x000D
|
||||
AlgNull Algorithm = 0x0010
|
||||
AlgRSASSA Algorithm = 0x0014
|
||||
AlgRSAES Algorithm = 0x0015
|
||||
AlgRSAPSS Algorithm = 0x0016
|
||||
AlgOAEP Algorithm = 0x0017
|
||||
AlgECDSA Algorithm = 0x0018
|
||||
AlgECDH Algorithm = 0x0019
|
||||
AlgECDAA Algorithm = 0x001A
|
||||
AlgKDF2 Algorithm = 0x0021
|
||||
AlgECC Algorithm = 0x0023
|
||||
AlgSymCipher Algorithm = 0x0025
|
||||
AlgSHA3_256 Algorithm = 0x0027
|
||||
AlgSHA3_384 Algorithm = 0x0028
|
||||
AlgSHA3_512 Algorithm = 0x0029
|
||||
AlgCTR Algorithm = 0x0040
|
||||
AlgOFB Algorithm = 0x0041
|
||||
AlgCBC Algorithm = 0x0042
|
||||
AlgCFB Algorithm = 0x0043
|
||||
AlgECB Algorithm = 0x0044
|
||||
)
|
||||
|
||||
// HandleType defines a type of handle.
|
||||
type HandleType uint8
|
||||
|
||||
// Supported handle types
|
||||
const (
|
||||
HandleTypePCR HandleType = 0x00
|
||||
HandleTypeNVIndex HandleType = 0x01
|
||||
HandleTypeHMACSession HandleType = 0x02
|
||||
HandleTypeLoadedSession HandleType = 0x02
|
||||
HandleTypePolicySession HandleType = 0x03
|
||||
HandleTypeSavedSession HandleType = 0x03
|
||||
HandleTypePermanent HandleType = 0x40
|
||||
HandleTypeTransient HandleType = 0x80
|
||||
HandleTypePersistent HandleType = 0x81
|
||||
)
|
||||
|
||||
// SessionType defines the type of session created in StartAuthSession.
|
||||
type SessionType uint8
|
||||
|
||||
// Supported session types.
|
||||
const (
|
||||
SessionHMAC SessionType = 0x00
|
||||
SessionPolicy SessionType = 0x01
|
||||
SessionTrial SessionType = 0x03
|
||||
)
|
||||
|
||||
// SessionAttributes represents an attribute of a session.
|
||||
type SessionAttributes byte
|
||||
|
||||
// Session Attributes (Structures 8.4 TPMA_SESSION)
|
||||
const (
|
||||
AttrContinueSession SessionAttributes = 1 << iota
|
||||
AttrAuditExclusive
|
||||
AttrAuditReset
|
||||
_ // bit 3 reserved
|
||||
_ // bit 4 reserved
|
||||
AttrDecrypt
|
||||
AttrEcrypt
|
||||
AttrAudit
|
||||
)
|
||||
|
||||
// EmptyAuth represents the empty authorization value.
|
||||
var EmptyAuth []byte
|
||||
|
||||
// KeyProp is a bitmask used in Attributes field of key templates. Individual
|
||||
// flags should be OR-ed to form a full mask.
|
||||
type KeyProp uint32
|
||||
|
||||
// Key properties.
|
||||
const (
|
||||
FlagFixedTPM KeyProp = 0x00000002
|
||||
FlagStClear KeyProp = 0x00000004
|
||||
FlagFixedParent KeyProp = 0x00000010
|
||||
FlagSensitiveDataOrigin KeyProp = 0x00000020
|
||||
FlagUserWithAuth KeyProp = 0x00000040
|
||||
FlagAdminWithPolicy KeyProp = 0x00000080
|
||||
FlagNoDA KeyProp = 0x00000400
|
||||
FlagRestricted KeyProp = 0x00010000
|
||||
FlagDecrypt KeyProp = 0x00020000
|
||||
FlagSign KeyProp = 0x00040000
|
||||
|
||||
FlagSealDefault = FlagFixedTPM | FlagFixedParent
|
||||
FlagSignerDefault = FlagSign | FlagRestricted | FlagFixedTPM |
|
||||
FlagFixedParent | FlagSensitiveDataOrigin | FlagUserWithAuth
|
||||
FlagStorageDefault = FlagDecrypt | FlagRestricted | FlagFixedTPM |
|
||||
FlagFixedParent | FlagSensitiveDataOrigin | FlagUserWithAuth
|
||||
)
|
||||
|
||||
// TPMProp represents a Property Tag (TPM_PT) used with calls to GetCapability(CapabilityTPMProperties).
|
||||
type TPMProp uint32
|
||||
|
||||
// TPM Capability Properties, see TPM 2.0 Spec, Rev 1.38, Table 23.
|
||||
// Fixed TPM Properties (PT_FIXED)
|
||||
const (
|
||||
FamilyIndicator TPMProp = 0x100 + iota
|
||||
SpecLevel
|
||||
SpecRevision
|
||||
SpecDayOfYear
|
||||
SpecYear
|
||||
Manufacturer
|
||||
VendorString1
|
||||
VendorString2
|
||||
VendorString3
|
||||
VendorString4
|
||||
VendorTPMType
|
||||
FirmwareVersion1
|
||||
FirmwareVersion2
|
||||
InputMaxBufferSize
|
||||
TransientObjectsMin
|
||||
PersistentObjectsMin
|
||||
LoadedObjectsMin
|
||||
ActiveSessionsMax
|
||||
PCRCount
|
||||
PCRSelectMin
|
||||
ContextGapMax
|
||||
_ // (PT_FIXED + 21) is skipped
|
||||
NVCountersMax
|
||||
NVIndexMax
|
||||
MemoryMethod
|
||||
ClockUpdate
|
||||
ContextHash
|
||||
ContextSym
|
||||
ContextSymSize
|
||||
OrderlyCount
|
||||
CommandMaxSize
|
||||
ResponseMaxSize
|
||||
DigestMaxSize
|
||||
ObjectContextMaxSize
|
||||
SessionContextMaxSize
|
||||
PSFamilyIndicator
|
||||
PSSpecLevel
|
||||
PSSpecRevision
|
||||
PSSpecDayOfYear
|
||||
PSSpecYear
|
||||
SplitSigningMax
|
||||
TotalCommands
|
||||
LibraryCommands
|
||||
VendorCommands
|
||||
NVMaxBufferSize
|
||||
TPMModes
|
||||
CapabilityMaxBufferSize
|
||||
)
|
||||
|
||||
// Variable TPM Properties (PT_VAR)
|
||||
const (
|
||||
TPMAPermanent TPMProp = 0x200 + iota
|
||||
TPMAStartupClear
|
||||
HRNVIndex
|
||||
HRLoaded
|
||||
HRLoadedAvail
|
||||
HRActive
|
||||
HRActiveAvail
|
||||
HRTransientAvail
|
||||
CurrentPersistent
|
||||
AvailPersistent
|
||||
NVCounters
|
||||
NVCountersAvail
|
||||
AlgorithmSet
|
||||
LoadedCurves
|
||||
LockoutCounter
|
||||
MaxAuthFail
|
||||
LockoutInterval
|
||||
LockoutRecovery
|
||||
NVWriteRecovery
|
||||
AuditCounter0
|
||||
AuditCounter1
|
||||
)
|
||||
|
||||
// Allowed ranges of different kinds of Handles (TPM_HANDLE)
|
||||
// These constants have type TPMProp for backwards compatibility.
|
||||
const (
|
||||
PCRFirst TPMProp = 0x00000000
|
||||
HMACSessionFirst TPMProp = 0x02000000
|
||||
LoadedSessionFirst TPMProp = 0x02000000
|
||||
PolicySessionFirst TPMProp = 0x03000000
|
||||
ActiveSessionFirst TPMProp = 0x03000000
|
||||
TransientFirst TPMProp = 0x80000000
|
||||
PersistentFirst TPMProp = 0x81000000
|
||||
PersistentLast TPMProp = 0x81FFFFFF
|
||||
PlatformPersistent TPMProp = 0x81800000
|
||||
NVIndexFirst TPMProp = 0x01000000
|
||||
NVIndexLast TPMProp = 0x01FFFFFF
|
||||
PermanentFirst TPMProp = 0x40000000
|
||||
PermanentLast TPMProp = 0x4000010F
|
||||
)
|
||||
|
||||
// Reserved Handles.
|
||||
const (
|
||||
HandleOwner tpmutil.Handle = 0x40000001 + iota
|
||||
HandleRevoke
|
||||
HandleTransport
|
||||
HandleOperator
|
||||
HandleAdmin
|
||||
HandleEK
|
||||
HandleNull
|
||||
HandleUnassigned
|
||||
HandlePasswordSession
|
||||
HandleLockout
|
||||
HandleEndorsement
|
||||
HandlePlatform
|
||||
)
|
||||
|
||||
// Capability identifies some TPM property or state type.
|
||||
type Capability uint32
|
||||
|
||||
// TPM Capabilities.
|
||||
const (
|
||||
CapabilityAlgs Capability = iota
|
||||
CapabilityHandles
|
||||
CapabilityCommands
|
||||
CapabilityPPCommands
|
||||
CapabilityAuditCommands
|
||||
CapabilityPCRs
|
||||
CapabilityTPMProperties
|
||||
CapabilityPCRProperties
|
||||
CapabilityECCCurves
|
||||
CapabilityAuthPolicies
|
||||
)
|
||||
|
||||
// TPM Structure Tags. Tags are used to disambiguate structures, similar to Alg
|
||||
// values: tag value defines what kind of data lives in a nested field.
|
||||
const (
|
||||
TagNull tpmutil.Tag = 0x8000
|
||||
TagNoSessions tpmutil.Tag = 0x8001
|
||||
TagSessions tpmutil.Tag = 0x8002
|
||||
TagAttestCertify tpmutil.Tag = 0x8017
|
||||
TagAttestQuote tpmutil.Tag = 0x8018
|
||||
TagAttestCreation tpmutil.Tag = 0x801a
|
||||
TagAuthSecret tpmutil.Tag = 0x8023
|
||||
TagHashCheck tpmutil.Tag = 0x8024
|
||||
TagAuthSigned tpmutil.Tag = 0x8025
|
||||
)
|
||||
|
||||
// StartupType instructs the TPM on how to handle its state during Shutdown or
|
||||
// Startup.
|
||||
type StartupType uint16
|
||||
|
||||
// Startup types
|
||||
const (
|
||||
StartupClear StartupType = iota
|
||||
StartupState
|
||||
)
|
||||
|
||||
// EllipticCurve identifies specific EC curves.
|
||||
type EllipticCurve uint16
|
||||
|
||||
// ECC curves supported by TPM 2.0 spec.
|
||||
const (
|
||||
CurveNISTP192 = EllipticCurve(iota + 1)
|
||||
CurveNISTP224
|
||||
CurveNISTP256
|
||||
CurveNISTP384
|
||||
CurveNISTP521
|
||||
|
||||
CurveBNP256 = EllipticCurve(iota + 10)
|
||||
CurveBNP638
|
||||
|
||||
CurveSM2P256 = EllipticCurve(0x0020)
|
||||
)
|
||||
|
||||
var toGoCurve = map[EllipticCurve]elliptic.Curve{
|
||||
CurveNISTP224: elliptic.P224(),
|
||||
CurveNISTP256: elliptic.P256(),
|
||||
CurveNISTP384: elliptic.P384(),
|
||||
CurveNISTP521: elliptic.P521(),
|
||||
}
|
||||
|
||||
// Supported TPM operations.
|
||||
const (
|
||||
CmdNVUndefineSpaceSpecial tpmutil.Command = 0x0000011F
|
||||
CmdEvictControl tpmutil.Command = 0x00000120
|
||||
CmdUndefineSpace tpmutil.Command = 0x00000122
|
||||
CmdClear tpmutil.Command = 0x00000126
|
||||
CmdHierarchyChangeAuth tpmutil.Command = 0x00000129
|
||||
CmdDefineSpace tpmutil.Command = 0x0000012A
|
||||
CmdPCRAllocate tpmutil.Command = 0x0000012B
|
||||
CmdCreatePrimary tpmutil.Command = 0x00000131
|
||||
CmdIncrementNVCounter tpmutil.Command = 0x00000134
|
||||
CmdWriteNV tpmutil.Command = 0x00000137
|
||||
CmdWriteLockNV tpmutil.Command = 0x00000138
|
||||
CmdDictionaryAttackLockReset tpmutil.Command = 0x00000139
|
||||
CmdDictionaryAttackParameters tpmutil.Command = 0x0000013A
|
||||
CmdPCREvent tpmutil.Command = 0x0000013C
|
||||
CmdPCRReset tpmutil.Command = 0x0000013D
|
||||
CmdSequenceComplete tpmutil.Command = 0x0000013E
|
||||
CmdStartup tpmutil.Command = 0x00000144
|
||||
CmdShutdown tpmutil.Command = 0x00000145
|
||||
CmdActivateCredential tpmutil.Command = 0x00000147
|
||||
CmdCertify tpmutil.Command = 0x00000148
|
||||
CmdCertifyCreation tpmutil.Command = 0x0000014A
|
||||
CmdReadNV tpmutil.Command = 0x0000014E
|
||||
CmdReadLockNV tpmutil.Command = 0x0000014F
|
||||
CmdPolicySecret tpmutil.Command = 0x00000151
|
||||
CmdCreate tpmutil.Command = 0x00000153
|
||||
CmdECDHZGen tpmutil.Command = 0x00000154
|
||||
CmdImport tpmutil.Command = 0x00000156
|
||||
CmdLoad tpmutil.Command = 0x00000157
|
||||
CmdQuote tpmutil.Command = 0x00000158
|
||||
CmdRSADecrypt tpmutil.Command = 0x00000159
|
||||
CmdSequenceUpdate tpmutil.Command = 0x0000015C
|
||||
CmdSign tpmutil.Command = 0x0000015D
|
||||
CmdUnseal tpmutil.Command = 0x0000015E
|
||||
CmdPolicySigned tpmutil.Command = 0x00000160
|
||||
CmdContextLoad tpmutil.Command = 0x00000161
|
||||
CmdContextSave tpmutil.Command = 0x00000162
|
||||
CmdECDHKeyGen tpmutil.Command = 0x00000163
|
||||
CmdEncryptDecrypt tpmutil.Command = 0x00000164
|
||||
CmdFlushContext tpmutil.Command = 0x00000165
|
||||
CmdLoadExternal tpmutil.Command = 0x00000167
|
||||
CmdMakeCredential tpmutil.Command = 0x00000168
|
||||
CmdReadPublicNV tpmutil.Command = 0x00000169
|
||||
CmdPolicyCommandCode tpmutil.Command = 0x0000016C
|
||||
CmdPolicyOr tpmutil.Command = 0x00000171
|
||||
CmdReadPublic tpmutil.Command = 0x00000173
|
||||
CmdRSAEncrypt tpmutil.Command = 0x00000174
|
||||
CmdStartAuthSession tpmutil.Command = 0x00000176
|
||||
CmdGetCapability tpmutil.Command = 0x0000017A
|
||||
CmdGetRandom tpmutil.Command = 0x0000017B
|
||||
CmdHash tpmutil.Command = 0x0000017D
|
||||
CmdPCRRead tpmutil.Command = 0x0000017E
|
||||
CmdPolicyPCR tpmutil.Command = 0x0000017F
|
||||
CmdReadClock tpmutil.Command = 0x00000181
|
||||
CmdPCRExtend tpmutil.Command = 0x00000182
|
||||
CmdEventSequenceComplete tpmutil.Command = 0x00000185
|
||||
CmdHashSequenceStart tpmutil.Command = 0x00000186
|
||||
CmdPolicyGetDigest tpmutil.Command = 0x00000189
|
||||
CmdPolicyPassword tpmutil.Command = 0x0000018C
|
||||
CmdEncryptDecrypt2 tpmutil.Command = 0x00000193
|
||||
)
|
||||
|
||||
// Regular TPM 2.0 devices use 24-bit mask (3 bytes) for PCR selection.
|
||||
const sizeOfPCRSelect = 3
|
||||
|
||||
const defaultRSAExponent = 1<<16 + 1
|
||||
|
||||
// NVAttr is a bitmask used in Attributes field of NV indexes. Individual
|
||||
// flags should be OR-ed to form a full mask.
|
||||
type NVAttr uint32
|
||||
|
||||
// NV Attributes
|
||||
const (
|
||||
AttrPPWrite NVAttr = 0x00000001
|
||||
AttrOwnerWrite NVAttr = 0x00000002
|
||||
AttrAuthWrite NVAttr = 0x00000004
|
||||
AttrPolicyWrite NVAttr = 0x00000008
|
||||
AttrPolicyDelete NVAttr = 0x00000400
|
||||
AttrWriteLocked NVAttr = 0x00000800
|
||||
AttrWriteAll NVAttr = 0x00001000
|
||||
AttrWriteDefine NVAttr = 0x00002000
|
||||
AttrWriteSTClear NVAttr = 0x00004000
|
||||
AttrGlobalLock NVAttr = 0x00008000
|
||||
AttrPPRead NVAttr = 0x00010000
|
||||
AttrOwnerRead NVAttr = 0x00020000
|
||||
AttrAuthRead NVAttr = 0x00040000
|
||||
AttrPolicyRead NVAttr = 0x00080000
|
||||
AttrNoDA NVAttr = 0x02000000
|
||||
AttrOrderly NVAttr = 0x04000000
|
||||
AttrClearSTClear NVAttr = 0x08000000
|
||||
AttrReadLocked NVAttr = 0x10000000
|
||||
AttrWritten NVAttr = 0x20000000
|
||||
AttrPlatformCreate NVAttr = 0x40000000
|
||||
AttrReadSTClear NVAttr = 0x80000000
|
||||
)
|
||||
|
||||
var permMap = map[NVAttr]string{
|
||||
AttrPPWrite: "PPWrite",
|
||||
AttrOwnerWrite: "OwnerWrite",
|
||||
AttrAuthWrite: "AuthWrite",
|
||||
AttrPolicyWrite: "PolicyWrite",
|
||||
AttrPolicyDelete: "PolicyDelete",
|
||||
AttrWriteLocked: "WriteLocked",
|
||||
AttrWriteAll: "WriteAll",
|
||||
AttrWriteDefine: "WriteDefine",
|
||||
AttrWriteSTClear: "WriteSTClear",
|
||||
AttrGlobalLock: "GlobalLock",
|
||||
AttrPPRead: "PPRead",
|
||||
AttrOwnerRead: "OwnerRead",
|
||||
AttrAuthRead: "AuthRead",
|
||||
AttrPolicyRead: "PolicyRead",
|
||||
AttrNoDA: "No Do",
|
||||
AttrOrderly: "Oderly",
|
||||
AttrClearSTClear: "ClearSTClear",
|
||||
AttrReadLocked: "ReadLocked",
|
||||
AttrWritten: "Writte",
|
||||
AttrPlatformCreate: "PlatformCreate",
|
||||
AttrReadSTClear: "ReadSTClear",
|
||||
}
|
||||
|
||||
// String returns a textual representation of the set of NVAttr
|
||||
func (p NVAttr) String() string {
|
||||
var retString strings.Builder
|
||||
for iterator, item := range permMap {
|
||||
if (p & iterator) != 0 {
|
||||
retString.WriteString(item + " + ")
|
||||
}
|
||||
}
|
||||
if retString.String() == "" {
|
||||
return "Permission/s not found"
|
||||
}
|
||||
return strings.TrimSuffix(retString.String(), " + ")
|
||||
|
||||
}
|
||||
-362
@@ -1,362 +0,0 @@
|
||||
package tpm2
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"github.com/google/go-tpm/tpmutil"
|
||||
)
|
||||
|
||||
type (
|
||||
// RCFmt0 holds Format 0 error codes
|
||||
RCFmt0 uint8
|
||||
|
||||
// RCFmt1 holds Format 1 error codes
|
||||
RCFmt1 uint8
|
||||
|
||||
// RCWarn holds error codes used in warnings
|
||||
RCWarn uint8
|
||||
|
||||
// RCIndex is used to reference arguments, handles and sessions in errors
|
||||
RCIndex uint8
|
||||
)
|
||||
|
||||
// Format 0 error codes.
|
||||
const (
|
||||
RCInitialize RCFmt0 = 0x00
|
||||
RCFailure RCFmt0 = 0x01
|
||||
RCSequence RCFmt0 = 0x03
|
||||
RCPrivate RCFmt0 = 0x0B
|
||||
RCHMAC RCFmt0 = 0x19
|
||||
RCDisabled RCFmt0 = 0x20
|
||||
RCExclusive RCFmt0 = 0x21
|
||||
RCAuthType RCFmt0 = 0x24
|
||||
RCAuthMissing RCFmt0 = 0x25
|
||||
RCPolicy RCFmt0 = 0x26
|
||||
RCPCR RCFmt0 = 0x27
|
||||
RCPCRChanged RCFmt0 = 0x28
|
||||
RCUpgrade RCFmt0 = 0x2D
|
||||
RCTooManyContexts RCFmt0 = 0x2E
|
||||
RCAuthUnavailable RCFmt0 = 0x2F
|
||||
RCReboot RCFmt0 = 0x30
|
||||
RCUnbalanced RCFmt0 = 0x31
|
||||
RCCommandSize RCFmt0 = 0x42
|
||||
RCCommandCode RCFmt0 = 0x43
|
||||
RCAuthSize RCFmt0 = 0x44
|
||||
RCAuthContext RCFmt0 = 0x45
|
||||
RCNVRange RCFmt0 = 0x46
|
||||
RCNVSize RCFmt0 = 0x47
|
||||
RCNVLocked RCFmt0 = 0x48
|
||||
RCNVAuthorization RCFmt0 = 0x49
|
||||
RCNVUninitialized RCFmt0 = 0x4A
|
||||
RCNVSpace RCFmt0 = 0x4B
|
||||
RCNVDefined RCFmt0 = 0x4C
|
||||
RCBadContext RCFmt0 = 0x50
|
||||
RCCPHash RCFmt0 = 0x51
|
||||
RCParent RCFmt0 = 0x52
|
||||
RCNeedsTest RCFmt0 = 0x53
|
||||
RCNoResult RCFmt0 = 0x54
|
||||
RCSensitive RCFmt0 = 0x55
|
||||
)
|
||||
|
||||
var fmt0Msg = map[RCFmt0]string{
|
||||
RCInitialize: "TPM not initialized by TPM2_Startup or already initialized",
|
||||
RCFailure: "commands not being accepted because of a TPM failure",
|
||||
RCSequence: "improper use of a sequence handle",
|
||||
RCPrivate: "not currently used",
|
||||
RCHMAC: "not currently used",
|
||||
RCDisabled: "the command is disabled",
|
||||
RCExclusive: "command failed because audit sequence required exclusivity",
|
||||
RCAuthType: "authorization handle is not correct for command",
|
||||
RCAuthMissing: "5 command requires an authorization session for handle and it is not present",
|
||||
RCPolicy: "policy failure in math operation or an invalid authPolicy value",
|
||||
RCPCR: "PCR check fail",
|
||||
RCPCRChanged: "PCR have changed since checked",
|
||||
RCUpgrade: "TPM is in field upgrade mode unless called via TPM2_FieldUpgradeData(), then it is not in field upgrade mode",
|
||||
RCTooManyContexts: "context ID counter is at maximum",
|
||||
RCAuthUnavailable: "authValue or authPolicy is not available for selected entity",
|
||||
RCReboot: "a _TPM_Init and Startup(CLEAR) is required before the TPM can resume operation",
|
||||
RCUnbalanced: "the protection algorithms (hash and symmetric) are not reasonably balanced; the digest size of the hash must be larger than the key size of the symmetric algorithm",
|
||||
RCCommandSize: "command commandSize value is inconsistent with contents of the command buffer; either the size is not the same as the octets loaded by the hardware interface layer or the value is not large enough to hold a command header",
|
||||
RCCommandCode: "command code not supported",
|
||||
RCAuthSize: "the value of authorizationSize is out of range or the number of octets in the Authorization Area is greater than required",
|
||||
RCAuthContext: "use of an authorization session with a context command or another command that cannot have an authorization session",
|
||||
RCNVRange: "NV offset+size is out of range",
|
||||
RCNVSize: "Requested allocation size is larger than allowed",
|
||||
RCNVLocked: "NV access locked",
|
||||
RCNVAuthorization: "NV access authorization fails in command actions",
|
||||
RCNVUninitialized: "an NV Index is used before being initialized or the state saved by TPM2_Shutdown(STATE) could not be restored",
|
||||
RCNVSpace: "insufficient space for NV allocation",
|
||||
RCNVDefined: "NV Index or persistent object already defined",
|
||||
RCBadContext: "context in TPM2_ContextLoad() is not valid",
|
||||
RCCPHash: "cpHash value already set or not correct for use",
|
||||
RCParent: "handle for parent is not a valid parent",
|
||||
RCNeedsTest: "some function needs testing",
|
||||
RCNoResult: "returned when an internal function cannot process a request due to an unspecified problem; this code is usually related to invalid parameters that are not properly filtered by the input unmarshaling code",
|
||||
RCSensitive: "the sensitive area did not unmarshal correctly after decryption",
|
||||
}
|
||||
|
||||
// Format 1 error codes.
|
||||
const (
|
||||
RCAsymmetric = 0x01
|
||||
RCAttributes = 0x02
|
||||
RCHash = 0x03
|
||||
RCValue = 0x04
|
||||
RCHierarchy = 0x05
|
||||
RCKeySize = 0x07
|
||||
RCMGF = 0x08
|
||||
RCMode = 0x09
|
||||
RCType = 0x0A
|
||||
RCHandle = 0x0B
|
||||
RCKDF = 0x0C
|
||||
RCRange = 0x0D
|
||||
RCAuthFail = 0x0E
|
||||
RCNonce = 0x0F
|
||||
RCPP = 0x10
|
||||
RCScheme = 0x12
|
||||
RCSize = 0x15
|
||||
RCSymmetric = 0x16
|
||||
RCTag = 0x17
|
||||
RCSelector = 0x18
|
||||
RCInsufficient = 0x1A
|
||||
RCSignature = 0x1B
|
||||
RCKey = 0x1C
|
||||
RCPolicyFail = 0x1D
|
||||
RCIntegrity = 0x1F
|
||||
RCTicket = 0x20
|
||||
RCReservedBits = 0x21
|
||||
RCBadAuth = 0x22
|
||||
RCExpired = 0x23
|
||||
RCPolicyCC = 0x24
|
||||
RCBinding = 0x25
|
||||
RCCurve = 0x26
|
||||
RCECCPoint = 0x27
|
||||
)
|
||||
|
||||
var fmt1Msg = map[RCFmt1]string{
|
||||
RCAsymmetric: "asymmetric algorithm not supported or not correct",
|
||||
RCAttributes: "inconsistent attributes",
|
||||
RCHash: "hash algorithm not supported or not appropriate",
|
||||
RCValue: "value is out of range or is not correct for the context",
|
||||
RCHierarchy: "hierarchy is not enabled or is not correct for the use",
|
||||
RCKeySize: "key size is not supported",
|
||||
RCMGF: "mask generation function not supported",
|
||||
RCMode: "mode of operation not supported",
|
||||
RCType: "the type of the value is not appropriate for the use",
|
||||
RCHandle: "the handle is not correct for the use",
|
||||
RCKDF: "unsupported key derivation function or function not appropriate for use",
|
||||
RCRange: "value was out of allowed range",
|
||||
RCAuthFail: "the authorization HMAC check failed and DA counter incremented",
|
||||
RCNonce: "invalid nonce size or nonce value mismatch",
|
||||
RCPP: "authorization requires assertion of PP",
|
||||
RCScheme: "unsupported or incompatible scheme",
|
||||
RCSize: "structure is the wrong size",
|
||||
RCSymmetric: "unsupported symmetric algorithm or key size, or not appropriate for instance",
|
||||
RCTag: "incorrect structure tag",
|
||||
RCSelector: "union selector is incorrect",
|
||||
RCInsufficient: "the TPM was unable to unmarshal a value because there were not enough octets in the input buffer",
|
||||
RCSignature: "the signature is not valid",
|
||||
RCKey: "key fields are not compatible with the selected use",
|
||||
RCPolicyFail: "a policy check failed",
|
||||
RCIntegrity: "integrity check failed",
|
||||
RCTicket: "invalid ticket",
|
||||
RCReservedBits: "reserved bits not set to zero as required",
|
||||
RCBadAuth: "authorization failure without DA implications",
|
||||
RCExpired: "the policy has expired",
|
||||
RCPolicyCC: "the commandCode in the policy is not the commandCode of the command or the command code in a policy command references a command that is not implemented",
|
||||
RCBinding: "public and sensitive portions of an object are not cryptographically bound",
|
||||
RCCurve: "curve not supported",
|
||||
RCECCPoint: "point is not on the required curve",
|
||||
}
|
||||
|
||||
// Warning codes.
|
||||
const (
|
||||
RCContextGap RCWarn = 0x01
|
||||
RCObjectMemory RCWarn = 0x02
|
||||
RCSessionMemory RCWarn = 0x03
|
||||
RCMemory RCWarn = 0x04
|
||||
RCSessionHandles RCWarn = 0x05
|
||||
RCObjectHandles RCWarn = 0x06
|
||||
RCLocality RCWarn = 0x07
|
||||
RCYielded RCWarn = 0x08
|
||||
RCCanceled RCWarn = 0x09
|
||||
RCTesting RCWarn = 0x0A
|
||||
RCReferenceH0 RCWarn = 0x10
|
||||
RCReferenceH1 RCWarn = 0x11
|
||||
RCReferenceH2 RCWarn = 0x12
|
||||
RCReferenceH3 RCWarn = 0x13
|
||||
RCReferenceH4 RCWarn = 0x14
|
||||
RCReferenceH5 RCWarn = 0x15
|
||||
RCReferenceH6 RCWarn = 0x16
|
||||
RCReferenceS0 RCWarn = 0x18
|
||||
RCReferenceS1 RCWarn = 0x19
|
||||
RCReferenceS2 RCWarn = 0x1A
|
||||
RCReferenceS3 RCWarn = 0x1B
|
||||
RCReferenceS4 RCWarn = 0x1C
|
||||
RCReferenceS5 RCWarn = 0x1D
|
||||
RCReferenceS6 RCWarn = 0x1E
|
||||
RCNVRate RCWarn = 0x20
|
||||
RCLockout RCWarn = 0x21
|
||||
RCRetry RCWarn = 0x22
|
||||
RCNVUnavailable RCWarn = 0x23
|
||||
)
|
||||
|
||||
var warnMsg = map[RCWarn]string{
|
||||
RCContextGap: "gap for context ID is too large",
|
||||
RCObjectMemory: "out of memory for object contexts",
|
||||
RCSessionMemory: "out of memory for session contexts",
|
||||
RCMemory: "out of shared object/session memory or need space for internal operations",
|
||||
RCSessionHandles: "out of session handles",
|
||||
RCObjectHandles: "out of object handles",
|
||||
RCLocality: "bad locality",
|
||||
RCYielded: "the TPM has suspended operation on the command; forward progress was made and the command may be retried",
|
||||
RCCanceled: "the command was canceled",
|
||||
RCTesting: "TPM is performing self-tests",
|
||||
RCReferenceH0: "the 1st handle in the handle area references a transient object or session that is not loaded",
|
||||
RCReferenceH1: "the 2nd handle in the handle area references a transient object or session that is not loaded",
|
||||
RCReferenceH2: "the 3rd handle in the handle area references a transient object or session that is not loaded",
|
||||
RCReferenceH3: "the 4th handle in the handle area references a transient object or session that is not loaded",
|
||||
RCReferenceH4: "the 5th handle in the handle area references a transient object or session that is not loaded",
|
||||
RCReferenceH5: "the 6th handle in the handle area references a transient object or session that is not loaded",
|
||||
RCReferenceH6: "the 7th handle in the handle area references a transient object or session that is not loaded",
|
||||
RCReferenceS0: "the 1st authorization session handle references a session that is not loaded",
|
||||
RCReferenceS1: "the 2nd authorization session handle references a session that is not loaded",
|
||||
RCReferenceS2: "the 3rd authorization session handle references a session that is not loaded",
|
||||
RCReferenceS3: "the 4th authorization session handle references a session that is not loaded",
|
||||
RCReferenceS4: "the 5th authorization session handle references a session that is not loaded",
|
||||
RCReferenceS5: "the 6th authorization session handle references a session that is not loaded",
|
||||
RCReferenceS6: "the 7th authorization session handle references a session that is not loaded",
|
||||
RCNVRate: "the TPM is rate-limiting accesses to prevent wearout of NV",
|
||||
RCLockout: "authorizations for objects subject to DA protection are not allowed at this time because the TPM is in DA lockout mode",
|
||||
RCRetry: "the TPM was not able to start the command",
|
||||
RCNVUnavailable: "the command may require writing of NV and NV is not current accessible",
|
||||
}
|
||||
|
||||
// Indexes for arguments, handles and sessions.
|
||||
const (
|
||||
RC1 RCIndex = iota + 1
|
||||
RC2
|
||||
RC3
|
||||
RC4
|
||||
RC5
|
||||
RC6
|
||||
RC7
|
||||
RC8
|
||||
RC9
|
||||
RCA
|
||||
RCB
|
||||
RCC
|
||||
RCD
|
||||
RCE
|
||||
RCF
|
||||
)
|
||||
|
||||
const unknownCode = "unknown error code"
|
||||
|
||||
// Error is returned for all Format 0 errors from the TPM. It is used for general
|
||||
// errors not specific to a parameter, handle or session.
|
||||
type Error struct {
|
||||
Code RCFmt0
|
||||
}
|
||||
|
||||
func (e Error) Error() string {
|
||||
msg := fmt0Msg[e.Code]
|
||||
if msg == "" {
|
||||
msg = unknownCode
|
||||
}
|
||||
return fmt.Sprintf("error code 0x%x : %s", e.Code, msg)
|
||||
}
|
||||
|
||||
// VendorError represents a vendor-specific error response. These types of responses
|
||||
// are not decoded and Code contains the complete response code.
|
||||
type VendorError struct {
|
||||
Code uint32
|
||||
}
|
||||
|
||||
func (e VendorError) Error() string {
|
||||
return fmt.Sprintf("vendor error code 0x%x", e.Code)
|
||||
}
|
||||
|
||||
// Warning is typically used to report transient errors.
|
||||
type Warning struct {
|
||||
Code RCWarn
|
||||
}
|
||||
|
||||
func (w Warning) Error() string {
|
||||
msg := warnMsg[w.Code]
|
||||
if msg == "" {
|
||||
msg = unknownCode
|
||||
}
|
||||
return fmt.Sprintf("warning code 0x%x : %s", w.Code, msg)
|
||||
}
|
||||
|
||||
// ParameterError describes an error related to a parameter, and the parameter number.
|
||||
type ParameterError struct {
|
||||
Code RCFmt1
|
||||
Parameter RCIndex
|
||||
}
|
||||
|
||||
func (e ParameterError) Error() string {
|
||||
msg := fmt1Msg[e.Code]
|
||||
if msg == "" {
|
||||
msg = unknownCode
|
||||
}
|
||||
return fmt.Sprintf("parameter %d, error code 0x%x : %s", e.Parameter, e.Code, msg)
|
||||
}
|
||||
|
||||
// HandleError describes an error related to a handle, and the handle number.
|
||||
type HandleError struct {
|
||||
Code RCFmt1
|
||||
Handle RCIndex
|
||||
}
|
||||
|
||||
func (e HandleError) Error() string {
|
||||
msg := fmt1Msg[e.Code]
|
||||
if msg == "" {
|
||||
msg = unknownCode
|
||||
}
|
||||
return fmt.Sprintf("handle %d, error code 0x%x : %s", e.Handle, e.Code, msg)
|
||||
}
|
||||
|
||||
// SessionError describes an error related to a session, and the session number.
|
||||
type SessionError struct {
|
||||
Code RCFmt1
|
||||
Session RCIndex
|
||||
}
|
||||
|
||||
func (e SessionError) Error() string {
|
||||
msg := fmt1Msg[e.Code]
|
||||
if msg == "" {
|
||||
msg = unknownCode
|
||||
}
|
||||
return fmt.Sprintf("session %d, error code 0x%x : %s", e.Session, e.Code, msg)
|
||||
}
|
||||
|
||||
// Decode a TPM2 response code and return the appropriate error. Logic
|
||||
// according to the "Response Code Evaluation" chart in Part 1 of the TPM 2.0
|
||||
// spec.
|
||||
func decodeResponse(code tpmutil.ResponseCode) error {
|
||||
if code == tpmutil.RCSuccess {
|
||||
return nil
|
||||
}
|
||||
if code&0x180 == 0 { // Bits 7:8 == 0 is a TPM1 error
|
||||
return fmt.Errorf("response status 0x%x", code)
|
||||
}
|
||||
if code&0x80 == 0 { // Bit 7 unset
|
||||
if code&0x400 > 0 { // Bit 10 set, vendor specific code
|
||||
return VendorError{uint32(code)}
|
||||
}
|
||||
if code&0x800 > 0 { // Bit 11 set, warning with code in bit 0:6
|
||||
return Warning{RCWarn(code & 0x7f)}
|
||||
}
|
||||
// error with code in bit 0:6
|
||||
return Error{RCFmt0(code & 0x7f)}
|
||||
}
|
||||
if code&0x40 > 0 { // Bit 6 set, code in 0:5, parameter number in 8:11
|
||||
return ParameterError{RCFmt1(code & 0x3f), RCIndex((code & 0xf00) >> 8)}
|
||||
}
|
||||
if code&0x800 == 0 { // Bit 11 unset, code in 0:5, handle in 8:10
|
||||
return HandleError{RCFmt1(code & 0x3f), RCIndex((code & 0x700) >> 8)}
|
||||
}
|
||||
// Code in 0:5, Session in 8:10
|
||||
return SessionError{RCFmt1(code & 0x3f), RCIndex((code & 0x700) >> 8)}
|
||||
}
|
||||
-116
@@ -1,116 +0,0 @@
|
||||
// Copyright (c) 2018, Google LLC All rights reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package tpm2
|
||||
|
||||
import (
|
||||
"crypto"
|
||||
"crypto/hmac"
|
||||
"encoding/binary"
|
||||
"hash"
|
||||
)
|
||||
|
||||
// KDFa implements TPM 2.0's default key derivation function, as defined in
|
||||
// section 11.4.9.2 of the TPM revision 2 specification part 1.
|
||||
// See: https://trustedcomputinggroup.org/resource/tpm-library-specification/
|
||||
// The key & label parameters must not be zero length.
|
||||
// The label parameter is a non-null-terminated string.
|
||||
// The contextU & contextV parameters are optional.
|
||||
// Deprecated: Use KDFaHash.
|
||||
func KDFa(hashAlg Algorithm, key []byte, label string, contextU, contextV []byte, bits int) ([]byte, error) {
|
||||
h, err := hashAlg.Hash()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return KDFaHash(h, key, label, contextU, contextV, bits), nil
|
||||
}
|
||||
|
||||
// KDFe implements TPM 2.0's ECDH key derivation function, as defined in
|
||||
// section 11.4.9.3 of the TPM revision 2 specification part 1.
|
||||
// See: https://trustedcomputinggroup.org/resource/tpm-library-specification/
|
||||
// The z parameter is the x coordinate of one party's private ECC key multiplied
|
||||
// by the other party's public ECC point.
|
||||
// The use parameter is a non-null-terminated string.
|
||||
// The partyUInfo and partyVInfo are the x coordinates of the initiator's and
|
||||
// Deprecated: Use KDFeHash.
|
||||
func KDFe(hashAlg Algorithm, z []byte, use string, partyUInfo, partyVInfo []byte, bits int) ([]byte, error) {
|
||||
h, err := hashAlg.Hash()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return KDFeHash(h, z, use, partyUInfo, partyVInfo, bits), nil
|
||||
}
|
||||
|
||||
// KDFaHash implements TPM 2.0's default key derivation function, as defined in
|
||||
// section 11.4.9.2 of the TPM revision 2 specification part 1.
|
||||
// See: https://trustedcomputinggroup.org/resource/tpm-library-specification/
|
||||
// The key & label parameters must not be zero length.
|
||||
// The label parameter is a non-null-terminated string.
|
||||
// The contextU & contextV parameters are optional.
|
||||
func KDFaHash(h crypto.Hash, key []byte, label string, contextU, contextV []byte, bits int) []byte {
|
||||
mac := hmac.New(h.New, key)
|
||||
|
||||
out := kdf(mac, bits, func() {
|
||||
mac.Write([]byte(label))
|
||||
mac.Write([]byte{0}) // Terminating null character for C-string.
|
||||
mac.Write(contextU)
|
||||
mac.Write(contextV)
|
||||
binary.Write(mac, binary.BigEndian, uint32(bits))
|
||||
})
|
||||
return out
|
||||
}
|
||||
|
||||
// KDFeHash implements TPM 2.0's ECDH key derivation function, as defined in
|
||||
// section 11.4.9.3 of the TPM revision 2 specification part 1.
|
||||
// See: https://trustedcomputinggroup.org/resource/tpm-library-specification/
|
||||
// The z parameter is the x coordinate of one party's private ECC key multiplied
|
||||
// by the other party's public ECC point.
|
||||
// The use parameter is a non-null-terminated string.
|
||||
// The partyUInfo and partyVInfo are the x coordinates of the initiator's and
|
||||
// the responder's ECC points, respectively.
|
||||
func KDFeHash(h crypto.Hash, z []byte, use string, partyUInfo, partyVInfo []byte, bits int) []byte {
|
||||
hash := h.New()
|
||||
|
||||
out := kdf(hash, bits, func() {
|
||||
hash.Write(z)
|
||||
hash.Write([]byte(use))
|
||||
hash.Write([]byte{0}) // Terminating null character for C-string.
|
||||
hash.Write(partyUInfo)
|
||||
hash.Write(partyVInfo)
|
||||
})
|
||||
return out
|
||||
}
|
||||
|
||||
func kdf(h hash.Hash, bits int, update func()) []byte {
|
||||
bytes := (bits + 7) / 8
|
||||
out := []byte{}
|
||||
|
||||
for counter := 1; len(out) < bytes; counter++ {
|
||||
h.Reset()
|
||||
binary.Write(h, binary.BigEndian, uint32(counter))
|
||||
update()
|
||||
|
||||
out = h.Sum(out)
|
||||
}
|
||||
// out's length is a multiple of hash size, so there will be excess
|
||||
// bytes if bytes isn't a multiple of hash size.
|
||||
out = out[:bytes]
|
||||
|
||||
// As mentioned in the KDFa and KDFe specs mentioned above,
|
||||
// the unused bits of the most significant octet are masked off.
|
||||
if maskBits := uint8(bits % 8); maskBits > 0 {
|
||||
out[0] &= (1 << maskBits) - 1
|
||||
}
|
||||
return out
|
||||
}
|
||||
Generated
Vendored
-57
@@ -1,57 +0,0 @@
|
||||
//go:build !windows
|
||||
|
||||
// Copyright (c) 2019, Google LLC All rights reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package tpm2
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
|
||||
"github.com/google/go-tpm/tpmutil"
|
||||
)
|
||||
|
||||
// OpenTPM opens a channel to the TPM at the given path. If the file is a
|
||||
// device, then it treats it like a normal TPM device, and if the file is a
|
||||
// Unix domain socket, then it opens a connection to the socket.
|
||||
//
|
||||
// This function may also be invoked with no paths, as tpm2.OpenTPM(). In this
|
||||
// case, the default paths on Linux (/dev/tpmrm0 then /dev/tpm0), will be used.
|
||||
func OpenTPM(path ...string) (tpm io.ReadWriteCloser, err error) {
|
||||
switch len(path) {
|
||||
case 0:
|
||||
tpm, err = tpmutil.OpenTPM("/dev/tpmrm0")
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
tpm, err = tpmutil.OpenTPM("/dev/tpm0")
|
||||
}
|
||||
case 1:
|
||||
tpm, err = tpmutil.OpenTPM(path[0])
|
||||
default:
|
||||
return nil, errors.New("cannot specify multiple paths to tpm2.OpenTPM")
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Make sure this is a TPM 2.0
|
||||
_, err = GetManufacturer(tpm)
|
||||
if err != nil {
|
||||
tpm.Close()
|
||||
return nil, fmt.Errorf("open %s: device is not a TPM 2.0", path)
|
||||
}
|
||||
return tpm, nil
|
||||
}
|
||||
Generated
Vendored
-39
@@ -1,39 +0,0 @@
|
||||
//go:build windows
|
||||
|
||||
// Copyright (c) 2018, Google LLC All rights reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package tpm2
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
|
||||
"github.com/google/go-tpm/tpmutil"
|
||||
"github.com/google/go-tpm/tpmutil/tbs"
|
||||
)
|
||||
|
||||
// OpenTPM opens a channel to the TPM.
|
||||
func OpenTPM() (io.ReadWriteCloser, error) {
|
||||
info, err := tbs.GetDeviceInfo()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if info.TPMVersion != tbs.TPMVersion20 {
|
||||
return nil, fmt.Errorf("openTPM: device is not a TPM 2.0")
|
||||
}
|
||||
|
||||
return tpmutil.OpenTPM()
|
||||
}
|
||||
Generated
Vendored
-1112
File diff suppressed because it is too large
Load Diff
-2354
File diff suppressed because it is too large
Load Diff
-211
@@ -1,211 +0,0 @@
|
||||
// Copyright (c) 2018, Google LLC All rights reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package tpmutil
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/binary"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"reflect"
|
||||
)
|
||||
|
||||
var (
|
||||
selfMarshalerType = reflect.TypeOf((*SelfMarshaler)(nil)).Elem()
|
||||
handlesAreaType = reflect.TypeOf((*[]Handle)(nil))
|
||||
)
|
||||
|
||||
// packWithHeader takes a header and a sequence of elements that are either of
|
||||
// fixed length or slices of fixed-length types and packs them into a single
|
||||
// byte array using binary.Write. It updates the CommandHeader to have the right
|
||||
// length.
|
||||
func packWithHeader(ch commandHeader, cmd ...interface{}) ([]byte, error) {
|
||||
hdrSize := binary.Size(ch)
|
||||
body, err := Pack(cmd...)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("couldn't pack message body: %v", err)
|
||||
}
|
||||
bodySize := len(body)
|
||||
ch.Size = uint32(hdrSize + bodySize)
|
||||
header, err := Pack(ch)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("couldn't pack message header: %v", err)
|
||||
}
|
||||
return append(header, body...), nil
|
||||
}
|
||||
|
||||
// Pack encodes a set of elements into a single byte array, using
|
||||
// encoding/binary. This means that all the elements must be encodeable
|
||||
// according to the rules of encoding/binary.
|
||||
//
|
||||
// It has one difference from encoding/binary: it encodes byte slices with a
|
||||
// prepended length, to match how the TPM encodes variable-length arrays. If
|
||||
// you wish to add a byte slice without length prefix, use RawBytes.
|
||||
func Pack(elts ...interface{}) ([]byte, error) {
|
||||
buf := new(bytes.Buffer)
|
||||
if err := packType(buf, elts...); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return buf.Bytes(), nil
|
||||
}
|
||||
|
||||
// tryMarshal attempts to use a TPMMarshal() method defined on the type
|
||||
// to pack v into buf. True is returned if the method exists and the
|
||||
// marshal was attempted.
|
||||
func tryMarshal(buf io.Writer, v reflect.Value) (bool, error) {
|
||||
t := v.Type()
|
||||
if t.Implements(selfMarshalerType) {
|
||||
if v.Kind() == reflect.Ptr && v.IsNil() {
|
||||
return true, fmt.Errorf("cannot call TPMMarshal on a nil pointer of type %T", v)
|
||||
}
|
||||
return true, v.Interface().(SelfMarshaler).TPMMarshal(buf)
|
||||
}
|
||||
|
||||
// We might have a non-pointer struct field, but we dont have a
|
||||
// pointer with which to implement the interface.
|
||||
// If the pointer of the type implements the interface, we should be
|
||||
// able to construct a value to call TPMMarshal() with.
|
||||
// TODO(awly): Try and avoid blowing away private data by using Addr() instead of Set()
|
||||
if reflect.PtrTo(t).Implements(selfMarshalerType) {
|
||||
tmp := reflect.New(t)
|
||||
tmp.Elem().Set(v)
|
||||
return true, tmp.Interface().(SelfMarshaler).TPMMarshal(buf)
|
||||
}
|
||||
|
||||
return false, nil
|
||||
}
|
||||
|
||||
func packValue(buf io.Writer, v reflect.Value) error {
|
||||
if v.Type() == handlesAreaType {
|
||||
v = v.Convert(reflect.TypeOf((*handleList)(nil)))
|
||||
}
|
||||
if canMarshal, err := tryMarshal(buf, v); canMarshal {
|
||||
return err
|
||||
}
|
||||
|
||||
switch v.Kind() {
|
||||
case reflect.Ptr:
|
||||
if v.IsNil() {
|
||||
return fmt.Errorf("cannot pack nil %s", v.Type().String())
|
||||
}
|
||||
return packValue(buf, v.Elem())
|
||||
case reflect.Struct:
|
||||
for i := 0; i < v.NumField(); i++ {
|
||||
f := v.Field(i)
|
||||
if err := packValue(buf, f); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
default:
|
||||
return binary.Write(buf, binary.BigEndian, v.Interface())
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func packType(buf io.Writer, elts ...interface{}) error {
|
||||
for _, e := range elts {
|
||||
if err := packValue(buf, reflect.ValueOf(e)); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// tryUnmarshal attempts to use TPMUnmarshal() to perform the
|
||||
// unpack, if the given value implements SelfMarshaler.
|
||||
// True is returned if v implements SelfMarshaler & TPMUnmarshal
|
||||
// was called, along with an error returned from TPMUnmarshal.
|
||||
func tryUnmarshal(buf io.Reader, v reflect.Value) (bool, error) {
|
||||
t := v.Type()
|
||||
if t.Implements(selfMarshalerType) {
|
||||
if v.Kind() == reflect.Ptr && v.IsNil() {
|
||||
return true, fmt.Errorf("cannot call TPMUnmarshal on a nil pointer")
|
||||
}
|
||||
return true, v.Interface().(SelfMarshaler).TPMUnmarshal(buf)
|
||||
}
|
||||
|
||||
// We might have a non-pointer struct field, which is addressable,
|
||||
// If the pointer of the type implements the interface, and the
|
||||
// value is addressable, we should be able to call TPMUnmarshal().
|
||||
if v.CanAddr() && reflect.PtrTo(t).Implements(selfMarshalerType) {
|
||||
return true, v.Addr().Interface().(SelfMarshaler).TPMUnmarshal(buf)
|
||||
}
|
||||
|
||||
return false, nil
|
||||
}
|
||||
|
||||
// Unpack is a convenience wrapper around UnpackBuf. Unpack returns the number
|
||||
// of bytes read from b to fill elts and error, if any.
|
||||
func Unpack(b []byte, elts ...interface{}) (int, error) {
|
||||
buf := bytes.NewBuffer(b)
|
||||
err := UnpackBuf(buf, elts...)
|
||||
read := len(b) - buf.Len()
|
||||
return read, err
|
||||
}
|
||||
|
||||
func unpackValue(buf io.Reader, v reflect.Value) error {
|
||||
if v.Type() == handlesAreaType {
|
||||
v = v.Convert(reflect.TypeOf((*handleList)(nil)))
|
||||
}
|
||||
if didUnmarshal, err := tryUnmarshal(buf, v); didUnmarshal {
|
||||
return err
|
||||
}
|
||||
|
||||
switch v.Kind() {
|
||||
case reflect.Ptr:
|
||||
if v.IsNil() {
|
||||
return fmt.Errorf("cannot unpack nil %s", v.Type().String())
|
||||
}
|
||||
return unpackValue(buf, v.Elem())
|
||||
case reflect.Struct:
|
||||
for i := 0; i < v.NumField(); i++ {
|
||||
f := v.Field(i)
|
||||
if err := unpackValue(buf, f); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
default:
|
||||
// binary.Read can only set pointer values, so we need to take the address.
|
||||
if !v.CanAddr() {
|
||||
return fmt.Errorf("cannot unpack unaddressable leaf type %q", v.Type().String())
|
||||
}
|
||||
return binary.Read(buf, binary.BigEndian, v.Addr().Interface())
|
||||
}
|
||||
}
|
||||
|
||||
// UnpackBuf recursively unpacks types from a reader just as encoding/binary
|
||||
// does under binary.BigEndian, but with one difference: it unpacks a byte
|
||||
// slice by first reading an integer with lengthPrefixSize bytes, then reading
|
||||
// that many bytes. It assumes that incoming values are pointers to values so
|
||||
// that, e.g., underlying slices can be resized as needed.
|
||||
func UnpackBuf(buf io.Reader, elts ...interface{}) error {
|
||||
for _, e := range elts {
|
||||
v := reflect.ValueOf(e)
|
||||
if v.Kind() != reflect.Ptr {
|
||||
return fmt.Errorf("non-pointer value %q passed to UnpackBuf", v.Type().String())
|
||||
}
|
||||
if v.IsNil() {
|
||||
return errors.New("nil pointer passed to UnpackBuf")
|
||||
}
|
||||
|
||||
if err := unpackValue(buf, v); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
-10
@@ -1,10 +0,0 @@
|
||||
//go:build !linux && !darwin
|
||||
|
||||
package tpmutil
|
||||
|
||||
import (
|
||||
"os"
|
||||
)
|
||||
|
||||
// Not implemented on Windows.
|
||||
func poll(_ *os.File) error { return nil }
|
||||
-32
@@ -1,32 +0,0 @@
|
||||
//go:build linux || darwin
|
||||
|
||||
package tpmutil
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
|
||||
"golang.org/x/sys/unix"
|
||||
)
|
||||
|
||||
// poll blocks until the file descriptor is ready for reading or an error occurs.
|
||||
func poll(f *os.File) error {
|
||||
var (
|
||||
fds = []unix.PollFd{{
|
||||
Fd: int32(f.Fd()),
|
||||
Events: 0x1, // POLLIN
|
||||
}}
|
||||
timeout = -1 // Indefinite timeout
|
||||
)
|
||||
|
||||
if _, err := unix.Poll(fds, timeout); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Revents is filled in by the kernel.
|
||||
// If the expected event happened, Revents should match Events.
|
||||
if fds[0].Revents != fds[0].Events {
|
||||
return fmt.Errorf("unexpected poll Revents 0x%x", fds[0].Revents)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
-113
@@ -1,113 +0,0 @@
|
||||
// Copyright (c) 2018, Google LLC All rights reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
// Package tpmutil provides common utility functions for both TPM 1.2 and TPM
|
||||
// 2.0 devices.
|
||||
package tpmutil
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"io"
|
||||
"os"
|
||||
"time"
|
||||
)
|
||||
|
||||
// maxTPMResponse is the largest possible response from the TPM. We need to know
|
||||
// this because we don't always know the length of the TPM response, and
|
||||
// /dev/tpm insists on giving it all back in a single value rather than
|
||||
// returning a header and a body in separate responses.
|
||||
const maxTPMResponse = 4096
|
||||
|
||||
// RunCommandRaw executes the given raw command and returns the raw response.
|
||||
// Does not check the response code except to execute retry logic.
|
||||
func RunCommandRaw(rw io.ReadWriter, inb []byte) ([]byte, error) {
|
||||
if rw == nil {
|
||||
return nil, errors.New("nil TPM handle")
|
||||
}
|
||||
|
||||
// f(t) = (2^t)ms, up to 2s
|
||||
var backoffFac uint
|
||||
var rh responseHeader
|
||||
var outb []byte
|
||||
|
||||
for {
|
||||
if _, err := rw.Write(inb); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// If the TPM is a real device, it may not be ready for reading
|
||||
// immediately after writing the command. Wait until the file
|
||||
// descriptor is ready to be read from.
|
||||
if f, ok := rw.(*os.File); ok {
|
||||
if err := poll(f); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
outb = make([]byte, maxTPMResponse)
|
||||
outlen, err := rw.Read(outb)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// Resize the buffer to match the amount read from the TPM.
|
||||
outb = outb[:outlen]
|
||||
|
||||
_, err = Unpack(outb, &rh)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// If TPM is busy, retry the command after waiting a few ms.
|
||||
if rh.Res == RCRetry {
|
||||
if backoffFac < 11 {
|
||||
dur := (1 << backoffFac) * time.Millisecond
|
||||
time.Sleep(dur)
|
||||
backoffFac++
|
||||
} else {
|
||||
return nil, err
|
||||
}
|
||||
} else {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
return outb, nil
|
||||
}
|
||||
|
||||
// RunCommand executes cmd with given tag and arguments. Returns TPM response
|
||||
// body (without response header) and response code from the header. Returned
|
||||
// error may be nil if response code is not RCSuccess; caller should check
|
||||
// both.
|
||||
func RunCommand(rw io.ReadWriter, tag Tag, cmd Command, in ...interface{}) ([]byte, ResponseCode, error) {
|
||||
inb, err := packWithHeader(commandHeader{tag, 0, cmd}, in...)
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
|
||||
outb, err := RunCommandRaw(rw, inb)
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
|
||||
var rh responseHeader
|
||||
read, err := Unpack(outb, &rh)
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
if rh.Res != RCSuccess {
|
||||
return nil, rh.Res, nil
|
||||
}
|
||||
|
||||
return outb[read:], rh.Res, nil
|
||||
}
|
||||
-111
@@ -1,111 +0,0 @@
|
||||
//go:build !windows
|
||||
|
||||
// Copyright (c) 2018, Google LLC All rights reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package tpmutil
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"os"
|
||||
)
|
||||
|
||||
// OpenTPM opens a channel to the TPM at the given path. If the file is a
|
||||
// device, then it treats it like a normal TPM device, and if the file is a
|
||||
// Unix domain socket, then it opens a connection to the socket.
|
||||
func OpenTPM(path string) (io.ReadWriteCloser, error) {
|
||||
// If it's a regular file, then open it
|
||||
var rwc io.ReadWriteCloser
|
||||
fi, err := os.Stat(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if fi.Mode()&os.ModeDevice != 0 {
|
||||
var f *os.File
|
||||
f, err = os.OpenFile(path, os.O_RDWR, 0600)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
rwc = io.ReadWriteCloser(f)
|
||||
} else if fi.Mode()&os.ModeSocket != 0 {
|
||||
rwc = NewEmulatorReadWriteCloser(path)
|
||||
} else {
|
||||
return nil, fmt.Errorf("unsupported TPM file mode %s", fi.Mode().String())
|
||||
}
|
||||
|
||||
return rwc, nil
|
||||
}
|
||||
|
||||
// dialer abstracts the net.Dial call so test code can provide its own net.Conn
|
||||
// implementation.
|
||||
type dialer func(network, path string) (net.Conn, error)
|
||||
|
||||
// EmulatorReadWriteCloser manages connections with a TPM emulator over a Unix
|
||||
// domain socket. These emulators often operate in a write/read/disconnect
|
||||
// sequence, so the Write method always connects, and the Read method always
|
||||
// closes. EmulatorReadWriteCloser is not thread safe.
|
||||
type EmulatorReadWriteCloser struct {
|
||||
path string
|
||||
conn net.Conn
|
||||
dialer dialer
|
||||
}
|
||||
|
||||
// NewEmulatorReadWriteCloser stores information about a Unix domain socket to
|
||||
// write to and read from.
|
||||
func NewEmulatorReadWriteCloser(path string) *EmulatorReadWriteCloser {
|
||||
return &EmulatorReadWriteCloser{
|
||||
path: path,
|
||||
dialer: net.Dial,
|
||||
}
|
||||
}
|
||||
|
||||
// Read implements io.Reader by reading from the Unix domain socket and closing
|
||||
// it.
|
||||
func (erw *EmulatorReadWriteCloser) Read(p []byte) (int, error) {
|
||||
// Read is always the second operation in a Write/Read sequence.
|
||||
if erw.conn == nil {
|
||||
return 0, fmt.Errorf("must call Write then Read in an alternating sequence")
|
||||
}
|
||||
n, err := erw.conn.Read(p)
|
||||
erw.conn.Close()
|
||||
erw.conn = nil
|
||||
return n, err
|
||||
}
|
||||
|
||||
// Write implements io.Writer by connecting to the Unix domain socket and
|
||||
// writing.
|
||||
func (erw *EmulatorReadWriteCloser) Write(p []byte) (int, error) {
|
||||
if erw.conn != nil {
|
||||
return 0, fmt.Errorf("must call Write then Read in an alternating sequence")
|
||||
}
|
||||
var err error
|
||||
erw.conn, err = erw.dialer("unix", erw.path)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return erw.conn.Write(p)
|
||||
}
|
||||
|
||||
// Close implements io.Closer by closing the Unix domain socket if one is open.
|
||||
func (erw *EmulatorReadWriteCloser) Close() error {
|
||||
if erw.conn == nil {
|
||||
return fmt.Errorf("cannot call Close when no connection is open")
|
||||
}
|
||||
err := erw.conn.Close()
|
||||
erw.conn = nil
|
||||
return err
|
||||
}
|
||||
Generated
Vendored
-84
@@ -1,84 +0,0 @@
|
||||
// Copyright (c) 2018, Google LLC All rights reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package tpmutil
|
||||
|
||||
import (
|
||||
"io"
|
||||
|
||||
"github.com/google/go-tpm/tpmutil/tbs"
|
||||
)
|
||||
|
||||
// winTPMBuffer is a ReadWriteCloser to access the TPM in Windows.
|
||||
type winTPMBuffer struct {
|
||||
context tbs.Context
|
||||
outBuffer []byte
|
||||
}
|
||||
|
||||
// Executes the TPM command specified by commandBuffer (at Normal Priority), returning the number
|
||||
// of bytes in the command and any error code returned by executing the TPM command. Command
|
||||
// response can be read by calling Read().
|
||||
func (rwc *winTPMBuffer) Write(commandBuffer []byte) (int, error) {
|
||||
// TPM spec defines longest possible response to be maxTPMResponse.
|
||||
rwc.outBuffer = rwc.outBuffer[:maxTPMResponse]
|
||||
|
||||
outBufferLen, err := rwc.context.SubmitCommand(
|
||||
tbs.NormalPriority,
|
||||
commandBuffer,
|
||||
rwc.outBuffer,
|
||||
)
|
||||
|
||||
if err != nil {
|
||||
rwc.outBuffer = rwc.outBuffer[:0]
|
||||
return 0, err
|
||||
}
|
||||
// Shrink outBuffer so it is length of response.
|
||||
rwc.outBuffer = rwc.outBuffer[:outBufferLen]
|
||||
return len(commandBuffer), nil
|
||||
}
|
||||
|
||||
// Provides TPM response from the command called in the last Write call.
|
||||
func (rwc *winTPMBuffer) Read(responseBuffer []byte) (int, error) {
|
||||
if len(rwc.outBuffer) == 0 {
|
||||
return 0, io.EOF
|
||||
}
|
||||
lenCopied := copy(responseBuffer, rwc.outBuffer)
|
||||
// Cut out the piece of slice which was just read out, maintaining original slice capacity.
|
||||
rwc.outBuffer = append(rwc.outBuffer[:0], rwc.outBuffer[lenCopied:]...)
|
||||
return lenCopied, nil
|
||||
}
|
||||
|
||||
func (rwc *winTPMBuffer) Close() error {
|
||||
return rwc.context.Close()
|
||||
}
|
||||
|
||||
// OpenTPM creates a new instance of a ReadWriteCloser which can interact with a
|
||||
// Windows TPM.
|
||||
func OpenTPM() (io.ReadWriteCloser, error) {
|
||||
tpmContext, err := tbs.CreateContext(tbs.TPMVersion20, tbs.IncludeTPM12|tbs.IncludeTPM20)
|
||||
rwc := &winTPMBuffer{
|
||||
context: tpmContext,
|
||||
outBuffer: make([]byte, 0, maxTPMResponse),
|
||||
}
|
||||
return rwc, err
|
||||
}
|
||||
|
||||
// FromContext creates a new instance of a ReadWriteCloser which can
|
||||
// interact with a Windows TPM, using the specified TBS handle.
|
||||
func FromContext(ctx tbs.Context) io.ReadWriteCloser {
|
||||
return &winTPMBuffer{
|
||||
context: ctx,
|
||||
outBuffer: make([]byte, 0, maxTPMResponse),
|
||||
}
|
||||
}
|
||||
-195
@@ -1,195 +0,0 @@
|
||||
// Copyright (c) 2018, Google LLC All rights reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package tpmutil
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/binary"
|
||||
"fmt"
|
||||
"io"
|
||||
)
|
||||
|
||||
// maxBytesBufferSize sets a sane upper bound on the size of a U32Bytes
|
||||
// buffer. This limit exists to prevent a maliciously large size prefix
|
||||
// from resulting in a massive memory allocation, potentially causing
|
||||
// an OOM condition on the system.
|
||||
// We expect no buffer from a TPM to approach 1Mb in size.
|
||||
const maxBytesBufferSize uint32 = 1024 * 1024 // 1Mb.
|
||||
|
||||
// RawBytes is for Pack and RunCommand arguments that are already encoded.
|
||||
// Compared to []byte, RawBytes will not be prepended with slice length during
|
||||
// encoding.
|
||||
type RawBytes []byte
|
||||
|
||||
// U16Bytes is a byte slice with a 16-bit header
|
||||
type U16Bytes []byte
|
||||
|
||||
// TPMMarshal packs U16Bytes
|
||||
func (b *U16Bytes) TPMMarshal(out io.Writer) error {
|
||||
size := len([]byte(*b))
|
||||
if err := binary.Write(out, binary.BigEndian, uint16(size)); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
n, err := out.Write(*b)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if n != size {
|
||||
return fmt.Errorf("unable to write all contents of U16Bytes")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// TPMUnmarshal unpacks a U16Bytes
|
||||
func (b *U16Bytes) TPMUnmarshal(in io.Reader) error {
|
||||
var tmpSize uint16
|
||||
if err := binary.Read(in, binary.BigEndian, &tmpSize); err != nil {
|
||||
return err
|
||||
}
|
||||
size := int(tmpSize)
|
||||
|
||||
if len(*b) >= size {
|
||||
*b = (*b)[:size]
|
||||
} else {
|
||||
*b = append(*b, make([]byte, size-len(*b))...)
|
||||
}
|
||||
|
||||
n, err := in.Read(*b)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if n != size {
|
||||
return io.ErrUnexpectedEOF
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// U32Bytes is a byte slice with a 32-bit header
|
||||
type U32Bytes []byte
|
||||
|
||||
// TPMMarshal packs U32Bytes
|
||||
func (b *U32Bytes) TPMMarshal(out io.Writer) error {
|
||||
size := len([]byte(*b))
|
||||
if err := binary.Write(out, binary.BigEndian, uint32(size)); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
n, err := out.Write(*b)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if n != size {
|
||||
return fmt.Errorf("unable to write all contents of U32Bytes")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// TPMUnmarshal unpacks a U32Bytes
|
||||
func (b *U32Bytes) TPMUnmarshal(in io.Reader) error {
|
||||
var tmpSize uint32
|
||||
if err := binary.Read(in, binary.BigEndian, &tmpSize); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if tmpSize > maxBytesBufferSize {
|
||||
return bytes.ErrTooLarge
|
||||
}
|
||||
// We can now safely cast to an int on 32-bit or 64-bit machines
|
||||
size := int(tmpSize)
|
||||
|
||||
if len(*b) >= size {
|
||||
*b = (*b)[:size]
|
||||
} else {
|
||||
*b = append(*b, make([]byte, size-len(*b))...)
|
||||
}
|
||||
|
||||
n, err := in.Read(*b)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if n != size {
|
||||
return fmt.Errorf("unable to read all contents in to U32Bytes")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Tag is a command tag.
|
||||
type Tag uint16
|
||||
|
||||
// Command is an identifier of a TPM command.
|
||||
type Command uint32
|
||||
|
||||
// A commandHeader is the header for a TPM command.
|
||||
type commandHeader struct {
|
||||
Tag Tag
|
||||
Size uint32
|
||||
Cmd Command
|
||||
}
|
||||
|
||||
// ResponseCode is a response code returned by TPM.
|
||||
type ResponseCode uint32
|
||||
|
||||
// RCSuccess is response code for successful command. Identical for TPM 1.2 and
|
||||
// 2.0.
|
||||
const RCSuccess ResponseCode = 0x000
|
||||
|
||||
// RCRetry is response code for TPM is busy.
|
||||
const RCRetry ResponseCode = 0x922
|
||||
|
||||
// A responseHeader is a header for TPM responses.
|
||||
type responseHeader struct {
|
||||
Tag Tag
|
||||
Size uint32
|
||||
Res ResponseCode
|
||||
}
|
||||
|
||||
// A Handle is a reference to a TPM object.
|
||||
type Handle uint32
|
||||
|
||||
// HandleValue returns the handle value. This behavior is intended to satisfy
|
||||
// an interface that can be implemented by other, more complex types as well.
|
||||
func (h Handle) HandleValue() uint32 {
|
||||
return uint32(h)
|
||||
}
|
||||
|
||||
type handleList []Handle
|
||||
|
||||
func (l *handleList) TPMMarshal(_ io.Writer) error {
|
||||
return fmt.Errorf("TPMMarhsal on []Handle is not supported yet")
|
||||
}
|
||||
|
||||
func (l *handleList) TPMUnmarshal(in io.Reader) error {
|
||||
var numHandles uint16
|
||||
if err := binary.Read(in, binary.BigEndian, &numHandles); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Make len(e) match size exactly.
|
||||
size := int(numHandles)
|
||||
if len(*l) >= size {
|
||||
*l = (*l)[:size]
|
||||
} else {
|
||||
*l = append(*l, make([]Handle, size-len(*l))...)
|
||||
}
|
||||
return binary.Read(in, binary.BigEndian, *l)
|
||||
}
|
||||
|
||||
// SelfMarshaler allows custom types to override default encoding/decoding
|
||||
// behavior in Pack, Unpack and UnpackBuf.
|
||||
type SelfMarshaler interface {
|
||||
TPMMarshal(out io.Writer) error
|
||||
TPMUnmarshal(in io.Reader) error
|
||||
}
|
||||
Generated
Vendored
-267
@@ -1,267 +0,0 @@
|
||||
// Copyright (c) 2018, Google LLC All rights reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
// Package tbs provides an low-level interface directly mapping to Windows
|
||||
// Tbs.dll system library commands:
|
||||
// https://docs.microsoft.com/en-us/windows/desktop/TBS/tpm-base-services-portal
|
||||
// Public field descriptions contain links to the high-level Windows documentation.
|
||||
package tbs
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"syscall"
|
||||
"unsafe"
|
||||
)
|
||||
|
||||
// Context references the current TPM context
|
||||
type Context uintptr
|
||||
|
||||
// Version of TPM being used by the application.
|
||||
type Version uint32
|
||||
|
||||
// Flag indicates TPM versions that are supported by the application.
|
||||
type Flag uint32
|
||||
|
||||
// CommandPriority is used to determine which pending command to submit whenever the TPM is free.
|
||||
type CommandPriority uint32
|
||||
|
||||
// Command parameters:
|
||||
// https://github.com/tpn/winsdk-10/blob/master/Include/10.0.10240.0/shared/tbs.h
|
||||
const (
|
||||
// https://docs.microsoft.com/en-us/windows/desktop/api/Tbs/ns-tbs-tdtbs_context_params2
|
||||
// OR flags to use multiple.
|
||||
RequestRaw Flag = 1 << iota // Add flag to request raw context
|
||||
IncludeTPM12 // Add flag to support TPM 1.2
|
||||
IncludeTPM20 // Add flag to support TPM 2
|
||||
|
||||
TPMVersion12 Version = 1 // For TPM 1.2 applications
|
||||
TPMVersion20 Version = 2 // For TPM 2 applications or applications using multiple TPM versions
|
||||
|
||||
// https://docs.microsoft.com/en-us/windows/desktop/tbs/command-scheduling
|
||||
// https://docs.microsoft.com/en-us/windows/desktop/api/Tbs/nf-tbs-tbsip_submit_command#parameters
|
||||
LowPriority CommandPriority = 100 // For low priority application use
|
||||
NormalPriority CommandPriority = 200 // For normal priority application use
|
||||
HighPriority CommandPriority = 300 // For high priority application use
|
||||
SystemPriority CommandPriority = 400 // For system tasks that access the TPM
|
||||
|
||||
commandLocalityZero uint32 = 0 // Windows currently only supports TBS_COMMAND_LOCALITY_ZERO.
|
||||
)
|
||||
|
||||
// Error is the return type of all functions in this package.
|
||||
type Error uint32
|
||||
|
||||
func (err Error) Error() string {
|
||||
if description, ok := errorDescriptions[err]; ok {
|
||||
return fmt.Sprintf("TBS Error 0x%X: %s", uint32(err), description)
|
||||
}
|
||||
return fmt.Sprintf("Unrecognized TBS Error 0x%X", uint32(err))
|
||||
}
|
||||
|
||||
func getError(err uintptr) error {
|
||||
// tbs.dll uses 0x0 as the return value for success.
|
||||
if err == 0 {
|
||||
return nil
|
||||
}
|
||||
return Error(err)
|
||||
}
|
||||
|
||||
// TBS Return Codes:
|
||||
// https://docs.microsoft.com/en-us/windows/desktop/TBS/tbs-return-codes
|
||||
const (
|
||||
ErrInternalError Error = 0x80284001
|
||||
ErrBadParameter Error = 0x80284002
|
||||
ErrInvalidOutputPointer Error = 0x80284003
|
||||
ErrInvalidContext Error = 0x80284004
|
||||
ErrInsufficientBuffer Error = 0x80284005
|
||||
ErrIOError Error = 0x80284006
|
||||
ErrInvalidContextParam Error = 0x80284007
|
||||
ErrServiceNotRunning Error = 0x80284008
|
||||
ErrTooManyTBSContexts Error = 0x80284009
|
||||
ErrTooManyResources Error = 0x8028400A
|
||||
ErrServiceStartPending Error = 0x8028400B
|
||||
ErrPPINotSupported Error = 0x8028400C
|
||||
ErrCommandCanceled Error = 0x8028400D
|
||||
ErrBufferTooLarge Error = 0x8028400E
|
||||
ErrTPMNotFound Error = 0x8028400F
|
||||
ErrServiceDisabled Error = 0x80284010
|
||||
ErrNoEventLog Error = 0x80284011
|
||||
ErrAccessDenied Error = 0x80284012
|
||||
ErrProvisioningNotAllowed Error = 0x80284013
|
||||
ErrPPIFunctionUnsupported Error = 0x80284014
|
||||
ErrOwnerauthNotFound Error = 0x80284015
|
||||
)
|
||||
|
||||
var errorDescriptions = map[Error]string{
|
||||
ErrInternalError: "An internal software error occurred.",
|
||||
ErrBadParameter: "One or more parameter values are not valid.",
|
||||
ErrInvalidOutputPointer: "A specified output pointer is bad.",
|
||||
ErrInvalidContext: "The specified context handle does not refer to a valid context.",
|
||||
ErrInsufficientBuffer: "The specified output buffer is too small.",
|
||||
ErrIOError: "An error occurred while communicating with the TPM.",
|
||||
ErrInvalidContextParam: "A context parameter that is not valid was passed when attempting to create a TBS context.",
|
||||
ErrServiceNotRunning: "The TBS service is not running and could not be started.",
|
||||
ErrTooManyTBSContexts: "A new context could not be created because there are too many open contexts.",
|
||||
ErrTooManyResources: "A new virtual resource could not be created because there are too many open virtual resources.",
|
||||
ErrServiceStartPending: "The TBS service has been started but is not yet running.",
|
||||
ErrPPINotSupported: "The physical presence interface is not supported.",
|
||||
ErrCommandCanceled: "The command was canceled.",
|
||||
ErrBufferTooLarge: "The input or output buffer is too large.",
|
||||
ErrTPMNotFound: "A compatible Trusted Platform Module (TPM) Security Device cannot be found on this computer.",
|
||||
ErrServiceDisabled: "The TBS service has been disabled.",
|
||||
ErrNoEventLog: "The TBS event log is not available.",
|
||||
ErrAccessDenied: "The caller does not have the appropriate rights to perform the requested operation.",
|
||||
ErrProvisioningNotAllowed: "The TPM provisioning action is not allowed by the specified flags.",
|
||||
ErrPPIFunctionUnsupported: "The Physical Presence Interface of this firmware does not support the requested method.",
|
||||
ErrOwnerauthNotFound: "The requested TPM OwnerAuth value was not found.",
|
||||
}
|
||||
|
||||
// Tbs.dll provides an API for making calls to the TPM:
|
||||
// https://docs.microsoft.com/en-us/windows/desktop/TBS/tpm-base-services-portal
|
||||
var (
|
||||
tbsDLL = syscall.NewLazyDLL("Tbs.dll")
|
||||
tbsGetDeviceInfo = tbsDLL.NewProc("Tbsi_GetDeviceInfo")
|
||||
tbsCreateContext = tbsDLL.NewProc("Tbsi_Context_Create")
|
||||
tbsContextClose = tbsDLL.NewProc("Tbsip_Context_Close")
|
||||
tbsSubmitCommand = tbsDLL.NewProc("Tbsip_Submit_Command")
|
||||
tbsGetTCGLog = tbsDLL.NewProc("Tbsi_Get_TCG_Log")
|
||||
)
|
||||
|
||||
// Returns the address of the beginning of a slice or 0 for a nil slice.
|
||||
func sliceAddress(s []byte) uintptr {
|
||||
if len(s) == 0 {
|
||||
return 0
|
||||
}
|
||||
return uintptr(unsafe.Pointer(&(s[0])))
|
||||
}
|
||||
|
||||
// DeviceInfo is TPM_DEVICE_INFO from tbs.h
|
||||
type DeviceInfo struct {
|
||||
StructVersion uint32
|
||||
TPMVersion Version
|
||||
TPMInterfaceType uint32
|
||||
TPMImpRevision uint32
|
||||
}
|
||||
|
||||
// GetDeviceInfo gets the DeviceInfo of the current TPM:
|
||||
// https://docs.microsoft.com/en-us/windows/win32/api/tbs/nf-tbs-tbsi_getdeviceinfo
|
||||
func GetDeviceInfo() (*DeviceInfo, error) {
|
||||
info := DeviceInfo{}
|
||||
// TBS_RESULT Tbsi_GetDeviceInfo(
|
||||
// UINT32 Size,
|
||||
// PVOID Info
|
||||
// );
|
||||
if err := tbsGetDeviceInfo.Find(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
result, _, _ := tbsGetDeviceInfo.Call(
|
||||
unsafe.Sizeof(info),
|
||||
uintptr(unsafe.Pointer(&info)),
|
||||
)
|
||||
return &info, getError(result)
|
||||
}
|
||||
|
||||
// CreateContext creates a new TPM context:
|
||||
// https://docs.microsoft.com/en-us/windows/desktop/api/Tbs/nf-tbs-tbsi_context_create
|
||||
func CreateContext(version Version, flag Flag) (Context, error) {
|
||||
var context Context
|
||||
params := struct {
|
||||
Version
|
||||
Flag
|
||||
}{version, flag}
|
||||
// TBS_RESULT Tbsi_Context_Create(
|
||||
// _In_ PCTBS_CONTEXT_PARAMS pContextParams,
|
||||
// _Out_ PTBS_HCONTEXT *phContext
|
||||
// );
|
||||
if err := tbsCreateContext.Find(); err != nil {
|
||||
return context, err
|
||||
}
|
||||
result, _, _ := tbsCreateContext.Call(
|
||||
uintptr(unsafe.Pointer(¶ms)),
|
||||
uintptr(unsafe.Pointer(&context)),
|
||||
)
|
||||
return context, getError(result)
|
||||
}
|
||||
|
||||
// Close closes an existing TPM context:
|
||||
// https://docs.microsoft.com/en-us/windows/desktop/api/Tbs/nf-tbs-tbsip_context_close
|
||||
func (context Context) Close() error {
|
||||
// TBS_RESULT Tbsip_Context_Close(
|
||||
// _In_ TBS_HCONTEXT hContext
|
||||
// );
|
||||
if err := tbsContextClose.Find(); err != nil {
|
||||
return err
|
||||
}
|
||||
result, _, _ := tbsContextClose.Call(uintptr(context))
|
||||
return getError(result)
|
||||
}
|
||||
|
||||
// SubmitCommand sends commandBuffer to the TPM, returning the number of bytes
|
||||
// written to responseBuffer. ErrInsufficientBuffer is returned if the
|
||||
// responseBuffer is too short. ErrInvalidOutputPointer is returned if the
|
||||
// responseBuffer is nil. On failure, the returned length is unspecified.
|
||||
// https://docs.microsoft.com/en-us/windows/desktop/api/Tbs/nf-tbs-tbsip_submit_command
|
||||
func (context Context) SubmitCommand(
|
||||
priority CommandPriority,
|
||||
commandBuffer []byte,
|
||||
responseBuffer []byte,
|
||||
) (uint32, error) {
|
||||
responseBufferLen := uint32(len(responseBuffer))
|
||||
|
||||
// TBS_RESULT Tbsip_Submit_Command(
|
||||
// _In_ TBS_HCONTEXT hContext,
|
||||
// _In_ TBS_COMMAND_LOCALITY Locality,
|
||||
// _In_ TBS_COMMAND_PRIORITY Priority,
|
||||
// _In_ const PCBYTE *pabCommand,
|
||||
// _In_ UINT32 cbCommand,
|
||||
// _Out_ PBYTE *pabResult,
|
||||
// _Inout_ UINT32 *pcbOutput
|
||||
// );
|
||||
if err := tbsSubmitCommand.Find(); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
result, _, _ := tbsSubmitCommand.Call(
|
||||
uintptr(context),
|
||||
uintptr(commandLocalityZero),
|
||||
uintptr(priority),
|
||||
sliceAddress(commandBuffer),
|
||||
uintptr(len(commandBuffer)),
|
||||
sliceAddress(responseBuffer),
|
||||
uintptr(unsafe.Pointer(&responseBufferLen)),
|
||||
)
|
||||
return responseBufferLen, getError(result)
|
||||
}
|
||||
|
||||
// GetTCGLog gets the system event log, returning the number of bytes written
|
||||
// to logBuffer. If logBuffer is nil, the size of the TCG log is returned.
|
||||
// ErrInsufficientBuffer is returned if the logBuffer is too short. On failure,
|
||||
// the returned length is unspecified.
|
||||
// https://docs.microsoft.com/en-us/windows/desktop/api/Tbs/nf-tbs-tbsi_get_tcg_log
|
||||
func (context Context) GetTCGLog(logBuffer []byte) (uint32, error) {
|
||||
logBufferLen := uint32(len(logBuffer))
|
||||
|
||||
// TBS_RESULT Tbsi_Get_TCG_Log(
|
||||
// TBS_HCONTEXT hContext,
|
||||
// PBYTE pOutputBuf,
|
||||
// PUINT32 pOutputBufLen
|
||||
// );
|
||||
if err := tbsGetTCGLog.Find(); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
result, _, _ := tbsGetTCGLog.Call(
|
||||
uintptr(context),
|
||||
sliceAddress(logBuffer),
|
||||
uintptr(unsafe.Pointer(&logBufferLen)),
|
||||
)
|
||||
return logBufferLen, getError(result)
|
||||
}
|
||||
-304
@@ -1,304 +0,0 @@
|
||||
Copyright (c) 2012 The Go Authors. All rights reserved.
|
||||
Copyright (c) 2019 Klaus Post. All rights reserved.
|
||||
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
modification, are permitted provided that the following conditions are
|
||||
met:
|
||||
|
||||
* Redistributions of source code must retain the above copyright
|
||||
notice, this list of conditions and the following disclaimer.
|
||||
* Redistributions in binary form must reproduce the above
|
||||
copyright notice, this list of conditions and the following disclaimer
|
||||
in the documentation and/or other materials provided with the
|
||||
distribution.
|
||||
* Neither the name of Google Inc. nor the names of its
|
||||
contributors may be used to endorse or promote products derived from
|
||||
this software without specific prior written permission.
|
||||
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||||
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
------------------
|
||||
|
||||
Files: gzhttp/*
|
||||
|
||||
Apache License
|
||||
Version 2.0, January 2004
|
||||
http://www.apache.org/licenses/
|
||||
|
||||
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
||||
|
||||
1. Definitions.
|
||||
|
||||
"License" shall mean the terms and conditions for use, reproduction,
|
||||
and distribution as defined by Sections 1 through 9 of this document.
|
||||
|
||||
"Licensor" shall mean the copyright owner or entity authorized by
|
||||
the copyright owner that is granting the License.
|
||||
|
||||
"Legal Entity" shall mean the union of the acting entity and all
|
||||
other entities that control, are controlled by, or are under common
|
||||
control with that entity. For the purposes of this definition,
|
||||
"control" means (i) the power, direct or indirect, to cause the
|
||||
direction or management of such entity, whether by contract or
|
||||
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
||||
outstanding shares, or (iii) beneficial ownership of such entity.
|
||||
|
||||
"You" (or "Your") shall mean an individual or Legal Entity
|
||||
exercising permissions granted by this License.
|
||||
|
||||
"Source" form shall mean the preferred form for making modifications,
|
||||
including but not limited to software source code, documentation
|
||||
source, and configuration files.
|
||||
|
||||
"Object" form shall mean any form resulting from mechanical
|
||||
transformation or translation of a Source form, including but
|
||||
not limited to compiled object code, generated documentation,
|
||||
and conversions to other media types.
|
||||
|
||||
"Work" shall mean the work of authorship, whether in Source or
|
||||
Object form, made available under the License, as indicated by a
|
||||
copyright notice that is included in or attached to the work
|
||||
(an example is provided in the Appendix below).
|
||||
|
||||
"Derivative Works" shall mean any work, whether in Source or Object
|
||||
form, that is based on (or derived from) the Work and for which the
|
||||
editorial revisions, annotations, elaborations, or other modifications
|
||||
represent, as a whole, an original work of authorship. For the purposes
|
||||
of this License, Derivative Works shall not include works that remain
|
||||
separable from, or merely link (or bind by name) to the interfaces of,
|
||||
the Work and Derivative Works thereof.
|
||||
|
||||
"Contribution" shall mean any work of authorship, including
|
||||
the original version of the Work and any modifications or additions
|
||||
to that Work or Derivative Works thereof, that is intentionally
|
||||
submitted to Licensor for inclusion in the Work by the copyright owner
|
||||
or by an individual or Legal Entity authorized to submit on behalf of
|
||||
the copyright owner. For the purposes of this definition, "submitted"
|
||||
means any form of electronic, verbal, or written communication sent
|
||||
to the Licensor or its representatives, including but not limited to
|
||||
communication on electronic mailing lists, source code control systems,
|
||||
and issue tracking systems that are managed by, or on behalf of, the
|
||||
Licensor for the purpose of discussing and improving the Work, but
|
||||
excluding communication that is conspicuously marked or otherwise
|
||||
designated in writing by the copyright owner as "Not a Contribution."
|
||||
|
||||
"Contributor" shall mean Licensor and any individual or Legal Entity
|
||||
on behalf of whom a Contribution has been received by Licensor and
|
||||
subsequently incorporated within the Work.
|
||||
|
||||
2. Grant of Copyright License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
copyright license to reproduce, prepare Derivative Works of,
|
||||
publicly display, publicly perform, sublicense, and distribute the
|
||||
Work and such Derivative Works in Source or Object form.
|
||||
|
||||
3. Grant of Patent License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
(except as stated in this section) patent license to make, have made,
|
||||
use, offer to sell, sell, import, and otherwise transfer the Work,
|
||||
where such license applies only to those patent claims licensable
|
||||
by such Contributor that are necessarily infringed by their
|
||||
Contribution(s) alone or by combination of their Contribution(s)
|
||||
with the Work to which such Contribution(s) was submitted. If You
|
||||
institute patent litigation against any entity (including a
|
||||
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
||||
or a Contribution incorporated within the Work constitutes direct
|
||||
or contributory patent infringement, then any patent licenses
|
||||
granted to You under this License for that Work shall terminate
|
||||
as of the date such litigation is filed.
|
||||
|
||||
4. Redistribution. You may reproduce and distribute copies of the
|
||||
Work or Derivative Works thereof in any medium, with or without
|
||||
modifications, and in Source or Object form, provided that You
|
||||
meet the following conditions:
|
||||
|
||||
(a) You must give any other recipients of the Work or
|
||||
Derivative Works a copy of this License; and
|
||||
|
||||
(b) You must cause any modified files to carry prominent notices
|
||||
stating that You changed the files; and
|
||||
|
||||
(c) You must retain, in the Source form of any Derivative Works
|
||||
that You distribute, all copyright, patent, trademark, and
|
||||
attribution notices from the Source form of the Work,
|
||||
excluding those notices that do not pertain to any part of
|
||||
the Derivative Works; and
|
||||
|
||||
(d) If the Work includes a "NOTICE" text file as part of its
|
||||
distribution, then any Derivative Works that You distribute must
|
||||
include a readable copy of the attribution notices contained
|
||||
within such NOTICE file, excluding those notices that do not
|
||||
pertain to any part of the Derivative Works, in at least one
|
||||
of the following places: within a NOTICE text file distributed
|
||||
as part of the Derivative Works; within the Source form or
|
||||
documentation, if provided along with the Derivative Works; or,
|
||||
within a display generated by the Derivative Works, if and
|
||||
wherever such third-party notices normally appear. The contents
|
||||
of the NOTICE file are for informational purposes only and
|
||||
do not modify the License. You may add Your own attribution
|
||||
notices within Derivative Works that You distribute, alongside
|
||||
or as an addendum to the NOTICE text from the Work, provided
|
||||
that such additional attribution notices cannot be construed
|
||||
as modifying the License.
|
||||
|
||||
You may add Your own copyright statement to Your modifications and
|
||||
may provide additional or different license terms and conditions
|
||||
for use, reproduction, or distribution of Your modifications, or
|
||||
for any such Derivative Works as a whole, provided Your use,
|
||||
reproduction, and distribution of the Work otherwise complies with
|
||||
the conditions stated in this License.
|
||||
|
||||
5. Submission of Contributions. Unless You explicitly state otherwise,
|
||||
any Contribution intentionally submitted for inclusion in the Work
|
||||
by You to the Licensor shall be under the terms and conditions of
|
||||
this License, without any additional terms or conditions.
|
||||
Notwithstanding the above, nothing herein shall supersede or modify
|
||||
the terms of any separate license agreement you may have executed
|
||||
with Licensor regarding such Contributions.
|
||||
|
||||
6. Trademarks. This License does not grant permission to use the trade
|
||||
names, trademarks, service marks, or product names of the Licensor,
|
||||
except as required for reasonable and customary use in describing the
|
||||
origin of the Work and reproducing the content of the NOTICE file.
|
||||
|
||||
7. Disclaimer of Warranty. Unless required by applicable law or
|
||||
agreed to in writing, Licensor provides the Work (and each
|
||||
Contributor provides its Contributions) on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
implied, including, without limitation, any warranties or conditions
|
||||
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
||||
PARTICULAR PURPOSE. You are solely responsible for determining the
|
||||
appropriateness of using or redistributing the Work and assume any
|
||||
risks associated with Your exercise of permissions under this License.
|
||||
|
||||
8. Limitation of Liability. In no event and under no legal theory,
|
||||
whether in tort (including negligence), contract, or otherwise,
|
||||
unless required by applicable law (such as deliberate and grossly
|
||||
negligent acts) or agreed to in writing, shall any Contributor be
|
||||
liable to You for damages, including any direct, indirect, special,
|
||||
incidental, or consequential damages of any character arising as a
|
||||
result of this License or out of the use or inability to use the
|
||||
Work (including but not limited to damages for loss of goodwill,
|
||||
work stoppage, computer failure or malfunction, or any and all
|
||||
other commercial damages or losses), even if such Contributor
|
||||
has been advised of the possibility of such damages.
|
||||
|
||||
9. Accepting Warranty or Additional Liability. While redistributing
|
||||
the Work or Derivative Works thereof, You may choose to offer,
|
||||
and charge a fee for, acceptance of support, warranty, indemnity,
|
||||
or other liability obligations and/or rights consistent with this
|
||||
License. However, in accepting such obligations, You may act only
|
||||
on Your own behalf and on Your sole responsibility, not on behalf
|
||||
of any other Contributor, and only if You agree to indemnify,
|
||||
defend, and hold each Contributor harmless for any liability
|
||||
incurred by, or claims asserted against, such Contributor by reason
|
||||
of your accepting any such warranty or additional liability.
|
||||
|
||||
END OF TERMS AND CONDITIONS
|
||||
|
||||
APPENDIX: How to apply the Apache License to your work.
|
||||
|
||||
To apply the Apache License to your work, attach the following
|
||||
boilerplate notice, with the fields enclosed by brackets "[]"
|
||||
replaced with your own identifying information. (Don't include
|
||||
the brackets!) The text should be enclosed in the appropriate
|
||||
comment syntax for the file format. We also recommend that a
|
||||
file or class name and description of purpose be included on the
|
||||
same "printed page" as the copyright notice for easier
|
||||
identification within third-party archives.
|
||||
|
||||
Copyright 2016-2017 The New York Times Company
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
|
||||
------------------
|
||||
|
||||
Files: s2/cmd/internal/readahead/*
|
||||
|
||||
The MIT License (MIT)
|
||||
|
||||
Copyright (c) 2015 Klaus Post
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all
|
||||
copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
SOFTWARE.
|
||||
|
||||
---------------------
|
||||
Files: snappy/*
|
||||
Files: internal/snapref/*
|
||||
|
||||
Copyright (c) 2011 The Snappy-Go Authors. All rights reserved.
|
||||
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
modification, are permitted provided that the following conditions are
|
||||
met:
|
||||
|
||||
* Redistributions of source code must retain the above copyright
|
||||
notice, this list of conditions and the following disclaimer.
|
||||
* Redistributions in binary form must reproduce the above
|
||||
copyright notice, this list of conditions and the following disclaimer
|
||||
in the documentation and/or other materials provided with the
|
||||
distribution.
|
||||
* Neither the name of Google Inc. nor the names of its
|
||||
contributors may be used to endorse or promote products derived from
|
||||
this software without specific prior written permission.
|
||||
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||||
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
-----------------
|
||||
|
||||
Files: s2/cmd/internal/filepathx/*
|
||||
|
||||
Copyright 2016 The filepathx Authors
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
-997
@@ -1,997 +0,0 @@
|
||||
// Copyright 2009 The Go Authors. All rights reserved.
|
||||
// Copyright (c) 2015 Klaus Post
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package flate
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"math"
|
||||
|
||||
"github.com/klauspost/compress/internal/le"
|
||||
)
|
||||
|
||||
const (
|
||||
NoCompression = 0
|
||||
BestSpeed = 1
|
||||
BestCompression = 9
|
||||
DefaultCompression = -1
|
||||
|
||||
// HuffmanOnly disables Lempel-Ziv match searching and only performs Huffman
|
||||
// entropy encoding. This mode is useful in compressing data that has
|
||||
// already been compressed with an LZ style algorithm (e.g. Snappy or LZ4)
|
||||
// that lacks an entropy encoder. Compression gains are achieved when
|
||||
// certain bytes in the input stream occur more frequently than others.
|
||||
//
|
||||
// Note that HuffmanOnly produces a compressed output that is
|
||||
// RFC 1951 compliant. That is, any valid DEFLATE decompressor will
|
||||
// continue to be able to decompress this output.
|
||||
HuffmanOnly = -2
|
||||
ConstantCompression = HuffmanOnly // compatibility alias.
|
||||
|
||||
logWindowSize = 15
|
||||
windowSize = 1 << logWindowSize
|
||||
windowMask = windowSize - 1
|
||||
logMaxOffsetSize = 15 // Standard DEFLATE
|
||||
minMatchLength = 4 // The smallest match that the compressor looks for
|
||||
maxMatchLength = 258 // The longest match for the compressor
|
||||
minOffsetSize = 1 // The shortest offset that makes any sense
|
||||
|
||||
// The maximum number of tokens we will encode at the time.
|
||||
// Smaller sizes usually creates less optimal blocks.
|
||||
// Bigger can make context switching slow.
|
||||
// We use this for levels 7-9, so we make it big.
|
||||
maxFlateBlockTokens = 1 << 15
|
||||
maxStoreBlockSize = 65535
|
||||
hashBits = 17 // After 17 performance degrades
|
||||
hashSize = 1 << hashBits
|
||||
hashMask = (1 << hashBits) - 1
|
||||
hashShift = (hashBits + minMatchLength - 1) / minMatchLength
|
||||
maxHashOffset = 1 << 28
|
||||
|
||||
skipNever = math.MaxInt32
|
||||
|
||||
debugDeflate = false
|
||||
)
|
||||
|
||||
type compressionLevel struct {
|
||||
good, lazy, nice, chain, fastSkipHashing, level int
|
||||
}
|
||||
|
||||
// Compression levels have been rebalanced from zlib deflate defaults
|
||||
// to give a bigger spread in speed and compression.
|
||||
// See https://blog.klauspost.com/rebalancing-deflate-compression-levels/
|
||||
var levels = []compressionLevel{
|
||||
{}, // 0
|
||||
// Level 1-6 uses specialized algorithm - values not used
|
||||
{0, 0, 0, 0, 0, 1},
|
||||
{0, 0, 0, 0, 0, 2},
|
||||
{0, 0, 0, 0, 0, 3},
|
||||
{0, 0, 0, 0, 0, 4},
|
||||
{0, 0, 0, 0, 0, 5},
|
||||
{0, 0, 0, 0, 0, 6},
|
||||
// Levels 7-9 use increasingly more lazy matching
|
||||
// and increasingly stringent conditions for "good enough".
|
||||
{8, 12, 16, 24, skipNever, 7},
|
||||
{16, 30, 40, 64, skipNever, 8},
|
||||
{32, 258, 258, 1024, skipNever, 9},
|
||||
}
|
||||
|
||||
// advancedState contains state for the advanced levels, with bigger hash tables, etc.
|
||||
type advancedState struct {
|
||||
// deflate state
|
||||
length int
|
||||
offset int
|
||||
maxInsertIndex int
|
||||
chainHead int
|
||||
hashOffset int
|
||||
|
||||
ii uint16 // position of last match, intended to overflow to reset.
|
||||
|
||||
// input window: unprocessed data is window[index:windowEnd]
|
||||
index int
|
||||
hashMatch [maxMatchLength + minMatchLength]uint32
|
||||
|
||||
// Input hash chains
|
||||
// hashHead[hashValue] contains the largest inputIndex with the specified hash value
|
||||
// If hashHead[hashValue] is within the current window, then
|
||||
// hashPrev[hashHead[hashValue] & windowMask] contains the previous index
|
||||
// with the same hash value.
|
||||
hashHead [hashSize]uint32
|
||||
hashPrev [windowSize]uint32
|
||||
}
|
||||
|
||||
type compressor struct {
|
||||
compressionLevel
|
||||
|
||||
h *huffmanEncoder
|
||||
w *huffmanBitWriter
|
||||
|
||||
// compression algorithm
|
||||
fill func(*compressor, []byte) int // copy data to window
|
||||
step func(*compressor) // process window
|
||||
|
||||
window []byte
|
||||
windowEnd int
|
||||
blockStart int // window index where current tokens start
|
||||
err error
|
||||
|
||||
// queued output tokens
|
||||
tokens tokens
|
||||
fast fastEnc
|
||||
state *advancedState
|
||||
|
||||
sync bool // requesting flush
|
||||
byteAvailable bool // if true, still need to process window[index-1].
|
||||
}
|
||||
|
||||
func (d *compressor) fillDeflate(b []byte) int {
|
||||
s := d.state
|
||||
if s.index >= 2*windowSize-(minMatchLength+maxMatchLength) {
|
||||
// shift the window by windowSize
|
||||
//copy(d.window[:], d.window[windowSize:2*windowSize])
|
||||
*(*[windowSize]byte)(d.window) = *(*[windowSize]byte)(d.window[windowSize:])
|
||||
s.index -= windowSize
|
||||
d.windowEnd -= windowSize
|
||||
if d.blockStart >= windowSize {
|
||||
d.blockStart -= windowSize
|
||||
} else {
|
||||
d.blockStart = math.MaxInt32
|
||||
}
|
||||
s.hashOffset += windowSize
|
||||
if s.hashOffset > maxHashOffset {
|
||||
delta := s.hashOffset - 1
|
||||
s.hashOffset -= delta
|
||||
s.chainHead -= delta
|
||||
// Iterate over slices instead of arrays to avoid copying
|
||||
// the entire table onto the stack (Issue #18625).
|
||||
for i, v := range s.hashPrev[:] {
|
||||
if int(v) > delta {
|
||||
s.hashPrev[i] = uint32(int(v) - delta)
|
||||
} else {
|
||||
s.hashPrev[i] = 0
|
||||
}
|
||||
}
|
||||
for i, v := range s.hashHead[:] {
|
||||
if int(v) > delta {
|
||||
s.hashHead[i] = uint32(int(v) - delta)
|
||||
} else {
|
||||
s.hashHead[i] = 0
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
n := copy(d.window[d.windowEnd:], b)
|
||||
d.windowEnd += n
|
||||
return n
|
||||
}
|
||||
|
||||
func (d *compressor) writeBlock(tok *tokens, index int, eof bool) error {
|
||||
if index > 0 || eof {
|
||||
var window []byte
|
||||
if d.blockStart <= index {
|
||||
window = d.window[d.blockStart:index]
|
||||
}
|
||||
d.blockStart = index
|
||||
//d.w.writeBlock(tok, eof, window)
|
||||
d.w.writeBlockDynamic(tok, eof, window, d.sync)
|
||||
return d.w.err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// writeBlockSkip writes the current block and uses the number of tokens
|
||||
// to determine if the block should be stored on no matches, or
|
||||
// only huffman encoded.
|
||||
func (d *compressor) writeBlockSkip(tok *tokens, index int, eof bool) error {
|
||||
if index > 0 || eof {
|
||||
if d.blockStart <= index {
|
||||
window := d.window[d.blockStart:index]
|
||||
// If we removed less than a 64th of all literals
|
||||
// we huffman compress the block.
|
||||
if int(tok.n) > len(window)-int(tok.n>>6) {
|
||||
d.w.writeBlockHuff(eof, window, d.sync)
|
||||
} else {
|
||||
// Write a dynamic huffman block.
|
||||
d.w.writeBlockDynamic(tok, eof, window, d.sync)
|
||||
}
|
||||
} else {
|
||||
d.w.writeBlock(tok, eof, nil)
|
||||
}
|
||||
d.blockStart = index
|
||||
return d.w.err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// fillWindow will fill the current window with the supplied
|
||||
// dictionary and calculate all hashes.
|
||||
// This is much faster than doing a full encode.
|
||||
// Should only be used after a start/reset.
|
||||
func (d *compressor) fillWindow(b []byte) {
|
||||
// Do not fill window if we are in store-only or huffman mode.
|
||||
if d.level <= 0 && d.level > -MinCustomWindowSize {
|
||||
return
|
||||
}
|
||||
if d.fast != nil {
|
||||
// encode the last data, but discard the result
|
||||
if len(b) > maxMatchOffset {
|
||||
b = b[len(b)-maxMatchOffset:]
|
||||
}
|
||||
d.fast.Encode(&d.tokens, b)
|
||||
d.tokens.Reset()
|
||||
return
|
||||
}
|
||||
s := d.state
|
||||
// If we are given too much, cut it.
|
||||
if len(b) > windowSize {
|
||||
b = b[len(b)-windowSize:]
|
||||
}
|
||||
// Add all to window.
|
||||
n := copy(d.window[d.windowEnd:], b)
|
||||
|
||||
// Calculate 256 hashes at the time (more L1 cache hits)
|
||||
loops := (n + 256 - minMatchLength) / 256
|
||||
for j := range loops {
|
||||
startindex := j * 256
|
||||
end := min(startindex+256+minMatchLength-1, n)
|
||||
tocheck := d.window[startindex:end]
|
||||
dstSize := len(tocheck) - minMatchLength + 1
|
||||
|
||||
if dstSize <= 0 {
|
||||
continue
|
||||
}
|
||||
|
||||
dst := s.hashMatch[:dstSize]
|
||||
bulkHash4(tocheck, dst)
|
||||
var newH uint32
|
||||
for i, val := range dst {
|
||||
di := i + startindex
|
||||
newH = val & hashMask
|
||||
// Get previous value with the same hash.
|
||||
// Our chain should point to the previous value.
|
||||
s.hashPrev[di&windowMask] = s.hashHead[newH]
|
||||
// Set the head of the hash chain to us.
|
||||
s.hashHead[newH] = uint32(di + s.hashOffset)
|
||||
}
|
||||
}
|
||||
// Update window information.
|
||||
d.windowEnd += n
|
||||
d.blockStart = d.windowEnd
|
||||
s.index = n
|
||||
}
|
||||
|
||||
// Try to find a match starting at index whose length is greater than prevSize.
|
||||
// We only look at chainCount possibilities before giving up.
|
||||
// pos = s.index, prevHead = s.chainHead-s.hashOffset, prevLength=minMatchLength-1, lookahead
|
||||
func (d *compressor) findMatch(pos int, prevHead int, lookahead int) (length, offset int, ok bool) {
|
||||
minMatchLook := min(lookahead, maxMatchLength)
|
||||
|
||||
win := d.window[0 : pos+minMatchLook]
|
||||
|
||||
// We quit when we get a match that's at least nice long
|
||||
nice := min(d.nice, len(win)-pos)
|
||||
|
||||
// If we've got a match that's good enough, only look in 1/4 the chain.
|
||||
tries := d.chain
|
||||
length = minMatchLength - 1
|
||||
|
||||
wEnd := win[pos+length]
|
||||
wPos := win[pos:]
|
||||
minIndex := max(pos-windowSize, 0)
|
||||
offset = 0
|
||||
|
||||
if d.chain < 100 {
|
||||
for i := prevHead; tries > 0; tries-- {
|
||||
if wEnd == win[i+length] {
|
||||
n := matchLen(win[i:i+minMatchLook], wPos)
|
||||
if n > length {
|
||||
length = n
|
||||
offset = pos - i
|
||||
ok = true
|
||||
if n >= nice {
|
||||
// The match is good enough that we don't try to find a better one.
|
||||
break
|
||||
}
|
||||
wEnd = win[pos+n]
|
||||
}
|
||||
}
|
||||
if i <= minIndex {
|
||||
// hashPrev[i & windowMask] has already been overwritten, so stop now.
|
||||
break
|
||||
}
|
||||
i = int(d.state.hashPrev[i&windowMask]) - d.state.hashOffset
|
||||
if i < minIndex {
|
||||
break
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// Minimum gain to accept a match.
|
||||
cGain := 4
|
||||
|
||||
// Some like it higher (CSV), some like it lower (JSON)
|
||||
const baseCost = 3
|
||||
// Base is 4 bytes at with an additional cost.
|
||||
// Matches must be better than this.
|
||||
|
||||
for i := prevHead; tries > 0; tries-- {
|
||||
if wEnd == win[i+length] {
|
||||
n := matchLen(win[i:i+minMatchLook], wPos)
|
||||
if n > length {
|
||||
// Calculate gain. Estimate
|
||||
newGain := d.h.bitLengthRaw(wPos[:n]) - int(offsetExtraBits[offsetCode(uint32(pos-i))]) - baseCost - int(lengthExtraBits[lengthCodes[(n-3)&255]])
|
||||
|
||||
//fmt.Println("gain:", newGain, "prev:", cGain, "raw:", d.h.bitLengthRaw(wPos[:n]), "this-len:", n, "prev-len:", length)
|
||||
if newGain > cGain {
|
||||
length = n
|
||||
offset = pos - i
|
||||
cGain = newGain
|
||||
ok = true
|
||||
if n >= nice {
|
||||
// The match is good enough that we don't try to find a better one.
|
||||
break
|
||||
}
|
||||
wEnd = win[pos+n]
|
||||
}
|
||||
}
|
||||
}
|
||||
if i <= minIndex {
|
||||
// hashPrev[i & windowMask] has already been overwritten, so stop now.
|
||||
break
|
||||
}
|
||||
i = int(d.state.hashPrev[i&windowMask]) - d.state.hashOffset
|
||||
if i < minIndex {
|
||||
break
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func (d *compressor) writeStoredBlock(buf []byte) error {
|
||||
if d.w.writeStoredHeader(len(buf), false); d.w.err != nil {
|
||||
return d.w.err
|
||||
}
|
||||
d.w.writeBytes(buf)
|
||||
return d.w.err
|
||||
}
|
||||
|
||||
// hash4 returns a hash representation of the first 4 bytes
|
||||
// of the supplied slice.
|
||||
// The caller must ensure that len(b) >= 4.
|
||||
func hash4(b []byte) uint32 {
|
||||
return hash4u(le.Load32(b, 0), hashBits)
|
||||
}
|
||||
|
||||
// hash4 returns the hash of u to fit in a hash table with h bits.
|
||||
// Preferably h should be a constant and should always be <32.
|
||||
func hash4u(u uint32, h uint8) uint32 {
|
||||
return (u * prime4bytes) >> (32 - h)
|
||||
}
|
||||
|
||||
// bulkHash4 will compute hashes using the same
|
||||
// algorithm as hash4
|
||||
func bulkHash4(b []byte, dst []uint32) {
|
||||
if len(b) < 4 {
|
||||
return
|
||||
}
|
||||
hb := le.Load32(b, 0)
|
||||
|
||||
dst[0] = hash4u(hb, hashBits)
|
||||
end := len(b) - 4 + 1
|
||||
for i := 1; i < end; i++ {
|
||||
hb = (hb >> 8) | uint32(b[i+3])<<24
|
||||
dst[i] = hash4u(hb, hashBits)
|
||||
}
|
||||
}
|
||||
|
||||
func (d *compressor) initDeflate() {
|
||||
d.window = make([]byte, 2*windowSize)
|
||||
d.byteAvailable = false
|
||||
d.err = nil
|
||||
if d.state == nil {
|
||||
return
|
||||
}
|
||||
s := d.state
|
||||
s.index = 0
|
||||
s.hashOffset = 1
|
||||
s.length = minMatchLength - 1
|
||||
s.offset = 0
|
||||
s.chainHead = -1
|
||||
}
|
||||
|
||||
// deflateLazy is the same as deflate, but with d.fastSkipHashing == skipNever,
|
||||
// meaning it always has lazy matching on.
|
||||
func (d *compressor) deflateLazy() {
|
||||
s := d.state
|
||||
// Sanity enables additional runtime tests.
|
||||
// It's intended to be used during development
|
||||
// to supplement the currently ad-hoc unit tests.
|
||||
const sanity = debugDeflate
|
||||
|
||||
if d.windowEnd-s.index < minMatchLength+maxMatchLength && !d.sync {
|
||||
return
|
||||
}
|
||||
if d.windowEnd != s.index && d.chain > 100 {
|
||||
// Get literal huffman coder.
|
||||
if d.h == nil {
|
||||
d.h = newHuffmanEncoder(maxFlateBlockTokens)
|
||||
}
|
||||
var tmp [256]uint16
|
||||
toIndex := d.window[s.index:d.windowEnd]
|
||||
toIndex = toIndex[:min(len(toIndex), maxFlateBlockTokens)]
|
||||
for _, v := range toIndex {
|
||||
tmp[v]++
|
||||
}
|
||||
d.h.generate(tmp[:], 15)
|
||||
}
|
||||
|
||||
s.maxInsertIndex = d.windowEnd - (minMatchLength - 1)
|
||||
|
||||
for {
|
||||
if sanity && s.index > d.windowEnd {
|
||||
panic("index > windowEnd")
|
||||
}
|
||||
lookahead := d.windowEnd - s.index
|
||||
if lookahead < minMatchLength+maxMatchLength {
|
||||
if !d.sync {
|
||||
return
|
||||
}
|
||||
if sanity && s.index > d.windowEnd {
|
||||
panic("index > windowEnd")
|
||||
}
|
||||
if lookahead == 0 {
|
||||
// Flush current output block if any.
|
||||
if d.byteAvailable {
|
||||
// There is still one pending token that needs to be flushed
|
||||
d.tokens.AddLiteral(d.window[s.index-1])
|
||||
d.byteAvailable = false
|
||||
}
|
||||
if d.tokens.n > 0 {
|
||||
if d.err = d.writeBlock(&d.tokens, s.index, false); d.err != nil {
|
||||
return
|
||||
}
|
||||
d.tokens.Reset()
|
||||
}
|
||||
return
|
||||
}
|
||||
}
|
||||
if s.index < s.maxInsertIndex {
|
||||
// Update the hash
|
||||
hash := hash4(d.window[s.index:])
|
||||
ch := s.hashHead[hash]
|
||||
s.chainHead = int(ch)
|
||||
s.hashPrev[s.index&windowMask] = ch
|
||||
s.hashHead[hash] = uint32(s.index + s.hashOffset)
|
||||
}
|
||||
prevLength := s.length
|
||||
prevOffset := s.offset
|
||||
s.length = minMatchLength - 1
|
||||
s.offset = 0
|
||||
minIndex := max(s.index-windowSize, 0)
|
||||
|
||||
if s.chainHead-s.hashOffset >= minIndex && lookahead > prevLength && prevLength < d.lazy {
|
||||
if newLength, newOffset, ok := d.findMatch(s.index, s.chainHead-s.hashOffset, lookahead); ok {
|
||||
s.length = newLength
|
||||
s.offset = newOffset
|
||||
}
|
||||
}
|
||||
|
||||
if prevLength >= minMatchLength && s.length <= prevLength {
|
||||
// No better match, but check for better match at end...
|
||||
//
|
||||
// Skip forward a number of bytes.
|
||||
// Offset of 2 seems to yield best results. 3 is sometimes better.
|
||||
const checkOff = 2
|
||||
|
||||
// Check all, except full length
|
||||
if prevLength < maxMatchLength-checkOff {
|
||||
prevIndex := s.index - 1
|
||||
if prevIndex+prevLength < s.maxInsertIndex {
|
||||
end := min(lookahead, maxMatchLength+checkOff)
|
||||
end += prevIndex
|
||||
|
||||
// Hash at match end.
|
||||
h := hash4(d.window[prevIndex+prevLength:])
|
||||
ch2 := int(s.hashHead[h]) - s.hashOffset - prevLength
|
||||
if prevIndex-ch2 != prevOffset && ch2 > minIndex+checkOff {
|
||||
length := matchLen(d.window[prevIndex+checkOff:end], d.window[ch2+checkOff:])
|
||||
// It seems like a pure length metric is best.
|
||||
if length > prevLength {
|
||||
prevLength = length
|
||||
prevOffset = prevIndex - ch2
|
||||
|
||||
// Extend back...
|
||||
for i := checkOff - 1; i >= 0; i-- {
|
||||
if prevLength >= maxMatchLength || d.window[prevIndex+i] != d.window[ch2+i] {
|
||||
// Emit tokens we "owe"
|
||||
for j := 0; j <= i; j++ {
|
||||
d.tokens.AddLiteral(d.window[prevIndex+j])
|
||||
if d.tokens.n == maxFlateBlockTokens {
|
||||
// The block includes the current character
|
||||
if d.err = d.writeBlock(&d.tokens, s.index, false); d.err != nil {
|
||||
return
|
||||
}
|
||||
d.tokens.Reset()
|
||||
}
|
||||
s.index++
|
||||
if s.index < s.maxInsertIndex {
|
||||
h := hash4(d.window[s.index:])
|
||||
ch := s.hashHead[h]
|
||||
s.chainHead = int(ch)
|
||||
s.hashPrev[s.index&windowMask] = ch
|
||||
s.hashHead[h] = uint32(s.index + s.hashOffset)
|
||||
}
|
||||
}
|
||||
break
|
||||
} else {
|
||||
prevLength++
|
||||
}
|
||||
}
|
||||
} else if false {
|
||||
// Check one further ahead.
|
||||
// Only rarely better, disabled for now.
|
||||
prevIndex++
|
||||
h := hash4(d.window[prevIndex+prevLength:])
|
||||
ch2 := int(s.hashHead[h]) - s.hashOffset - prevLength
|
||||
if prevIndex-ch2 != prevOffset && ch2 > minIndex+checkOff {
|
||||
length := matchLen(d.window[prevIndex+checkOff:end], d.window[ch2+checkOff:])
|
||||
// It seems like a pure length metric is best.
|
||||
if length > prevLength+checkOff {
|
||||
prevLength = length
|
||||
prevOffset = prevIndex - ch2
|
||||
prevIndex--
|
||||
|
||||
// Extend back...
|
||||
for i := checkOff; i >= 0; i-- {
|
||||
if prevLength >= maxMatchLength || d.window[prevIndex+i] != d.window[ch2+i-1] {
|
||||
// Emit tokens we "owe"
|
||||
for j := 0; j <= i; j++ {
|
||||
d.tokens.AddLiteral(d.window[prevIndex+j])
|
||||
if d.tokens.n == maxFlateBlockTokens {
|
||||
// The block includes the current character
|
||||
if d.err = d.writeBlock(&d.tokens, s.index, false); d.err != nil {
|
||||
return
|
||||
}
|
||||
d.tokens.Reset()
|
||||
}
|
||||
s.index++
|
||||
if s.index < s.maxInsertIndex {
|
||||
h := hash4(d.window[s.index:])
|
||||
ch := s.hashHead[h]
|
||||
s.chainHead = int(ch)
|
||||
s.hashPrev[s.index&windowMask] = ch
|
||||
s.hashHead[h] = uint32(s.index + s.hashOffset)
|
||||
}
|
||||
}
|
||||
break
|
||||
} else {
|
||||
prevLength++
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
// There was a match at the previous step, and the current match is
|
||||
// not better. Output the previous match.
|
||||
d.tokens.AddMatch(uint32(prevLength-3), uint32(prevOffset-minOffsetSize))
|
||||
|
||||
// Insert in the hash table all strings up to the end of the match.
|
||||
// index and index-1 are already inserted. If there is not enough
|
||||
// lookahead, the last two strings are not inserted into the hash
|
||||
// table.
|
||||
newIndex := s.index + prevLength - 1
|
||||
// Calculate missing hashes
|
||||
end := min(newIndex, s.maxInsertIndex)
|
||||
end += minMatchLength - 1
|
||||
startindex := min(s.index+1, s.maxInsertIndex)
|
||||
tocheck := d.window[startindex:end]
|
||||
dstSize := len(tocheck) - minMatchLength + 1
|
||||
if dstSize > 0 {
|
||||
dst := s.hashMatch[:dstSize]
|
||||
bulkHash4(tocheck, dst)
|
||||
var newH uint32
|
||||
for i, val := range dst {
|
||||
di := i + startindex
|
||||
newH = val & hashMask
|
||||
// Get previous value with the same hash.
|
||||
// Our chain should point to the previous value.
|
||||
s.hashPrev[di&windowMask] = s.hashHead[newH]
|
||||
// Set the head of the hash chain to us.
|
||||
s.hashHead[newH] = uint32(di + s.hashOffset)
|
||||
}
|
||||
}
|
||||
|
||||
s.index = newIndex
|
||||
d.byteAvailable = false
|
||||
s.length = minMatchLength - 1
|
||||
if d.tokens.n == maxFlateBlockTokens {
|
||||
// The block includes the current character
|
||||
if d.err = d.writeBlock(&d.tokens, s.index, false); d.err != nil {
|
||||
return
|
||||
}
|
||||
d.tokens.Reset()
|
||||
}
|
||||
s.ii = 0
|
||||
} else {
|
||||
// Reset, if we got a match this run.
|
||||
if s.length >= minMatchLength {
|
||||
s.ii = 0
|
||||
}
|
||||
// We have a byte waiting. Emit it.
|
||||
if d.byteAvailable {
|
||||
s.ii++
|
||||
d.tokens.AddLiteral(d.window[s.index-1])
|
||||
if d.tokens.n == maxFlateBlockTokens {
|
||||
if d.err = d.writeBlock(&d.tokens, s.index, false); d.err != nil {
|
||||
return
|
||||
}
|
||||
d.tokens.Reset()
|
||||
}
|
||||
s.index++
|
||||
|
||||
// If we have a long run of no matches, skip additional bytes
|
||||
// Resets when s.ii overflows after 64KB.
|
||||
if n := int(s.ii) - d.chain; n > 0 {
|
||||
n = 1 + int(n>>6)
|
||||
for j := 0; j < n; j++ {
|
||||
if s.index >= d.windowEnd-1 {
|
||||
break
|
||||
}
|
||||
d.tokens.AddLiteral(d.window[s.index-1])
|
||||
if d.tokens.n == maxFlateBlockTokens {
|
||||
if d.err = d.writeBlock(&d.tokens, s.index, false); d.err != nil {
|
||||
return
|
||||
}
|
||||
d.tokens.Reset()
|
||||
}
|
||||
// Index...
|
||||
if s.index < s.maxInsertIndex {
|
||||
h := hash4(d.window[s.index:])
|
||||
ch := s.hashHead[h]
|
||||
s.chainHead = int(ch)
|
||||
s.hashPrev[s.index&windowMask] = ch
|
||||
s.hashHead[h] = uint32(s.index + s.hashOffset)
|
||||
}
|
||||
s.index++
|
||||
}
|
||||
// Flush last byte
|
||||
d.tokens.AddLiteral(d.window[s.index-1])
|
||||
d.byteAvailable = false
|
||||
// s.length = minMatchLength - 1 // not needed, since s.ii is reset above, so it should never be > minMatchLength
|
||||
if d.tokens.n == maxFlateBlockTokens {
|
||||
if d.err = d.writeBlock(&d.tokens, s.index, false); d.err != nil {
|
||||
return
|
||||
}
|
||||
d.tokens.Reset()
|
||||
}
|
||||
}
|
||||
} else {
|
||||
s.index++
|
||||
d.byteAvailable = true
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (d *compressor) store() {
|
||||
if d.windowEnd > 0 && (d.windowEnd == maxStoreBlockSize || d.sync) {
|
||||
d.err = d.writeStoredBlock(d.window[:d.windowEnd])
|
||||
d.windowEnd = 0
|
||||
}
|
||||
}
|
||||
|
||||
// fillWindow will fill the buffer with data for huffman-only compression.
|
||||
// The number of bytes copied is returned.
|
||||
func (d *compressor) fillBlock(b []byte) int {
|
||||
n := copy(d.window[d.windowEnd:], b)
|
||||
d.windowEnd += n
|
||||
return n
|
||||
}
|
||||
|
||||
// storeHuff will compress and store the currently added data,
|
||||
// if enough has been accumulated or we at the end of the stream.
|
||||
// Any error that occurred will be in d.err
|
||||
func (d *compressor) storeHuff() {
|
||||
if d.windowEnd < len(d.window) && !d.sync || d.windowEnd == 0 {
|
||||
return
|
||||
}
|
||||
d.w.writeBlockHuff(false, d.window[:d.windowEnd], d.sync)
|
||||
d.err = d.w.err
|
||||
d.windowEnd = 0
|
||||
}
|
||||
|
||||
// storeFast will compress and store the currently added data,
|
||||
// if enough has been accumulated or we at the end of the stream.
|
||||
// Any error that occurred will be in d.err
|
||||
func (d *compressor) storeFast() {
|
||||
// We only compress if we have maxStoreBlockSize.
|
||||
if d.windowEnd < len(d.window) {
|
||||
if !d.sync {
|
||||
return
|
||||
}
|
||||
// Handle extremely small sizes.
|
||||
if d.windowEnd < 128 {
|
||||
if d.windowEnd == 0 {
|
||||
return
|
||||
}
|
||||
if d.windowEnd <= 32 {
|
||||
d.err = d.writeStoredBlock(d.window[:d.windowEnd])
|
||||
} else {
|
||||
d.w.writeBlockHuff(false, d.window[:d.windowEnd], true)
|
||||
d.err = d.w.err
|
||||
}
|
||||
d.tokens.Reset()
|
||||
d.windowEnd = 0
|
||||
d.fast.Reset()
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
d.fast.Encode(&d.tokens, d.window[:d.windowEnd])
|
||||
// If we made zero matches, store the block as is.
|
||||
if d.tokens.n == 0 {
|
||||
d.err = d.writeStoredBlock(d.window[:d.windowEnd])
|
||||
// If we removed less than 1/16th, huffman compress the block.
|
||||
} else if int(d.tokens.n) > d.windowEnd-(d.windowEnd>>4) {
|
||||
d.w.writeBlockHuff(false, d.window[:d.windowEnd], d.sync)
|
||||
d.err = d.w.err
|
||||
} else {
|
||||
d.w.writeBlockDynamic(&d.tokens, false, d.window[:d.windowEnd], d.sync)
|
||||
d.err = d.w.err
|
||||
}
|
||||
d.tokens.Reset()
|
||||
d.windowEnd = 0
|
||||
}
|
||||
|
||||
// write will add input byte to the stream.
|
||||
// Unless an error occurs all bytes will be consumed.
|
||||
func (d *compressor) write(b []byte) (n int, err error) {
|
||||
if d.err != nil {
|
||||
return 0, d.err
|
||||
}
|
||||
n = len(b)
|
||||
for len(b) > 0 {
|
||||
if d.windowEnd == len(d.window) || d.sync {
|
||||
d.step(d)
|
||||
}
|
||||
b = b[d.fill(d, b):]
|
||||
if d.err != nil {
|
||||
return 0, d.err
|
||||
}
|
||||
}
|
||||
return n, d.err
|
||||
}
|
||||
|
||||
func (d *compressor) syncFlush() error {
|
||||
d.sync = true
|
||||
if d.err != nil {
|
||||
return d.err
|
||||
}
|
||||
d.step(d)
|
||||
if d.err == nil {
|
||||
d.w.writeStoredHeader(0, false)
|
||||
d.w.flush()
|
||||
d.err = d.w.err
|
||||
}
|
||||
d.sync = false
|
||||
return d.err
|
||||
}
|
||||
|
||||
func (d *compressor) init(w io.Writer, level int) (err error) {
|
||||
d.w = newHuffmanBitWriter(w)
|
||||
|
||||
switch {
|
||||
case level == NoCompression:
|
||||
d.window = make([]byte, maxStoreBlockSize)
|
||||
d.fill = (*compressor).fillBlock
|
||||
d.step = (*compressor).store
|
||||
case level == ConstantCompression:
|
||||
d.w.logNewTablePenalty = 10
|
||||
d.window = make([]byte, 32<<10)
|
||||
d.fill = (*compressor).fillBlock
|
||||
d.step = (*compressor).storeHuff
|
||||
case level == DefaultCompression:
|
||||
level = 5
|
||||
fallthrough
|
||||
case level >= 1 && level <= 6:
|
||||
d.w.logNewTablePenalty = 7
|
||||
d.fast = newFastEnc(level)
|
||||
d.window = make([]byte, maxStoreBlockSize)
|
||||
d.fill = (*compressor).fillBlock
|
||||
d.step = (*compressor).storeFast
|
||||
case 7 <= level && level <= 9:
|
||||
d.w.logNewTablePenalty = 8
|
||||
d.state = &advancedState{}
|
||||
d.compressionLevel = levels[level]
|
||||
d.initDeflate()
|
||||
d.fill = (*compressor).fillDeflate
|
||||
d.step = (*compressor).deflateLazy
|
||||
case -level >= MinCustomWindowSize && -level <= MaxCustomWindowSize:
|
||||
d.w.logNewTablePenalty = 7
|
||||
d.fast = &fastEncL5Window{maxOffset: int32(-level), cur: maxStoreBlockSize}
|
||||
d.window = make([]byte, maxStoreBlockSize)
|
||||
d.fill = (*compressor).fillBlock
|
||||
d.step = (*compressor).storeFast
|
||||
default:
|
||||
return fmt.Errorf("flate: invalid compression level %d: want value in range [-2, 9]", level)
|
||||
}
|
||||
d.level = level
|
||||
return nil
|
||||
}
|
||||
|
||||
// reset the state of the compressor.
|
||||
func (d *compressor) reset(w io.Writer) {
|
||||
d.w.reset(w)
|
||||
d.sync = false
|
||||
d.err = nil
|
||||
// We only need to reset a few things for Snappy.
|
||||
if d.fast != nil {
|
||||
d.fast.Reset()
|
||||
d.windowEnd = 0
|
||||
d.tokens.Reset()
|
||||
return
|
||||
}
|
||||
switch d.compressionLevel.chain {
|
||||
case 0:
|
||||
// level was NoCompression or ConstantCompression.
|
||||
d.windowEnd = 0
|
||||
default:
|
||||
s := d.state
|
||||
s.chainHead = -1
|
||||
for i := range s.hashHead {
|
||||
s.hashHead[i] = 0
|
||||
}
|
||||
for i := range s.hashPrev {
|
||||
s.hashPrev[i] = 0
|
||||
}
|
||||
s.hashOffset = 1
|
||||
s.index, d.windowEnd = 0, 0
|
||||
d.blockStart, d.byteAvailable = 0, false
|
||||
d.tokens.Reset()
|
||||
s.length = minMatchLength - 1
|
||||
s.offset = 0
|
||||
s.ii = 0
|
||||
s.maxInsertIndex = 0
|
||||
}
|
||||
}
|
||||
|
||||
func (d *compressor) close() error {
|
||||
if d.err != nil {
|
||||
return d.err
|
||||
}
|
||||
d.sync = true
|
||||
d.step(d)
|
||||
if d.err != nil {
|
||||
return d.err
|
||||
}
|
||||
if d.w.writeStoredHeader(0, true); d.w.err != nil {
|
||||
return d.w.err
|
||||
}
|
||||
d.w.flush()
|
||||
d.w.reset(nil)
|
||||
return d.w.err
|
||||
}
|
||||
|
||||
// NewWriter returns a new Writer compressing data at the given level.
|
||||
// Following zlib, levels range from 1 (BestSpeed) to 9 (BestCompression);
|
||||
// higher levels typically run slower but compress more.
|
||||
// Level 0 (NoCompression) does not attempt any compression; it only adds the
|
||||
// necessary DEFLATE framing.
|
||||
// Level -1 (DefaultCompression) uses the default compression level.
|
||||
// Level -2 (ConstantCompression) will use Huffman compression only, giving
|
||||
// a very fast compression for all types of input, but sacrificing considerable
|
||||
// compression efficiency.
|
||||
//
|
||||
// If level is in the range [-2, 9] then the error returned will be nil.
|
||||
// Otherwise the error returned will be non-nil.
|
||||
func NewWriter(w io.Writer, level int) (*Writer, error) {
|
||||
var dw Writer
|
||||
if err := dw.d.init(w, level); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &dw, nil
|
||||
}
|
||||
|
||||
// NewWriterDict is like NewWriter but initializes the new
|
||||
// Writer with a preset dictionary. The returned Writer behaves
|
||||
// as if the dictionary had been written to it without producing
|
||||
// any compressed output. The compressed data written to w
|
||||
// can only be decompressed by a Reader initialized with the
|
||||
// same dictionary.
|
||||
func NewWriterDict(w io.Writer, level int, dict []byte) (*Writer, error) {
|
||||
zw, err := NewWriter(w, level)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
zw.d.fillWindow(dict)
|
||||
zw.dict = append(zw.dict, dict...) // duplicate dictionary for Reset method.
|
||||
return zw, err
|
||||
}
|
||||
|
||||
// MinCustomWindowSize is the minimum window size that can be sent to NewWriterWindow.
|
||||
const MinCustomWindowSize = 32
|
||||
|
||||
// MaxCustomWindowSize is the maximum custom window that can be sent to NewWriterWindow.
|
||||
const MaxCustomWindowSize = windowSize
|
||||
|
||||
// NewWriterWindow returns a new Writer compressing data with a custom window size.
|
||||
// windowSize must be from MinCustomWindowSize to MaxCustomWindowSize.
|
||||
func NewWriterWindow(w io.Writer, windowSize int) (*Writer, error) {
|
||||
if windowSize < MinCustomWindowSize {
|
||||
return nil, errors.New("flate: requested window size less than MinWindowSize")
|
||||
}
|
||||
if windowSize > MaxCustomWindowSize {
|
||||
return nil, errors.New("flate: requested window size bigger than MaxCustomWindowSize")
|
||||
}
|
||||
var dw Writer
|
||||
if err := dw.d.init(w, -windowSize); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &dw, nil
|
||||
}
|
||||
|
||||
// A Writer takes data written to it and writes the compressed
|
||||
// form of that data to an underlying writer (see NewWriter).
|
||||
type Writer struct {
|
||||
d compressor
|
||||
dict []byte
|
||||
}
|
||||
|
||||
// Write writes data to w, which will eventually write the
|
||||
// compressed form of data to its underlying writer.
|
||||
func (w *Writer) Write(data []byte) (n int, err error) {
|
||||
return w.d.write(data)
|
||||
}
|
||||
|
||||
// Flush flushes any pending data to the underlying writer.
|
||||
// It is useful mainly in compressed network protocols, to ensure that
|
||||
// a remote reader has enough data to reconstruct a packet.
|
||||
// Flush does not return until the data has been written.
|
||||
// Calling Flush when there is no pending data still causes the Writer
|
||||
// to emit a sync marker of at least 4 bytes.
|
||||
// If the underlying writer returns an error, Flush returns that error.
|
||||
//
|
||||
// In the terminology of the zlib library, Flush is equivalent to Z_SYNC_FLUSH.
|
||||
func (w *Writer) Flush() error {
|
||||
// For more about flushing:
|
||||
// http://www.bolet.org/~pornin/deflate-flush.html
|
||||
return w.d.syncFlush()
|
||||
}
|
||||
|
||||
// Close flushes and closes the writer.
|
||||
func (w *Writer) Close() error {
|
||||
return w.d.close()
|
||||
}
|
||||
|
||||
// Reset discards the writer's state and makes it equivalent to
|
||||
// the result of NewWriter or NewWriterDict called with dst
|
||||
// and w's level and dictionary.
|
||||
func (w *Writer) Reset(dst io.Writer) {
|
||||
if len(w.dict) > 0 {
|
||||
// w was created with NewWriterDict
|
||||
w.d.reset(dst)
|
||||
if dst != nil {
|
||||
w.d.fillWindow(w.dict)
|
||||
}
|
||||
} else {
|
||||
// w was created with NewWriter
|
||||
w.d.reset(dst)
|
||||
}
|
||||
}
|
||||
|
||||
// ResetDict discards the writer's state and makes it equivalent to
|
||||
// the result of NewWriter or NewWriterDict called with dst
|
||||
// and w's level, but sets a specific dictionary.
|
||||
func (w *Writer) ResetDict(dst io.Writer, dict []byte) {
|
||||
w.dict = dict
|
||||
w.d.reset(dst)
|
||||
w.d.fillWindow(w.dict)
|
||||
}
|
||||
Generated
Vendored
-199
@@ -1,199 +0,0 @@
|
||||
// Copyright 2016 The Go Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package flate
|
||||
|
||||
// dictDecoder implements the LZ77 sliding dictionary as used in decompression.
|
||||
// LZ77 decompresses data through sequences of two forms of commands:
|
||||
//
|
||||
// - Literal insertions: Runs of one or more symbols are inserted into the data
|
||||
// stream as is. This is accomplished through the writeByte method for a
|
||||
// single symbol, or combinations of writeSlice/writeMark for multiple symbols.
|
||||
// Any valid stream must start with a literal insertion if no preset dictionary
|
||||
// is used.
|
||||
//
|
||||
// - Backward copies: Runs of one or more symbols are copied from previously
|
||||
// emitted data. Backward copies come as the tuple (dist, length) where dist
|
||||
// determines how far back in the stream to copy from and length determines how
|
||||
// many bytes to copy. Note that it is valid for the length to be greater than
|
||||
// the distance. Since LZ77 uses forward copies, that situation is used to
|
||||
// perform a form of run-length encoding on repeated runs of symbols.
|
||||
// The writeCopy and tryWriteCopy are used to implement this command.
|
||||
//
|
||||
// For performance reasons, this implementation performs little to no sanity
|
||||
// checks about the arguments. As such, the invariants documented for each
|
||||
// method call must be respected.
|
||||
type dictDecoder struct {
|
||||
hist []byte // Sliding window history
|
||||
|
||||
// Invariant: 0 <= rdPos <= wrPos <= len(hist)
|
||||
wrPos int // Current output position in buffer
|
||||
rdPos int // Have emitted hist[:rdPos] already
|
||||
flushed int64 // Total bytes returned by readFlush since init
|
||||
full bool // Has a full window length been written yet?
|
||||
}
|
||||
|
||||
// init initializes dictDecoder to have a sliding window dictionary of the given
|
||||
// size. If a preset dict is provided, it will initialize the dictionary with
|
||||
// the contents of dict.
|
||||
func (dd *dictDecoder) init(size int, dict []byte) {
|
||||
*dd = dictDecoder{hist: dd.hist}
|
||||
|
||||
if cap(dd.hist) < size {
|
||||
dd.hist = make([]byte, size)
|
||||
}
|
||||
dd.hist = dd.hist[:size]
|
||||
|
||||
if len(dict) > len(dd.hist) {
|
||||
dict = dict[len(dict)-len(dd.hist):]
|
||||
}
|
||||
dd.wrPos = copy(dd.hist, dict)
|
||||
if dd.wrPos == len(dd.hist) {
|
||||
dd.wrPos = 0
|
||||
dd.full = true
|
||||
}
|
||||
dd.rdPos = dd.wrPos
|
||||
}
|
||||
|
||||
// histSize reports the total amount of historical data in the dictionary.
|
||||
func (dd *dictDecoder) histSize() int {
|
||||
if dd.full {
|
||||
return len(dd.hist)
|
||||
}
|
||||
return dd.wrPos
|
||||
}
|
||||
|
||||
// availRead reports the number of bytes that can be flushed by readFlush.
|
||||
func (dd *dictDecoder) availRead() int {
|
||||
return dd.wrPos - dd.rdPos
|
||||
}
|
||||
|
||||
// availWrite reports the available amount of output buffer space.
|
||||
func (dd *dictDecoder) availWrite() int {
|
||||
return len(dd.hist) - dd.wrPos
|
||||
}
|
||||
|
||||
// writeSlice returns a slice of the available buffer to write data to.
|
||||
//
|
||||
// This invariant will be kept: len(s) <= availWrite()
|
||||
func (dd *dictDecoder) writeSlice() []byte {
|
||||
return dd.hist[dd.wrPos:]
|
||||
}
|
||||
|
||||
// writeMark advances the writer pointer by cnt.
|
||||
//
|
||||
// This invariant must be kept: 0 <= cnt <= availWrite()
|
||||
func (dd *dictDecoder) writeMark(cnt int) {
|
||||
dd.wrPos += cnt
|
||||
}
|
||||
|
||||
// writeByte writes a single byte to the dictionary.
|
||||
//
|
||||
// This invariant must be kept: 0 < availWrite()
|
||||
func (dd *dictDecoder) writeByte(c byte) {
|
||||
dd.hist[dd.wrPos] = c
|
||||
dd.wrPos++
|
||||
}
|
||||
|
||||
// writeCopy copies a string at a given (dist, length) to the output.
|
||||
// This returns the number of bytes copied and may be less than the requested
|
||||
// length if the available space in the output buffer is too small.
|
||||
//
|
||||
// This invariant must be kept: 0 < dist <= histSize()
|
||||
func (dd *dictDecoder) writeCopy(dist, length int) int {
|
||||
dstBase := dd.wrPos
|
||||
dstPos := dstBase
|
||||
srcPos := dstPos - dist
|
||||
endPos := min(dstPos+length, len(dd.hist))
|
||||
|
||||
// Copy non-overlapping section after destination position.
|
||||
//
|
||||
// This section is non-overlapping in that the copy length for this section
|
||||
// is always less than or equal to the backwards distance. This can occur
|
||||
// if a distance refers to data that wraps-around in the buffer.
|
||||
// Thus, a backwards copy is performed here; that is, the exact bytes in
|
||||
// the source prior to the copy is placed in the destination.
|
||||
if srcPos < 0 {
|
||||
srcPos += len(dd.hist)
|
||||
dstPos += copy(dd.hist[dstPos:endPos], dd.hist[srcPos:])
|
||||
srcPos = 0
|
||||
}
|
||||
|
||||
// Copy possibly overlapping section before destination position.
|
||||
//
|
||||
// This section can overlap if the copy length for this section is larger
|
||||
// than the backwards distance. This is allowed by LZ77 so that repeated
|
||||
// strings can be succinctly represented using (dist, length) pairs.
|
||||
// Thus, a forwards copy is performed here; that is, the bytes copied is
|
||||
// possibly dependent on the resulting bytes in the destination as the copy
|
||||
// progresses along. This is functionally equivalent to the following:
|
||||
//
|
||||
// for i := 0; i < endPos-dstPos; i++ {
|
||||
// dd.hist[dstPos+i] = dd.hist[srcPos+i]
|
||||
// }
|
||||
// dstPos = endPos
|
||||
//
|
||||
for dstPos < endPos {
|
||||
dstPos += copy(dd.hist[dstPos:endPos], dd.hist[srcPos:dstPos])
|
||||
}
|
||||
|
||||
dd.wrPos = dstPos
|
||||
return dstPos - dstBase
|
||||
}
|
||||
|
||||
// tryWriteCopy tries to copy a string at a given (distance, length) to the
|
||||
// output. This specialized version is optimized for short distances.
|
||||
//
|
||||
// This method is designed to be inlined for performance reasons.
|
||||
//
|
||||
// This invariant must be kept: 0 < dist <= histSize()
|
||||
func (dd *dictDecoder) tryWriteCopy(dist, length int) int {
|
||||
dstPos := dd.wrPos
|
||||
endPos := dstPos + length
|
||||
if dstPos < dist || endPos > len(dd.hist) {
|
||||
return 0
|
||||
}
|
||||
dstBase := dstPos
|
||||
srcPos := dstPos - dist
|
||||
|
||||
// Copy possibly overlapping section before destination position.
|
||||
loop:
|
||||
dstPos += copy(dd.hist[dstPos:endPos], dd.hist[srcPos:dstPos])
|
||||
if dstPos < endPos {
|
||||
goto loop // Avoid for-loop so that this function can be inlined
|
||||
}
|
||||
|
||||
dd.wrPos = dstPos
|
||||
return dstPos - dstBase
|
||||
}
|
||||
|
||||
// appendWindow appends the current sliding window (up to len(hist) most recent
|
||||
// bytes, oldest first) to dst.
|
||||
func (dd *dictDecoder) appendWindow(dst []byte) []byte {
|
||||
if dd.full {
|
||||
dst = append(dst, dd.hist[dd.wrPos:]...)
|
||||
return append(dst, dd.hist[:dd.wrPos]...)
|
||||
}
|
||||
return append(dst, dd.hist[:dd.wrPos]...)
|
||||
}
|
||||
|
||||
// readFlush returns a slice of the historical buffer that is ready to be
|
||||
// emitted to the user. The data returned by readFlush must be fully consumed
|
||||
// before calling any other dictDecoder methods.
|
||||
func (dd *dictDecoder) readFlush() []byte {
|
||||
toRead := dd.hist[dd.rdPos:dd.wrPos]
|
||||
dd.flushed += int64(len(toRead))
|
||||
dd.rdPos = dd.wrPos
|
||||
if dd.wrPos == len(dd.hist) {
|
||||
dd.wrPos, dd.rdPos = 0, 0
|
||||
dd.full = true
|
||||
}
|
||||
return toRead
|
||||
}
|
||||
|
||||
// decoded reports the total number of bytes written into the dictionary since
|
||||
// init (i.e. excluding any preset dict bytes).
|
||||
func (dd *dictDecoder) decoded() int64 {
|
||||
return dd.flushed + int64(dd.wrPos-dd.rdPos)
|
||||
}
|
||||
Generated
Vendored
-190
@@ -1,190 +0,0 @@
|
||||
// Copyright 2011 The Snappy-Go Authors. All rights reserved.
|
||||
// Modified for deflate by Klaus Post (c) 2015.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package flate
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"github.com/klauspost/compress/internal/le"
|
||||
"github.com/klauspost/compress/internal/regmask"
|
||||
)
|
||||
|
||||
type fastEnc interface {
|
||||
Encode(dst *tokens, src []byte)
|
||||
Reset()
|
||||
}
|
||||
|
||||
func newFastEnc(level int) fastEnc {
|
||||
switch level {
|
||||
case 1:
|
||||
return &fastEncL1{fastGen: fastGen{cur: maxStoreBlockSize}}
|
||||
case 2:
|
||||
return &fastEncL2{fastGen: fastGen{cur: maxStoreBlockSize}}
|
||||
case 3:
|
||||
return &fastEncL3{fastGen: fastGen{cur: maxStoreBlockSize}}
|
||||
case 4:
|
||||
return &fastEncL4{fastGen: fastGen{cur: maxStoreBlockSize}}
|
||||
case 5:
|
||||
return &fastEncL5{fastGen: fastGen{cur: maxStoreBlockSize}}
|
||||
case 6:
|
||||
return &fastEncL6{fastGen: fastGen{cur: maxStoreBlockSize}}
|
||||
default:
|
||||
panic("invalid level specified")
|
||||
}
|
||||
}
|
||||
|
||||
const (
|
||||
tableBits = 15 // Bits used in the table
|
||||
tableSize = 1 << tableBits // Size of the table
|
||||
tableShift = 32 - tableBits // Right-shift to get the tableBits most significant bits of a uint32.
|
||||
baseMatchOffset = 1 // The smallest match offset
|
||||
baseMatchLength = 3 // The smallest match length per the RFC section 3.2.5
|
||||
maxMatchOffset = 1 << 15 // The largest match offset
|
||||
|
||||
bTableBits = 17 // Bits used in the big tables
|
||||
bTableSize = 1 << bTableBits // Size of the table
|
||||
allocHistory = maxStoreBlockSize * 5 // Size to preallocate for history.
|
||||
bufferReset = (1 << 31) - allocHistory - maxStoreBlockSize - 1 // Reset the buffer offset when reaching this.
|
||||
)
|
||||
|
||||
const (
|
||||
prime3bytes = 506832829
|
||||
prime4bytes = 2654435761
|
||||
prime5bytes = 889523592379
|
||||
prime6bytes = 227718039650203
|
||||
prime7bytes = 58295818150454627
|
||||
prime8bytes = 0xcf1bbcdcb7a56463
|
||||
)
|
||||
|
||||
func load3232(b []byte, i int32) uint32 {
|
||||
return le.Load32(b, i)
|
||||
}
|
||||
|
||||
func load6432(b []byte, i int32) uint64 {
|
||||
return le.Load64(b, i)
|
||||
}
|
||||
|
||||
type tableEntry struct {
|
||||
offset int32
|
||||
}
|
||||
|
||||
// fastGen maintains the table for matches,
|
||||
// and the previous byte block for level 2.
|
||||
// This is the generic implementation.
|
||||
type fastGen struct {
|
||||
hist []byte
|
||||
cur int32
|
||||
}
|
||||
|
||||
func (e *fastGen) addBlock(src []byte) int32 {
|
||||
// check if we have space already
|
||||
if len(e.hist)+len(src) > cap(e.hist) {
|
||||
if cap(e.hist) == 0 {
|
||||
e.hist = make([]byte, 0, allocHistory)
|
||||
} else {
|
||||
if cap(e.hist) < maxMatchOffset*2 {
|
||||
panic("unexpected buffer size")
|
||||
}
|
||||
// Move down
|
||||
offset := int32(len(e.hist)) - maxMatchOffset
|
||||
// copy(e.hist[0:maxMatchOffset], e.hist[offset:])
|
||||
*(*[maxMatchOffset]byte)(e.hist) = *(*[maxMatchOffset]byte)(e.hist[offset:])
|
||||
e.cur += offset
|
||||
e.hist = e.hist[:maxMatchOffset]
|
||||
}
|
||||
}
|
||||
s := int32(len(e.hist))
|
||||
e.hist = append(e.hist, src...)
|
||||
return s
|
||||
}
|
||||
|
||||
type tableEntryPrev struct {
|
||||
Cur tableEntry
|
||||
Prev tableEntry
|
||||
}
|
||||
|
||||
// hash7 returns the hash of the lowest 7 bytes of u to fit in a hash table with h bits.
|
||||
// Preferably h should be a constant and should always be <64.
|
||||
func hash7(u uint64, h uint8) uint32 {
|
||||
return uint32(((u << (64 - 56)) * prime7bytes) >> ((64 - h) & regmask.Shift64ByUint8))
|
||||
}
|
||||
|
||||
// hashLen returns a hash of the lowest mls bytes of with length output bits.
|
||||
// mls must be >=3 and <=8. Any other value will return hash for 4 bytes.
|
||||
// length should always be < 32.
|
||||
// Preferably length and mls should be a constant for inlining.
|
||||
func hashLen(u uint64, length, mls uint8) uint32 {
|
||||
switch mls {
|
||||
case 3:
|
||||
return (uint32(u<<8) * prime3bytes) >> (32 - length)
|
||||
case 5:
|
||||
return uint32(((u << (64 - 40)) * prime5bytes) >> (64 - length))
|
||||
case 6:
|
||||
return uint32(((u << (64 - 48)) * prime6bytes) >> (64 - length))
|
||||
case 7:
|
||||
return uint32(((u << (64 - 56)) * prime7bytes) >> (64 - length))
|
||||
case 8:
|
||||
return uint32((u * prime8bytes) >> (64 - length))
|
||||
default:
|
||||
return (uint32(u) * prime4bytes) >> (32 - length)
|
||||
}
|
||||
}
|
||||
|
||||
// matchlen will return the match length between offsets and t in src.
|
||||
// The maximum length returned is maxMatchLength - 4.
|
||||
// It is assumed that s > t, that t >=0 and s < len(src).
|
||||
func (e *fastGen) matchlen(s, t int, src []byte) int32 {
|
||||
if debugDeflate {
|
||||
if t >= s {
|
||||
panic(fmt.Sprint("t >=s:", t, s))
|
||||
}
|
||||
if int(s) >= len(src) {
|
||||
panic(fmt.Sprint("s >= len(src):", s, len(src)))
|
||||
}
|
||||
if t < 0 {
|
||||
panic(fmt.Sprint("t < 0:", t))
|
||||
}
|
||||
if s-t > maxMatchOffset {
|
||||
panic(fmt.Sprint(s, "-", t, "(", s-t, ") > maxMatchLength (", maxMatchOffset, ")"))
|
||||
}
|
||||
}
|
||||
a := src[s:min(s+maxMatchLength-4, len(src))]
|
||||
b := src[t:]
|
||||
return int32(matchLen(a, b))
|
||||
}
|
||||
|
||||
// matchlenLong will return the match length between offsets and t in src.
|
||||
// It is assumed that s > t, that t >=0 and s < len(src).
|
||||
func (e *fastGen) matchlenLong(s, t int, src []byte) int32 {
|
||||
if debugDeflate {
|
||||
if t >= s {
|
||||
panic(fmt.Sprint("t >=s:", t, s))
|
||||
}
|
||||
if int(s) >= len(src) {
|
||||
panic(fmt.Sprint("s >= len(src):", s, len(src)))
|
||||
}
|
||||
if t < 0 {
|
||||
panic(fmt.Sprint("t < 0:", t))
|
||||
}
|
||||
if s-t > maxMatchOffset {
|
||||
panic(fmt.Sprint(s, "-", t, "(", s-t, ") > maxMatchLength (", maxMatchOffset, ")"))
|
||||
}
|
||||
}
|
||||
return int32(matchLen(src[s:], src[t:]))
|
||||
}
|
||||
|
||||
// Reset the encoding table.
|
||||
func (e *fastGen) Reset() {
|
||||
if cap(e.hist) < allocHistory {
|
||||
e.hist = make([]byte, 0, allocHistory)
|
||||
}
|
||||
// We offset current position so everything will be out of reach.
|
||||
// If we are above the buffer reset it will be cleared anyway since len(hist) == 0.
|
||||
if e.cur <= bufferReset {
|
||||
e.cur += maxMatchOffset + int32(len(e.hist))
|
||||
}
|
||||
e.hist = e.hist[:0]
|
||||
}
|
||||
Generated
Vendored
-1174
File diff suppressed because it is too large
Load Diff
Generated
Vendored
-417
@@ -1,417 +0,0 @@
|
||||
// Copyright 2009 The Go Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package flate
|
||||
|
||||
import (
|
||||
"math"
|
||||
"math/bits"
|
||||
)
|
||||
|
||||
const (
|
||||
maxBitsLimit = 16
|
||||
// number of valid literals
|
||||
literalCount = 286
|
||||
)
|
||||
|
||||
// hcode is a huffman code with a bit code and bit length.
|
||||
type hcode uint32
|
||||
|
||||
func (h hcode) len() uint8 {
|
||||
return uint8(h)
|
||||
}
|
||||
|
||||
func (h hcode) code64() uint64 {
|
||||
return uint64(h >> 8)
|
||||
}
|
||||
|
||||
func (h hcode) zero() bool {
|
||||
return h == 0
|
||||
}
|
||||
|
||||
type huffmanEncoder struct {
|
||||
codes []hcode
|
||||
bitCount [17]int32
|
||||
|
||||
// Allocate a reusable buffer with the longest possible frequency table.
|
||||
// Possible lengths are codegenCodeCount, offsetCodeCount and literalCount.
|
||||
// The largest of these is literalCount, so we allocate for that case.
|
||||
freqcache [literalCount + 1]literalNode
|
||||
}
|
||||
|
||||
type literalNode struct {
|
||||
literal uint16
|
||||
freq uint16
|
||||
}
|
||||
|
||||
// A levelInfo describes the state of the constructed tree for a given depth.
|
||||
type levelInfo struct {
|
||||
// Our level. for better printing
|
||||
level int32
|
||||
|
||||
// The frequency of the last node at this level
|
||||
lastFreq int32
|
||||
|
||||
// The frequency of the next character to add to this level
|
||||
nextCharFreq int32
|
||||
|
||||
// The frequency of the next pair (from level below) to add to this level.
|
||||
// Only valid if the "needed" value of the next lower level is 0.
|
||||
nextPairFreq int32
|
||||
|
||||
// The number of chains remaining to generate for this level before moving
|
||||
// up to the next level
|
||||
needed int32
|
||||
}
|
||||
|
||||
// set sets the code and length of an hcode.
|
||||
func (h *hcode) set(code uint16, length uint8) {
|
||||
*h = hcode(length) | (hcode(code) << 8)
|
||||
}
|
||||
|
||||
func newhcode(code uint16, length uint8) hcode {
|
||||
return hcode(length) | (hcode(code) << 8)
|
||||
}
|
||||
|
||||
func reverseBits(number uint16, bitLength byte) uint16 {
|
||||
return bits.Reverse16(number << ((16 - bitLength) & 15))
|
||||
}
|
||||
|
||||
func maxNode() literalNode { return literalNode{math.MaxUint16, math.MaxUint16} }
|
||||
|
||||
func newHuffmanEncoder(size int) *huffmanEncoder {
|
||||
// Make capacity to next power of two.
|
||||
c := uint(bits.Len32(uint32(size - 1)))
|
||||
return &huffmanEncoder{codes: make([]hcode, size, 1<<c)}
|
||||
}
|
||||
|
||||
// Generates a HuffmanCode corresponding to the fixed literal table
|
||||
func generateFixedLiteralEncoding() *huffmanEncoder {
|
||||
h := newHuffmanEncoder(literalCount)
|
||||
codes := h.codes
|
||||
var ch uint16
|
||||
for ch = range uint16(literalCount) {
|
||||
var bits uint16
|
||||
var size uint8
|
||||
switch {
|
||||
case ch < 144:
|
||||
// size 8, 000110000 .. 10111111
|
||||
bits = ch + 48
|
||||
size = 8
|
||||
case ch < 256:
|
||||
// size 9, 110010000 .. 111111111
|
||||
bits = ch + 400 - 144
|
||||
size = 9
|
||||
case ch < 280:
|
||||
// size 7, 0000000 .. 0010111
|
||||
bits = ch - 256
|
||||
size = 7
|
||||
default:
|
||||
// size 8, 11000000 .. 11000111
|
||||
bits = ch + 192 - 280
|
||||
size = 8
|
||||
}
|
||||
codes[ch] = newhcode(reverseBits(bits, size), size)
|
||||
}
|
||||
return h
|
||||
}
|
||||
|
||||
func generateFixedOffsetEncoding() *huffmanEncoder {
|
||||
h := newHuffmanEncoder(30)
|
||||
codes := h.codes
|
||||
for ch := range codes {
|
||||
codes[ch] = newhcode(reverseBits(uint16(ch), 5), 5)
|
||||
}
|
||||
return h
|
||||
}
|
||||
|
||||
var fixedLiteralEncoding = generateFixedLiteralEncoding()
|
||||
var fixedOffsetEncoding = generateFixedOffsetEncoding()
|
||||
|
||||
func (h *huffmanEncoder) bitLength(freq []uint16) int {
|
||||
var total int
|
||||
for i, f := range freq {
|
||||
if f != 0 {
|
||||
total += int(f) * int(h.codes[i].len())
|
||||
}
|
||||
}
|
||||
return total
|
||||
}
|
||||
|
||||
func (h *huffmanEncoder) bitLengthRaw(b []byte) int {
|
||||
var total int
|
||||
for _, f := range b {
|
||||
total += int(h.codes[f].len())
|
||||
}
|
||||
return total
|
||||
}
|
||||
|
||||
// canReuseBits returns the number of bits or math.MaxInt32 if the encoder cannot be reused.
|
||||
func (h *huffmanEncoder) canReuseBits(freq []uint16) int {
|
||||
var total int
|
||||
for i, f := range freq {
|
||||
if f != 0 {
|
||||
code := h.codes[i]
|
||||
if code.zero() {
|
||||
return math.MaxInt32
|
||||
}
|
||||
total += int(f) * int(code.len())
|
||||
}
|
||||
}
|
||||
return total
|
||||
}
|
||||
|
||||
// Return the number of literals assigned to each bit size in the Huffman encoding
|
||||
//
|
||||
// This method is only called when list.length >= 3
|
||||
// The cases of 0, 1, and 2 literals are handled by special case code.
|
||||
//
|
||||
// list An array of the literals with non-zero frequencies
|
||||
//
|
||||
// and their associated frequencies. The array is in order of increasing
|
||||
// frequency, and has as its last element a special element with frequency
|
||||
// MaxInt32
|
||||
//
|
||||
// maxBits The maximum number of bits that should be used to encode any literal.
|
||||
//
|
||||
// Must be less than 16.
|
||||
//
|
||||
// return An integer array in which array[i] indicates the number of literals
|
||||
//
|
||||
// that should be encoded in i bits.
|
||||
func (h *huffmanEncoder) bitCounts(list []literalNode, maxBits int32) []int32 {
|
||||
if maxBits >= maxBitsLimit {
|
||||
panic("flate: maxBits too large")
|
||||
}
|
||||
n := int32(len(list))
|
||||
list = list[0 : n+1]
|
||||
list[n] = maxNode()
|
||||
|
||||
// The tree can't have greater depth than n - 1, no matter what. This
|
||||
// saves a little bit of work in some small cases
|
||||
if maxBits > n-1 {
|
||||
maxBits = n - 1
|
||||
}
|
||||
|
||||
// Create information about each of the levels.
|
||||
// A bogus "Level 0" whose sole purpose is so that
|
||||
// level1.prev.needed==0. This makes level1.nextPairFreq
|
||||
// be a legitimate value that never gets chosen.
|
||||
var levels [maxBitsLimit]levelInfo
|
||||
// leafCounts[i] counts the number of literals at the left
|
||||
// of ancestors of the rightmost node at level i.
|
||||
// leafCounts[i][j] is the number of literals at the left
|
||||
// of the level j ancestor.
|
||||
var leafCounts [maxBitsLimit][maxBitsLimit]int32
|
||||
|
||||
// Descending to only have 1 bounds check.
|
||||
l2f := int32(list[2].freq)
|
||||
l1f := int32(list[1].freq)
|
||||
l0f := int32(list[0].freq) + int32(list[1].freq)
|
||||
|
||||
for level := int32(1); level <= maxBits; level++ {
|
||||
// For every level, the first two items are the first two characters.
|
||||
// We initialize the levels as if we had already figured this out.
|
||||
levels[level] = levelInfo{
|
||||
level: level,
|
||||
lastFreq: l1f,
|
||||
nextCharFreq: l2f,
|
||||
nextPairFreq: l0f,
|
||||
}
|
||||
leafCounts[level][level] = 2
|
||||
if level == 1 {
|
||||
levels[level].nextPairFreq = math.MaxInt32
|
||||
}
|
||||
}
|
||||
|
||||
// We need a total of 2*n - 2 items at top level and have already generated 2.
|
||||
levels[maxBits].needed = 2*n - 4
|
||||
|
||||
level := uint32(maxBits)
|
||||
for level < 16 {
|
||||
l := &levels[level]
|
||||
if l.nextPairFreq == math.MaxInt32 && l.nextCharFreq == math.MaxInt32 {
|
||||
// We've run out of both leafs and pairs.
|
||||
// End all calculations for this level.
|
||||
// To make sure we never come back to this level or any lower level,
|
||||
// set nextPairFreq impossibly large.
|
||||
l.needed = 0
|
||||
levels[level+1].nextPairFreq = math.MaxInt32
|
||||
level++
|
||||
continue
|
||||
}
|
||||
|
||||
prevFreq := l.lastFreq
|
||||
if l.nextCharFreq < l.nextPairFreq {
|
||||
// The next item on this row is a leaf node.
|
||||
n := leafCounts[level][level] + 1
|
||||
l.lastFreq = l.nextCharFreq
|
||||
// Lower leafCounts are the same of the previous node.
|
||||
leafCounts[level][level] = n
|
||||
e := list[n]
|
||||
if e.literal < math.MaxUint16 {
|
||||
l.nextCharFreq = int32(e.freq)
|
||||
} else {
|
||||
l.nextCharFreq = math.MaxInt32
|
||||
}
|
||||
} else {
|
||||
// The next item on this row is a pair from the previous row.
|
||||
// nextPairFreq isn't valid until we generate two
|
||||
// more values in the level below
|
||||
l.lastFreq = l.nextPairFreq
|
||||
// Take leaf counts from the lower level, except counts[level] remains the same.
|
||||
if true {
|
||||
save := leafCounts[level][level]
|
||||
leafCounts[level] = leafCounts[level-1]
|
||||
leafCounts[level][level] = save
|
||||
} else {
|
||||
copy(leafCounts[level][:level], leafCounts[level-1][:level])
|
||||
}
|
||||
levels[l.level-1].needed = 2
|
||||
}
|
||||
|
||||
if l.needed--; l.needed == 0 {
|
||||
// We've done everything we need to do for this level.
|
||||
// Continue calculating one level up. Fill in nextPairFreq
|
||||
// of that level with the sum of the two nodes we've just calculated on
|
||||
// this level.
|
||||
if l.level == maxBits {
|
||||
// All done!
|
||||
break
|
||||
}
|
||||
levels[l.level+1].nextPairFreq = prevFreq + l.lastFreq
|
||||
level++
|
||||
} else {
|
||||
// If we stole from below, move down temporarily to replenish it.
|
||||
for levels[level-1].needed > 0 {
|
||||
level--
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Somethings is wrong if at the end, the top level is null or hasn't used
|
||||
// all of the leaves.
|
||||
if leafCounts[maxBits][maxBits] != n {
|
||||
panic("leafCounts[maxBits][maxBits] != n")
|
||||
}
|
||||
|
||||
bitCount := h.bitCount[:maxBits+1]
|
||||
bits := 1
|
||||
counts := &leafCounts[maxBits]
|
||||
for level := maxBits; level > 0; level-- {
|
||||
// chain.leafCount gives the number of literals requiring at least "bits"
|
||||
// bits to encode.
|
||||
bitCount[bits] = counts[level] - counts[level-1]
|
||||
bits++
|
||||
}
|
||||
return bitCount
|
||||
}
|
||||
|
||||
// Look at the leaves and assign them a bit count and an encoding as specified
|
||||
// in RFC 1951 3.2.2
|
||||
func (h *huffmanEncoder) assignEncodingAndSize(bitCount []int32, list []literalNode) {
|
||||
code := uint16(0)
|
||||
for n, bits := range bitCount {
|
||||
code <<= 1
|
||||
if n == 0 || bits == 0 {
|
||||
continue
|
||||
}
|
||||
// The literals list[len(list)-bits] .. list[len(list)-bits]
|
||||
// are encoded using "bits" bits, and get the values
|
||||
// code, code + 1, .... The code values are
|
||||
// assigned in literal order (not frequency order).
|
||||
chunk := list[len(list)-int(bits):]
|
||||
|
||||
sortByLiteral(chunk)
|
||||
for _, node := range chunk {
|
||||
h.codes[node.literal] = newhcode(reverseBits(code, uint8(n)), uint8(n))
|
||||
code++
|
||||
}
|
||||
list = list[0 : len(list)-int(bits)]
|
||||
}
|
||||
}
|
||||
|
||||
// Update this Huffman Code object to be the minimum code for the specified frequency count.
|
||||
//
|
||||
// freq An array of frequencies, in which frequency[i] gives the frequency of literal i.
|
||||
// maxBits The maximum number of bits to use for any literal.
|
||||
func (h *huffmanEncoder) generate(freq []uint16, maxBits int32) {
|
||||
list := h.freqcache[:len(freq)+1]
|
||||
codes := h.codes[:len(freq)]
|
||||
// Number of non-zero literals
|
||||
count := 0
|
||||
// Set list to be the set of all non-zero literals and their frequencies
|
||||
for i, f := range freq {
|
||||
if f != 0 {
|
||||
list[count] = literalNode{uint16(i), f}
|
||||
count++
|
||||
} else {
|
||||
codes[i] = 0
|
||||
}
|
||||
}
|
||||
list[count] = literalNode{}
|
||||
|
||||
list = list[:count]
|
||||
if count <= 2 {
|
||||
// Handle the small cases here, because they are awkward for the general case code. With
|
||||
// two or fewer literals, everything has bit length 1.
|
||||
for i, node := range list {
|
||||
// "list" is in order of increasing literal value.
|
||||
h.codes[node.literal].set(uint16(i), 1)
|
||||
}
|
||||
return
|
||||
}
|
||||
sortByFreq(list)
|
||||
|
||||
// Get the number of literals for each bit count
|
||||
bitCount := h.bitCounts(list, maxBits)
|
||||
// And do the assignment
|
||||
h.assignEncodingAndSize(bitCount, list)
|
||||
}
|
||||
|
||||
// atLeastOne clamps the result between 1 and 15.
|
||||
func atLeastOne(v float32) float32 {
|
||||
if v < 1 {
|
||||
return 1
|
||||
}
|
||||
if v > 15 {
|
||||
return 15
|
||||
}
|
||||
return v
|
||||
}
|
||||
|
||||
func histogram(b []byte, h []uint16) {
|
||||
if true && len(b) >= 8<<10 {
|
||||
// Split for bigger inputs
|
||||
histogramSplit(b, h)
|
||||
} else {
|
||||
h = h[:256]
|
||||
for _, t := range b {
|
||||
h[t]++
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func histogramSplit(b []byte, h []uint16) {
|
||||
// Tested, and slightly faster than 2-way.
|
||||
// Writing to separate arrays and combining is also slightly slower.
|
||||
h = h[:256]
|
||||
for len(b)&3 != 0 {
|
||||
h[b[0]]++
|
||||
b = b[1:]
|
||||
}
|
||||
n := len(b) / 4
|
||||
x, y, z, w := b[:n], b[n:], b[n+n:], b[n+n+n:]
|
||||
y, z, w = y[:len(x)], z[:len(x)], w[:len(x)]
|
||||
for i, t := range x {
|
||||
v0 := &h[t]
|
||||
v1 := &h[y[i]]
|
||||
v2 := &h[z[i]]
|
||||
v3 := &h[w[i]]
|
||||
*v0++
|
||||
*v1++
|
||||
*v2++
|
||||
*v3++
|
||||
}
|
||||
}
|
||||
Generated
Vendored
-159
@@ -1,159 +0,0 @@
|
||||
// Copyright 2009 The Go Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package flate
|
||||
|
||||
// Sort sorts data.
|
||||
// It makes one call to data.Len to determine n, and O(n*log(n)) calls to
|
||||
// data.Less and data.Swap. The sort is not guaranteed to be stable.
|
||||
func sortByFreq(data []literalNode) {
|
||||
n := len(data)
|
||||
quickSortByFreq(data, 0, n, maxDepth(n))
|
||||
}
|
||||
|
||||
func quickSortByFreq(data []literalNode, a, b, maxDepth int) {
|
||||
for b-a > 12 { // Use ShellSort for slices <= 12 elements
|
||||
if maxDepth == 0 {
|
||||
heapSort(data, a, b)
|
||||
return
|
||||
}
|
||||
maxDepth--
|
||||
mlo, mhi := doPivotByFreq(data, a, b)
|
||||
// Avoiding recursion on the larger subproblem guarantees
|
||||
// a stack depth of at most lg(b-a).
|
||||
if mlo-a < b-mhi {
|
||||
quickSortByFreq(data, a, mlo, maxDepth)
|
||||
a = mhi // i.e., quickSortByFreq(data, mhi, b)
|
||||
} else {
|
||||
quickSortByFreq(data, mhi, b, maxDepth)
|
||||
b = mlo // i.e., quickSortByFreq(data, a, mlo)
|
||||
}
|
||||
}
|
||||
if b-a > 1 {
|
||||
// Do ShellSort pass with gap 6
|
||||
// It could be written in this simplified form cause b-a <= 12
|
||||
for i := a + 6; i < b; i++ {
|
||||
if data[i].freq == data[i-6].freq && data[i].literal < data[i-6].literal || data[i].freq < data[i-6].freq {
|
||||
data[i], data[i-6] = data[i-6], data[i]
|
||||
}
|
||||
}
|
||||
insertionSortByFreq(data, a, b)
|
||||
}
|
||||
}
|
||||
|
||||
func doPivotByFreq(data []literalNode, lo, hi int) (midlo, midhi int) {
|
||||
m := int(uint(lo+hi) >> 1) // Written like this to avoid integer overflow.
|
||||
if hi-lo > 40 {
|
||||
// Tukey's ``Ninther,'' median of three medians of three.
|
||||
s := (hi - lo) / 8
|
||||
medianOfThreeSortByFreq(data, lo, lo+s, lo+2*s)
|
||||
medianOfThreeSortByFreq(data, m, m-s, m+s)
|
||||
medianOfThreeSortByFreq(data, hi-1, hi-1-s, hi-1-2*s)
|
||||
}
|
||||
medianOfThreeSortByFreq(data, lo, m, hi-1)
|
||||
|
||||
// Invariants are:
|
||||
// data[lo] = pivot (set up by ChoosePivot)
|
||||
// data[lo < i < a] < pivot
|
||||
// data[a <= i < b] <= pivot
|
||||
// data[b <= i < c] unexamined
|
||||
// data[c <= i < hi-1] > pivot
|
||||
// data[hi-1] >= pivot
|
||||
pivot := lo
|
||||
a, c := lo+1, hi-1
|
||||
|
||||
for ; a < c && (data[a].freq == data[pivot].freq && data[a].literal < data[pivot].literal || data[a].freq < data[pivot].freq); a++ {
|
||||
}
|
||||
b := a
|
||||
for {
|
||||
for ; b < c && (data[pivot].freq == data[b].freq && data[pivot].literal > data[b].literal || data[pivot].freq > data[b].freq); b++ { // data[b] <= pivot
|
||||
}
|
||||
for ; b < c && (data[pivot].freq == data[c-1].freq && data[pivot].literal < data[c-1].literal || data[pivot].freq < data[c-1].freq); c-- { // data[c-1] > pivot
|
||||
}
|
||||
if b >= c {
|
||||
break
|
||||
}
|
||||
// data[b] > pivot; data[c-1] <= pivot
|
||||
data[b], data[c-1] = data[c-1], data[b]
|
||||
b++
|
||||
c--
|
||||
}
|
||||
// If hi-c<3 then there are duplicates (by property of median of nine).
|
||||
// Let's be a bit more conservative, and set border to 5.
|
||||
protect := hi-c < 5
|
||||
if !protect && hi-c < (hi-lo)/4 {
|
||||
// Lets test some points for equality to pivot
|
||||
dups := 0
|
||||
if data[pivot].freq == data[hi-1].freq && data[pivot].literal > data[hi-1].literal || data[pivot].freq > data[hi-1].freq { // data[hi-1] = pivot
|
||||
data[c], data[hi-1] = data[hi-1], data[c]
|
||||
c++
|
||||
dups++
|
||||
}
|
||||
if data[b-1].freq == data[pivot].freq && data[b-1].literal > data[pivot].literal || data[b-1].freq > data[pivot].freq { // data[b-1] = pivot
|
||||
b--
|
||||
dups++
|
||||
}
|
||||
// m-lo = (hi-lo)/2 > 6
|
||||
// b-lo > (hi-lo)*3/4-1 > 8
|
||||
// ==> m < b ==> data[m] <= pivot
|
||||
if data[m].freq == data[pivot].freq && data[m].literal > data[pivot].literal || data[m].freq > data[pivot].freq { // data[m] = pivot
|
||||
data[m], data[b-1] = data[b-1], data[m]
|
||||
b--
|
||||
dups++
|
||||
}
|
||||
// if at least 2 points are equal to pivot, assume skewed distribution
|
||||
protect = dups > 1
|
||||
}
|
||||
if protect {
|
||||
// Protect against a lot of duplicates
|
||||
// Add invariant:
|
||||
// data[a <= i < b] unexamined
|
||||
// data[b <= i < c] = pivot
|
||||
for {
|
||||
for ; a < b && (data[b-1].freq == data[pivot].freq && data[b-1].literal > data[pivot].literal || data[b-1].freq > data[pivot].freq); b-- { // data[b] == pivot
|
||||
}
|
||||
for ; a < b && (data[a].freq == data[pivot].freq && data[a].literal < data[pivot].literal || data[a].freq < data[pivot].freq); a++ { // data[a] < pivot
|
||||
}
|
||||
if a >= b {
|
||||
break
|
||||
}
|
||||
// data[a] == pivot; data[b-1] < pivot
|
||||
data[a], data[b-1] = data[b-1], data[a]
|
||||
a++
|
||||
b--
|
||||
}
|
||||
}
|
||||
// Swap pivot into middle
|
||||
data[pivot], data[b-1] = data[b-1], data[pivot]
|
||||
return b - 1, c
|
||||
}
|
||||
|
||||
// Insertion sort
|
||||
func insertionSortByFreq(data []literalNode, a, b int) {
|
||||
for i := a + 1; i < b; i++ {
|
||||
for j := i; j > a && (data[j].freq == data[j-1].freq && data[j].literal < data[j-1].literal || data[j].freq < data[j-1].freq); j-- {
|
||||
data[j], data[j-1] = data[j-1], data[j]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// quickSortByFreq, loosely following Bentley and McIlroy,
|
||||
// ``Engineering a Sort Function,'' SP&E November 1993.
|
||||
|
||||
// medianOfThreeSortByFreq moves the median of the three values data[m0], data[m1], data[m2] into data[m1].
|
||||
func medianOfThreeSortByFreq(data []literalNode, m1, m0, m2 int) {
|
||||
// sort 3 elements
|
||||
if data[m1].freq == data[m0].freq && data[m1].literal < data[m0].literal || data[m1].freq < data[m0].freq {
|
||||
data[m1], data[m0] = data[m0], data[m1]
|
||||
}
|
||||
// data[m0] <= data[m1]
|
||||
if data[m2].freq == data[m1].freq && data[m2].literal < data[m1].literal || data[m2].freq < data[m1].freq {
|
||||
data[m2], data[m1] = data[m1], data[m2]
|
||||
// data[m0] <= data[m2] && data[m1] < data[m2]
|
||||
if data[m1].freq == data[m0].freq && data[m1].literal < data[m0].literal || data[m1].freq < data[m0].freq {
|
||||
data[m1], data[m0] = data[m0], data[m1]
|
||||
}
|
||||
}
|
||||
// now data[m0] <= data[m1] <= data[m2]
|
||||
}
|
||||
Generated
Vendored
-201
@@ -1,201 +0,0 @@
|
||||
// Copyright 2009 The Go Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package flate
|
||||
|
||||
// Sort sorts data.
|
||||
// It makes one call to data.Len to determine n, and O(n*log(n)) calls to
|
||||
// data.Less and data.Swap. The sort is not guaranteed to be stable.
|
||||
func sortByLiteral(data []literalNode) {
|
||||
n := len(data)
|
||||
quickSort(data, 0, n, maxDepth(n))
|
||||
}
|
||||
|
||||
func quickSort(data []literalNode, a, b, maxDepth int) {
|
||||
for b-a > 12 { // Use ShellSort for slices <= 12 elements
|
||||
if maxDepth == 0 {
|
||||
heapSort(data, a, b)
|
||||
return
|
||||
}
|
||||
maxDepth--
|
||||
mlo, mhi := doPivot(data, a, b)
|
||||
// Avoiding recursion on the larger subproblem guarantees
|
||||
// a stack depth of at most lg(b-a).
|
||||
if mlo-a < b-mhi {
|
||||
quickSort(data, a, mlo, maxDepth)
|
||||
a = mhi // i.e., quickSort(data, mhi, b)
|
||||
} else {
|
||||
quickSort(data, mhi, b, maxDepth)
|
||||
b = mlo // i.e., quickSort(data, a, mlo)
|
||||
}
|
||||
}
|
||||
if b-a > 1 {
|
||||
// Do ShellSort pass with gap 6
|
||||
// It could be written in this simplified form cause b-a <= 12
|
||||
for i := a + 6; i < b; i++ {
|
||||
if data[i].literal < data[i-6].literal {
|
||||
data[i], data[i-6] = data[i-6], data[i]
|
||||
}
|
||||
}
|
||||
insertionSort(data, a, b)
|
||||
}
|
||||
}
|
||||
func heapSort(data []literalNode, a, b int) {
|
||||
first := a
|
||||
lo := 0
|
||||
hi := b - a
|
||||
|
||||
// Build heap with greatest element at top.
|
||||
for i := (hi - 1) / 2; i >= 0; i-- {
|
||||
siftDown(data, i, hi, first)
|
||||
}
|
||||
|
||||
// Pop elements, largest first, into end of data.
|
||||
for i := hi - 1; i >= 0; i-- {
|
||||
data[first], data[first+i] = data[first+i], data[first]
|
||||
siftDown(data, lo, i, first)
|
||||
}
|
||||
}
|
||||
|
||||
// siftDown implements the heap property on data[lo, hi).
|
||||
// first is an offset into the array where the root of the heap lies.
|
||||
func siftDown(data []literalNode, lo, hi, first int) {
|
||||
root := lo
|
||||
for {
|
||||
child := 2*root + 1
|
||||
if child >= hi {
|
||||
break
|
||||
}
|
||||
if child+1 < hi && data[first+child].literal < data[first+child+1].literal {
|
||||
child++
|
||||
}
|
||||
if data[first+root].literal > data[first+child].literal {
|
||||
return
|
||||
}
|
||||
data[first+root], data[first+child] = data[first+child], data[first+root]
|
||||
root = child
|
||||
}
|
||||
}
|
||||
func doPivot(data []literalNode, lo, hi int) (midlo, midhi int) {
|
||||
m := int(uint(lo+hi) >> 1) // Written like this to avoid integer overflow.
|
||||
if hi-lo > 40 {
|
||||
// Tukey's ``Ninther,'' median of three medians of three.
|
||||
s := (hi - lo) / 8
|
||||
medianOfThree(data, lo, lo+s, lo+2*s)
|
||||
medianOfThree(data, m, m-s, m+s)
|
||||
medianOfThree(data, hi-1, hi-1-s, hi-1-2*s)
|
||||
}
|
||||
medianOfThree(data, lo, m, hi-1)
|
||||
|
||||
// Invariants are:
|
||||
// data[lo] = pivot (set up by ChoosePivot)
|
||||
// data[lo < i < a] < pivot
|
||||
// data[a <= i < b] <= pivot
|
||||
// data[b <= i < c] unexamined
|
||||
// data[c <= i < hi-1] > pivot
|
||||
// data[hi-1] >= pivot
|
||||
pivot := lo
|
||||
a, c := lo+1, hi-1
|
||||
|
||||
for ; a < c && data[a].literal < data[pivot].literal; a++ {
|
||||
}
|
||||
b := a
|
||||
for {
|
||||
for ; b < c && data[pivot].literal > data[b].literal; b++ { // data[b] <= pivot
|
||||
}
|
||||
for ; b < c && data[pivot].literal < data[c-1].literal; c-- { // data[c-1] > pivot
|
||||
}
|
||||
if b >= c {
|
||||
break
|
||||
}
|
||||
// data[b] > pivot; data[c-1] <= pivot
|
||||
data[b], data[c-1] = data[c-1], data[b]
|
||||
b++
|
||||
c--
|
||||
}
|
||||
// If hi-c<3 then there are duplicates (by property of median of nine).
|
||||
// Let's be a bit more conservative, and set border to 5.
|
||||
protect := hi-c < 5
|
||||
if !protect && hi-c < (hi-lo)/4 {
|
||||
// Lets test some points for equality to pivot
|
||||
dups := 0
|
||||
if data[pivot].literal > data[hi-1].literal { // data[hi-1] = pivot
|
||||
data[c], data[hi-1] = data[hi-1], data[c]
|
||||
c++
|
||||
dups++
|
||||
}
|
||||
if data[b-1].literal > data[pivot].literal { // data[b-1] = pivot
|
||||
b--
|
||||
dups++
|
||||
}
|
||||
// m-lo = (hi-lo)/2 > 6
|
||||
// b-lo > (hi-lo)*3/4-1 > 8
|
||||
// ==> m < b ==> data[m] <= pivot
|
||||
if data[m].literal > data[pivot].literal { // data[m] = pivot
|
||||
data[m], data[b-1] = data[b-1], data[m]
|
||||
b--
|
||||
dups++
|
||||
}
|
||||
// if at least 2 points are equal to pivot, assume skewed distribution
|
||||
protect = dups > 1
|
||||
}
|
||||
if protect {
|
||||
// Protect against a lot of duplicates
|
||||
// Add invariant:
|
||||
// data[a <= i < b] unexamined
|
||||
// data[b <= i < c] = pivot
|
||||
for {
|
||||
for ; a < b && data[b-1].literal > data[pivot].literal; b-- { // data[b] == pivot
|
||||
}
|
||||
for ; a < b && data[a].literal < data[pivot].literal; a++ { // data[a] < pivot
|
||||
}
|
||||
if a >= b {
|
||||
break
|
||||
}
|
||||
// data[a] == pivot; data[b-1] < pivot
|
||||
data[a], data[b-1] = data[b-1], data[a]
|
||||
a++
|
||||
b--
|
||||
}
|
||||
}
|
||||
// Swap pivot into middle
|
||||
data[pivot], data[b-1] = data[b-1], data[pivot]
|
||||
return b - 1, c
|
||||
}
|
||||
|
||||
// Insertion sort
|
||||
func insertionSort(data []literalNode, a, b int) {
|
||||
for i := a + 1; i < b; i++ {
|
||||
for j := i; j > a && data[j].literal < data[j-1].literal; j-- {
|
||||
data[j], data[j-1] = data[j-1], data[j]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// maxDepth returns a threshold at which quicksort should switch
|
||||
// to heapsort. It returns 2*ceil(lg(n+1)).
|
||||
func maxDepth(n int) int {
|
||||
var depth int
|
||||
for i := n; i > 0; i >>= 1 {
|
||||
depth++
|
||||
}
|
||||
return depth * 2
|
||||
}
|
||||
|
||||
// medianOfThree moves the median of the three values data[m0], data[m1], data[m2] into data[m1].
|
||||
func medianOfThree(data []literalNode, m1, m0, m2 int) {
|
||||
// sort 3 elements
|
||||
if data[m1].literal < data[m0].literal {
|
||||
data[m1], data[m0] = data[m0], data[m1]
|
||||
}
|
||||
// data[m0] <= data[m1]
|
||||
if data[m2].literal < data[m1].literal {
|
||||
data[m2], data[m1] = data[m1], data[m2]
|
||||
// data[m0] <= data[m2] && data[m1] < data[m2]
|
||||
if data[m1].literal < data[m0].literal {
|
||||
data[m1], data[m0] = data[m0], data[m1]
|
||||
}
|
||||
}
|
||||
// now data[m0] <= data[m1] <= data[m2]
|
||||
}
|
||||
-959
@@ -1,959 +0,0 @@
|
||||
// Copyright 2009 The Go Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
// Package flate implements the DEFLATE compressed data format, described in
|
||||
// RFC 1951. The gzip and zlib packages implement access to DEFLATE-based file
|
||||
// formats.
|
||||
package flate
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"compress/flate"
|
||||
"fmt"
|
||||
"io"
|
||||
"math/bits"
|
||||
"sync"
|
||||
|
||||
"github.com/klauspost/compress/internal/regmask"
|
||||
)
|
||||
|
||||
const (
|
||||
maxCodeLen = 16 // max length of Huffman code
|
||||
maxCodeLenMask = 15 // mask for max length of Huffman code
|
||||
// The next three numbers come from the RFC section 3.2.7, with the
|
||||
// additional proviso in section 3.2.5 which implies that distance codes
|
||||
// 30 and 31 should never occur in compressed data.
|
||||
maxNumLit = 286
|
||||
maxNumDist = 30
|
||||
numCodes = 19 // number of codes in Huffman meta-code
|
||||
|
||||
debugDecode = false
|
||||
)
|
||||
|
||||
// Value of length - 3 and extra bits.
|
||||
type lengthExtra struct {
|
||||
length, extra uint8
|
||||
}
|
||||
|
||||
var decCodeToLen = [32]lengthExtra{{length: 0x0, extra: 0x0}, {length: 0x1, extra: 0x0}, {length: 0x2, extra: 0x0}, {length: 0x3, extra: 0x0}, {length: 0x4, extra: 0x0}, {length: 0x5, extra: 0x0}, {length: 0x6, extra: 0x0}, {length: 0x7, extra: 0x0}, {length: 0x8, extra: 0x1}, {length: 0xa, extra: 0x1}, {length: 0xc, extra: 0x1}, {length: 0xe, extra: 0x1}, {length: 0x10, extra: 0x2}, {length: 0x14, extra: 0x2}, {length: 0x18, extra: 0x2}, {length: 0x1c, extra: 0x2}, {length: 0x20, extra: 0x3}, {length: 0x28, extra: 0x3}, {length: 0x30, extra: 0x3}, {length: 0x38, extra: 0x3}, {length: 0x40, extra: 0x4}, {length: 0x50, extra: 0x4}, {length: 0x60, extra: 0x4}, {length: 0x70, extra: 0x4}, {length: 0x80, extra: 0x5}, {length: 0xa0, extra: 0x5}, {length: 0xc0, extra: 0x5}, {length: 0xe0, extra: 0x5}, {length: 0xff, extra: 0x0}, {length: 0x0, extra: 0x0}, {length: 0x0, extra: 0x0}, {length: 0x0, extra: 0x0}}
|
||||
|
||||
var bitMask32 = [32]uint32{
|
||||
0, 1, 3, 7, 0xF, 0x1F, 0x3F, 0x7F, 0xFF,
|
||||
0x1FF, 0x3FF, 0x7FF, 0xFFF, 0x1FFF, 0x3FFF, 0x7FFF, 0xFFFF,
|
||||
0x1ffff, 0x3ffff, 0x7FFFF, 0xfFFFF, 0x1fFFFF, 0x3fFFFF, 0x7fFFFF, 0xffFFFF,
|
||||
0x1ffFFFF, 0x3ffFFFF, 0x7ffFFFF, 0xfffFFFF, 0x1fffFFFF, 0x3fffFFFF, 0x7fffFFFF,
|
||||
} // up to 32 bits
|
||||
|
||||
// Initialize the fixedHuffmanDecoder only once upon first use.
|
||||
var fixedOnce sync.Once
|
||||
var fixedHuffmanDecoder huffmanDecoder
|
||||
|
||||
// A CorruptInputError reports the presence of corrupt input at a given offset.
|
||||
type CorruptInputError = flate.CorruptInputError
|
||||
|
||||
// An InternalError reports an error in the flate code itself.
|
||||
type InternalError string
|
||||
|
||||
func (e InternalError) Error() string { return "flate: internal error: " + string(e) }
|
||||
|
||||
// A ReadError reports an error encountered while reading input.
|
||||
//
|
||||
// Deprecated: No longer returned.
|
||||
type ReadError = flate.ReadError
|
||||
|
||||
// A WriteError reports an error encountered while writing output.
|
||||
//
|
||||
// Deprecated: No longer returned.
|
||||
type WriteError = flate.WriteError
|
||||
|
||||
// Resetter resets a ReadCloser returned by NewReader or NewReaderDict to
|
||||
// to switch to a new underlying Reader. This permits reusing a ReadCloser
|
||||
// instead of allocating a new one.
|
||||
type Resetter interface {
|
||||
// Reset discards any buffered data and resets the Resetter as if it was
|
||||
// newly initialized with the given reader.
|
||||
Reset(r io.Reader, dict []byte) error
|
||||
}
|
||||
|
||||
// The data structure for decoding Huffman tables is based on that of
|
||||
// zlib. There is a lookup table of a fixed bit width (huffmanChunkBits),
|
||||
// For codes smaller than the table width, there are multiple entries
|
||||
// (each combination of trailing bits has the same value). For codes
|
||||
// larger than the table width, the table contains a link to an overflow
|
||||
// table. The width of each entry in the link table is the maximum code
|
||||
// size minus the chunk width.
|
||||
//
|
||||
// Note that you can do a lookup in the table even without all bits
|
||||
// filled. Since the extra bits are zero, and the DEFLATE Huffman codes
|
||||
// have the property that shorter codes come before longer ones, the
|
||||
// bit length estimate in the result is a lower bound on the actual
|
||||
// number of bits.
|
||||
//
|
||||
// See the following:
|
||||
// http://www.gzip.org/algorithm.txt
|
||||
|
||||
// chunk & 15 is number of bits
|
||||
// chunk >> 4 is value, including table link
|
||||
|
||||
const (
|
||||
huffmanChunkBits = 9
|
||||
huffmanNumChunks = 1 << huffmanChunkBits
|
||||
huffmanCountMask = 15
|
||||
huffmanValueShift = 4
|
||||
)
|
||||
|
||||
type huffmanDecoder struct {
|
||||
maxRead int // the maximum number of bits we can read and not overread
|
||||
chunks *[huffmanNumChunks]uint16 // chunks as described above
|
||||
links [][]uint16 // overflow links
|
||||
linkMask uint32 // mask the width of the link table
|
||||
}
|
||||
|
||||
// Initialize Huffman decoding tables from array of code lengths.
|
||||
// Following this function, h is guaranteed to be initialized into a complete
|
||||
// tree (i.e., neither over-subscribed nor under-subscribed). The exception is a
|
||||
// degenerate case where the tree has only a single symbol with length 1. Empty
|
||||
// trees are permitted.
|
||||
func (h *huffmanDecoder) init(lengths []int) bool {
|
||||
// Sanity enables additional runtime tests during Huffman
|
||||
// table construction. It's intended to be used during
|
||||
// development to supplement the currently ad-hoc unit tests.
|
||||
const sanity = false
|
||||
|
||||
if h.chunks == nil {
|
||||
h.chunks = new([huffmanNumChunks]uint16)
|
||||
}
|
||||
|
||||
if h.maxRead != 0 {
|
||||
*h = huffmanDecoder{chunks: h.chunks, links: h.links}
|
||||
}
|
||||
|
||||
// Count number of codes of each length,
|
||||
// compute maxRead and max length.
|
||||
var count [maxCodeLen]int
|
||||
var min, max int
|
||||
for _, n := range lengths {
|
||||
if n == 0 {
|
||||
continue
|
||||
}
|
||||
if min == 0 || n < min {
|
||||
min = n
|
||||
}
|
||||
if n > max {
|
||||
max = n
|
||||
}
|
||||
count[n&maxCodeLenMask]++
|
||||
}
|
||||
|
||||
// Empty tree. The decompressor.huffSym function will fail later if the tree
|
||||
// is used. Technically, an empty tree is only valid for the HDIST tree and
|
||||
// not the HCLEN and HLIT tree. However, a stream with an empty HCLEN tree
|
||||
// is guaranteed to fail since it will attempt to use the tree to decode the
|
||||
// codes for the HLIT and HDIST trees. Similarly, an empty HLIT tree is
|
||||
// guaranteed to fail later since the compressed data section must be
|
||||
// composed of at least one symbol (the end-of-block marker).
|
||||
if max == 0 {
|
||||
return true
|
||||
}
|
||||
|
||||
code := 0
|
||||
var nextcode [maxCodeLen]int
|
||||
for i := min; i <= max; i++ {
|
||||
code <<= 1
|
||||
nextcode[i&maxCodeLenMask] = code
|
||||
code += count[i&maxCodeLenMask]
|
||||
}
|
||||
|
||||
// Check that the coding is complete (i.e., that we've
|
||||
// assigned all 2-to-the-max possible bit sequences).
|
||||
// Exception: To be compatible with zlib, we also need to
|
||||
// accept degenerate single-code codings. See also
|
||||
// TestDegenerateHuffmanCoding.
|
||||
if code != 1<<uint(max) && !(code == 1 && max == 1) {
|
||||
if debugDecode {
|
||||
fmt.Println("coding failed, code, max:", code, max, code == 1<<uint(max), code == 1 && max == 1, "(one should be true)")
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
h.maxRead = min
|
||||
|
||||
chunks := h.chunks[:]
|
||||
for i := range chunks {
|
||||
chunks[i] = 0
|
||||
}
|
||||
|
||||
if max > huffmanChunkBits {
|
||||
numLinks := 1 << (uint(max) - huffmanChunkBits)
|
||||
h.linkMask = uint32(numLinks - 1)
|
||||
|
||||
// create link tables
|
||||
link := nextcode[huffmanChunkBits+1] >> 1
|
||||
if cap(h.links) < huffmanNumChunks-link {
|
||||
h.links = make([][]uint16, huffmanNumChunks-link)
|
||||
} else {
|
||||
h.links = h.links[:huffmanNumChunks-link]
|
||||
}
|
||||
for j := uint(link); j < huffmanNumChunks; j++ {
|
||||
reverse := int(bits.Reverse16(uint16(j)))
|
||||
reverse >>= uint(16 - huffmanChunkBits)
|
||||
off := j - uint(link)
|
||||
if sanity && h.chunks[reverse] != 0 {
|
||||
panic("impossible: overwriting existing chunk")
|
||||
}
|
||||
h.chunks[reverse] = uint16(off<<huffmanValueShift | (huffmanChunkBits + 1))
|
||||
if cap(h.links[off]) < numLinks {
|
||||
h.links[off] = make([]uint16, numLinks)
|
||||
} else {
|
||||
h.links[off] = h.links[off][:numLinks]
|
||||
}
|
||||
}
|
||||
} else {
|
||||
h.links = h.links[:0]
|
||||
}
|
||||
|
||||
for i, n := range lengths {
|
||||
if n == 0 {
|
||||
continue
|
||||
}
|
||||
code := nextcode[n]
|
||||
nextcode[n]++
|
||||
chunk := uint16(i<<huffmanValueShift | n)
|
||||
reverse := int(bits.Reverse16(uint16(code)))
|
||||
reverse >>= uint(16 - n)
|
||||
if n <= huffmanChunkBits {
|
||||
for off := reverse; off < len(h.chunks); off += 1 << uint(n) {
|
||||
// We should never need to overwrite
|
||||
// an existing chunk. Also, 0 is
|
||||
// never a valid chunk, because the
|
||||
// lower 4 "count" bits should be
|
||||
// between 1 and 15.
|
||||
if sanity && h.chunks[off] != 0 {
|
||||
panic("impossible: overwriting existing chunk")
|
||||
}
|
||||
h.chunks[off] = chunk
|
||||
}
|
||||
} else {
|
||||
j := reverse & (huffmanNumChunks - 1)
|
||||
if sanity && h.chunks[j]&huffmanCountMask != huffmanChunkBits+1 {
|
||||
// Longer codes should have been
|
||||
// associated with a link table above.
|
||||
panic("impossible: not an indirect chunk")
|
||||
}
|
||||
value := h.chunks[j] >> huffmanValueShift
|
||||
linktab := h.links[value]
|
||||
reverse >>= huffmanChunkBits
|
||||
for off := reverse; off < len(linktab); off += 1 << uint(n-huffmanChunkBits) {
|
||||
if sanity && linktab[off] != 0 {
|
||||
panic("impossible: overwriting existing chunk")
|
||||
}
|
||||
linktab[off] = chunk
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if sanity {
|
||||
// Above we've sanity checked that we never overwrote
|
||||
// an existing entry. Here we additionally check that
|
||||
// we filled the tables completely.
|
||||
for i, chunk := range h.chunks {
|
||||
if chunk == 0 {
|
||||
// As an exception, in the degenerate
|
||||
// single-code case, we allow odd
|
||||
// chunks to be missing.
|
||||
if code == 1 && i%2 == 1 {
|
||||
continue
|
||||
}
|
||||
panic("impossible: missing chunk")
|
||||
}
|
||||
}
|
||||
for _, linktab := range h.links {
|
||||
for _, chunk := range linktab {
|
||||
if chunk == 0 {
|
||||
panic("impossible: missing chunk")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
// Reader is the actual read interface needed by NewReader.
|
||||
// If the passed in io.Reader does not also have ReadByte,
|
||||
// the NewReader will introduce its own buffering.
|
||||
type Reader interface {
|
||||
io.Reader
|
||||
io.ByteReader
|
||||
}
|
||||
|
||||
type step uint8
|
||||
|
||||
const (
|
||||
copyData step = iota + 1
|
||||
nextBlock
|
||||
huffmanBytesBuffer
|
||||
huffmanBytesReader
|
||||
huffmanBufioReader
|
||||
huffmanStringsReader
|
||||
huffmanGenericReader
|
||||
)
|
||||
|
||||
// flushMode tells decompressor when to return data
|
||||
type flushMode uint8
|
||||
|
||||
const (
|
||||
syncFlush flushMode = iota // return data after sync flush block
|
||||
partialFlush // return data after each block
|
||||
)
|
||||
|
||||
// Decompress state.
|
||||
type decompressor struct {
|
||||
// Input source.
|
||||
r Reader
|
||||
roffset int64
|
||||
|
||||
// Huffman decoders for literal/length, distance.
|
||||
h1, h2 huffmanDecoder
|
||||
|
||||
// Length arrays used to define Huffman codes.
|
||||
bits *[maxNumLit + maxNumDist]int
|
||||
codebits *[numCodes]int
|
||||
|
||||
// Output history, buffer.
|
||||
dict dictDecoder
|
||||
|
||||
// Next step in the decompression,
|
||||
// and decompression state.
|
||||
step step
|
||||
stepState int
|
||||
err error
|
||||
toRead []byte
|
||||
hl, hd *huffmanDecoder
|
||||
copyLen int
|
||||
copyDist int
|
||||
|
||||
// Temporary buffer (avoids repeated allocation).
|
||||
buf [4]byte
|
||||
|
||||
// Input bits, in top of b.
|
||||
b uint32
|
||||
|
||||
nb uint
|
||||
final bool
|
||||
|
||||
flushMode flushMode
|
||||
cb func(InflateCheckpoint)
|
||||
cp InflateCheckpoint
|
||||
hasCP bool // WithResumeFrom was supplied
|
||||
uncOffset int64 // baseline uncompressed offset (from a resume checkpoint)
|
||||
cpBuf []byte
|
||||
}
|
||||
|
||||
func (f *decompressor) nextBlock() {
|
||||
for f.nb < 1+2 {
|
||||
if f.err = f.moreBits(); f.err != nil {
|
||||
return
|
||||
}
|
||||
}
|
||||
f.final = f.b&1 == 1
|
||||
f.b >>= 1
|
||||
typ := f.b & 3
|
||||
f.b >>= 2
|
||||
f.nb -= 1 + 2
|
||||
switch typ {
|
||||
case 0:
|
||||
f.dataBlock()
|
||||
if debugDecode {
|
||||
fmt.Println("stored block")
|
||||
}
|
||||
case 1:
|
||||
// compressed, fixed Huffman tables
|
||||
f.hl = &fixedHuffmanDecoder
|
||||
f.hd = nil
|
||||
f.huffmanBlockDecoder()
|
||||
if debugDecode {
|
||||
fmt.Println("predefinied huffman block")
|
||||
}
|
||||
case 2:
|
||||
// compressed, dynamic Huffman tables
|
||||
if f.err = f.readHuffman(); f.err != nil {
|
||||
break
|
||||
}
|
||||
f.hl = &f.h1
|
||||
f.hd = &f.h2
|
||||
f.huffmanBlockDecoder()
|
||||
if debugDecode {
|
||||
fmt.Println("dynamic huffman block")
|
||||
}
|
||||
default:
|
||||
// 3 is reserved.
|
||||
if debugDecode {
|
||||
fmt.Println("reserved data block encountered")
|
||||
}
|
||||
f.err = CorruptInputError(f.roffset)
|
||||
}
|
||||
}
|
||||
|
||||
func (f *decompressor) Read(b []byte) (int, error) {
|
||||
for {
|
||||
if len(f.toRead) > 0 {
|
||||
n := copy(b, f.toRead)
|
||||
f.toRead = f.toRead[n:]
|
||||
if len(f.toRead) == 0 {
|
||||
return n, f.err
|
||||
}
|
||||
return n, nil
|
||||
}
|
||||
if f.err != nil {
|
||||
return 0, f.err
|
||||
}
|
||||
|
||||
f.doStep()
|
||||
|
||||
if f.err != nil && len(f.toRead) == 0 {
|
||||
f.toRead = f.dict.readFlush() // Flush what's left in case of error
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// WriteTo implements the io.WriteTo interface for io.Copy and friends.
|
||||
func (f *decompressor) WriteTo(w io.Writer) (int64, error) {
|
||||
total := int64(0)
|
||||
flushed := false
|
||||
for {
|
||||
if len(f.toRead) > 0 {
|
||||
n, err := w.Write(f.toRead)
|
||||
total += int64(n)
|
||||
if err != nil {
|
||||
f.err = err
|
||||
return total, err
|
||||
}
|
||||
if n != len(f.toRead) {
|
||||
return total, io.ErrShortWrite
|
||||
}
|
||||
f.toRead = f.toRead[:0]
|
||||
}
|
||||
if f.err != nil && flushed {
|
||||
if f.err == io.EOF {
|
||||
return total, nil
|
||||
}
|
||||
return total, f.err
|
||||
}
|
||||
if f.err == nil {
|
||||
f.doStep()
|
||||
}
|
||||
if len(f.toRead) == 0 && f.err != nil && !flushed {
|
||||
f.toRead = f.dict.readFlush() // Flush what's left in case of error
|
||||
flushed = true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (f *decompressor) Close() error {
|
||||
if f.err == io.EOF {
|
||||
return nil
|
||||
}
|
||||
return f.err
|
||||
}
|
||||
|
||||
// RFC 1951 section 3.2.7.
|
||||
// Compression with dynamic Huffman codes
|
||||
|
||||
var codeOrder = [...]int{16, 17, 18, 0, 8, 7, 9, 6, 10, 5, 11, 4, 12, 3, 13, 2, 14, 1, 15}
|
||||
|
||||
func (f *decompressor) readHuffman() error {
|
||||
// HLIT[5], HDIST[5], HCLEN[4].
|
||||
for f.nb < 5+5+4 {
|
||||
if err := f.moreBits(); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
nlit := int(f.b&0x1F) + 257
|
||||
if nlit > maxNumLit {
|
||||
if debugDecode {
|
||||
fmt.Println("nlit > maxNumLit", nlit)
|
||||
}
|
||||
return CorruptInputError(f.roffset)
|
||||
}
|
||||
f.b >>= 5
|
||||
ndist := int(f.b&0x1F) + 1
|
||||
if ndist > maxNumDist {
|
||||
if debugDecode {
|
||||
fmt.Println("ndist > maxNumDist", ndist)
|
||||
}
|
||||
return CorruptInputError(f.roffset)
|
||||
}
|
||||
f.b >>= 5
|
||||
nclen := int(f.b&0xF) + 4
|
||||
// numCodes is 19, so nclen is always valid.
|
||||
f.b >>= 4
|
||||
f.nb -= 5 + 5 + 4
|
||||
|
||||
// (HCLEN+4)*3 bits: code lengths in the magic codeOrder order.
|
||||
for i := range nclen {
|
||||
for f.nb < 3 {
|
||||
if err := f.moreBits(); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
f.codebits[codeOrder[i]] = int(f.b & 0x7)
|
||||
f.b >>= 3
|
||||
f.nb -= 3
|
||||
}
|
||||
for i := nclen; i < len(codeOrder); i++ {
|
||||
f.codebits[codeOrder[i]] = 0
|
||||
}
|
||||
if !f.h1.init(f.codebits[0:]) {
|
||||
if debugDecode {
|
||||
fmt.Println("init codebits failed")
|
||||
}
|
||||
return CorruptInputError(f.roffset)
|
||||
}
|
||||
|
||||
// HLIT + 257 code lengths, HDIST + 1 code lengths,
|
||||
// using the code length Huffman code.
|
||||
for i, n := 0, nlit+ndist; i < n; {
|
||||
x, err := f.huffSym(&f.h1)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if x < 16 {
|
||||
// Actual length.
|
||||
f.bits[i] = x
|
||||
i++
|
||||
continue
|
||||
}
|
||||
// Repeat previous length or zero.
|
||||
var rep int
|
||||
var nb uint
|
||||
var b int
|
||||
switch x {
|
||||
default:
|
||||
return InternalError("unexpected length code")
|
||||
case 16:
|
||||
rep = 3
|
||||
nb = 2
|
||||
if i == 0 {
|
||||
if debugDecode {
|
||||
fmt.Println("i==0")
|
||||
}
|
||||
return CorruptInputError(f.roffset)
|
||||
}
|
||||
b = f.bits[i-1]
|
||||
case 17:
|
||||
rep = 3
|
||||
nb = 3
|
||||
b = 0
|
||||
case 18:
|
||||
rep = 11
|
||||
nb = 7
|
||||
b = 0
|
||||
}
|
||||
for f.nb < nb {
|
||||
if err := f.moreBits(); err != nil {
|
||||
if debugDecode {
|
||||
fmt.Println("morebits:", err)
|
||||
}
|
||||
return err
|
||||
}
|
||||
}
|
||||
rep += int(f.b & uint32(1<<(nb®mask.Shift32ByUint)-1))
|
||||
f.b >>= nb & regmask.Shift32ByUint
|
||||
f.nb -= nb
|
||||
if i+rep > n {
|
||||
if debugDecode {
|
||||
fmt.Println("i+rep > n", i, rep, n)
|
||||
}
|
||||
return CorruptInputError(f.roffset)
|
||||
}
|
||||
for j := 0; j < rep; j++ {
|
||||
f.bits[i] = b
|
||||
i++
|
||||
}
|
||||
}
|
||||
|
||||
if !f.h1.init(f.bits[0:nlit]) || !f.h2.init(f.bits[nlit:nlit+ndist]) {
|
||||
if debugDecode {
|
||||
fmt.Println("init2 failed")
|
||||
}
|
||||
return CorruptInputError(f.roffset)
|
||||
}
|
||||
|
||||
// As an optimization, we can initialize the maxRead bits to read at a time
|
||||
// for the HLIT tree to the length of the EOB marker since we know that
|
||||
// every block must terminate with one. This preserves the property that
|
||||
// we never read any extra bytes after the end of the DEFLATE stream.
|
||||
if f.h1.maxRead < f.bits[endBlockMarker] {
|
||||
f.h1.maxRead = f.bits[endBlockMarker]
|
||||
}
|
||||
if !f.final {
|
||||
// If not the final block, the smallest block possible is
|
||||
// a predefined table, BTYPE=01, with a single EOB marker.
|
||||
// This will take up 3 + 7 bits.
|
||||
f.h1.maxRead += 10
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Copy a single uncompressed data block from input to output.
|
||||
func (f *decompressor) dataBlock() {
|
||||
// Uncompressed.
|
||||
// Discard current half-byte.
|
||||
left := (f.nb) & 7
|
||||
f.nb -= left
|
||||
f.b >>= left
|
||||
|
||||
offBytes := f.nb >> 3
|
||||
// Unfilled values will be overwritten.
|
||||
f.buf[0] = uint8(f.b)
|
||||
f.buf[1] = uint8(f.b >> 8)
|
||||
f.buf[2] = uint8(f.b >> 16)
|
||||
f.buf[3] = uint8(f.b >> 24)
|
||||
|
||||
f.roffset += int64(offBytes)
|
||||
f.nb, f.b = 0, 0
|
||||
|
||||
// Length then ones-complement of length.
|
||||
nr, err := io.ReadFull(f.r, f.buf[offBytes:4])
|
||||
f.roffset += int64(nr)
|
||||
if err != nil {
|
||||
f.err = noEOF(err)
|
||||
return
|
||||
}
|
||||
n := uint16(f.buf[0]) | uint16(f.buf[1])<<8
|
||||
nn := uint16(f.buf[2]) | uint16(f.buf[3])<<8
|
||||
if nn != ^n {
|
||||
if debugDecode {
|
||||
ncomp := ^n
|
||||
fmt.Println("uint16(nn) != uint16(^n)", nn, ncomp)
|
||||
}
|
||||
f.err = CorruptInputError(f.roffset)
|
||||
return
|
||||
}
|
||||
|
||||
if n == 0 {
|
||||
if f.flushMode == syncFlush {
|
||||
f.toRead = f.dict.readFlush()
|
||||
}
|
||||
|
||||
f.finishBlock()
|
||||
return
|
||||
}
|
||||
|
||||
f.copyLen = int(n)
|
||||
f.copyData()
|
||||
}
|
||||
|
||||
// copyData copies f.copyLen bytes from the underlying reader into f.hist.
|
||||
// It pauses for reads when f.hist is full.
|
||||
func (f *decompressor) copyData() {
|
||||
buf := f.dict.writeSlice()
|
||||
if len(buf) > f.copyLen {
|
||||
buf = buf[:f.copyLen]
|
||||
}
|
||||
|
||||
cnt, err := io.ReadFull(f.r, buf)
|
||||
f.roffset += int64(cnt)
|
||||
f.copyLen -= cnt
|
||||
f.dict.writeMark(cnt)
|
||||
if err != nil {
|
||||
f.err = noEOF(err)
|
||||
return
|
||||
}
|
||||
|
||||
if f.dict.availWrite() == 0 || f.copyLen > 0 {
|
||||
f.toRead = f.dict.readFlush()
|
||||
f.step = copyData
|
||||
return
|
||||
}
|
||||
f.finishBlock()
|
||||
}
|
||||
|
||||
func (f *decompressor) finishBlock() {
|
||||
if f.final {
|
||||
if f.dict.availRead() > 0 {
|
||||
f.toRead = f.dict.readFlush()
|
||||
}
|
||||
|
||||
f.err = io.EOF
|
||||
} else if f.flushMode == partialFlush && f.dict.availRead() > 0 {
|
||||
f.toRead = f.dict.readFlush()
|
||||
}
|
||||
|
||||
if f.cb != nil {
|
||||
bitPos := f.roffset*8 - int64(f.nb)
|
||||
f.cpBuf = f.dict.appendWindow(f.cpBuf[:0])
|
||||
f.cb(InflateCheckpoint{
|
||||
UncompressedOffset: f.uncOffset + f.dict.decoded(),
|
||||
CompressedOffset: bitPos / 8,
|
||||
Final: f.final,
|
||||
BitOffset: uint8(bitPos & 7),
|
||||
Window: f.cpBuf,
|
||||
})
|
||||
}
|
||||
|
||||
f.step = nextBlock
|
||||
}
|
||||
|
||||
func (f *decompressor) doStep() {
|
||||
switch f.step {
|
||||
case copyData:
|
||||
f.copyData()
|
||||
case nextBlock:
|
||||
f.nextBlock()
|
||||
case huffmanBytesBuffer:
|
||||
f.huffmanBytesBuffer()
|
||||
case huffmanBytesReader:
|
||||
f.huffmanBytesReader()
|
||||
case huffmanBufioReader:
|
||||
f.huffmanBufioReader()
|
||||
case huffmanStringsReader:
|
||||
f.huffmanStringsReader()
|
||||
case huffmanGenericReader:
|
||||
f.huffmanGenericReader()
|
||||
default:
|
||||
panic("BUG: unexpected step state")
|
||||
}
|
||||
}
|
||||
|
||||
// noEOF returns err, unless err == io.EOF, in which case it returns io.ErrUnexpectedEOF.
|
||||
func noEOF(e error) error {
|
||||
if e == io.EOF {
|
||||
return io.ErrUnexpectedEOF
|
||||
}
|
||||
return e
|
||||
}
|
||||
|
||||
func (f *decompressor) moreBits() error {
|
||||
c, err := f.r.ReadByte()
|
||||
if err != nil {
|
||||
return noEOF(err)
|
||||
}
|
||||
f.roffset++
|
||||
f.b |= uint32(c) << (f.nb & regmask.Shift32ByUint)
|
||||
f.nb += 8
|
||||
return nil
|
||||
}
|
||||
|
||||
// Read the next Huffman-encoded symbol from f according to h.
|
||||
func (f *decompressor) huffSym(h *huffmanDecoder) (int, error) {
|
||||
// Since a huffmanDecoder can be empty or be composed of a degenerate tree
|
||||
// with single element, huffSym must error on these two edge cases. In both
|
||||
// cases, the chunks slice will be 0 for the invalid sequence, leading it
|
||||
// satisfy the n == 0 check below.
|
||||
n := uint(h.maxRead)
|
||||
// Optimization. Compiler isn't smart enough to keep f.b,f.nb in registers,
|
||||
// but is smart enough to keep local variables in registers, so use nb and b,
|
||||
// inline call to moreBits and reassign b,nb back to f on return.
|
||||
nb, b := f.nb, f.b
|
||||
for {
|
||||
for nb < n {
|
||||
c, err := f.r.ReadByte()
|
||||
if err != nil {
|
||||
f.b = b
|
||||
f.nb = nb
|
||||
return 0, noEOF(err)
|
||||
}
|
||||
f.roffset++
|
||||
b |= uint32(c) << (nb & regmask.Shift32ByUint)
|
||||
nb += 8
|
||||
}
|
||||
chunk := h.chunks[b&(huffmanNumChunks-1)]
|
||||
n = uint(chunk & huffmanCountMask)
|
||||
if n > huffmanChunkBits {
|
||||
chunk = h.links[chunk>>huffmanValueShift][(b>>huffmanChunkBits)&h.linkMask]
|
||||
n = uint(chunk & huffmanCountMask)
|
||||
}
|
||||
if n <= nb {
|
||||
if n == 0 {
|
||||
f.b = b
|
||||
f.nb = nb
|
||||
if debugDecode {
|
||||
fmt.Println("huffsym: n==0")
|
||||
}
|
||||
f.err = CorruptInputError(f.roffset)
|
||||
return 0, f.err
|
||||
}
|
||||
f.b = b >> (n & regmask.Shift32ByUint)
|
||||
f.nb = nb - n
|
||||
return int(chunk >> huffmanValueShift), nil
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func makeReader(r io.Reader) Reader {
|
||||
if rr, ok := r.(Reader); ok {
|
||||
return rr
|
||||
}
|
||||
return bufio.NewReader(r)
|
||||
}
|
||||
|
||||
func fixedHuffmanDecoderInit() {
|
||||
fixedOnce.Do(func() {
|
||||
// These come from the RFC section 3.2.6.
|
||||
var bits [288]int
|
||||
for i := range 144 {
|
||||
bits[i] = 8
|
||||
}
|
||||
for i := 144; i < 256; i++ {
|
||||
bits[i] = 9
|
||||
}
|
||||
for i := 256; i < 280; i++ {
|
||||
bits[i] = 7
|
||||
}
|
||||
for i := 280; i < 288; i++ {
|
||||
bits[i] = 8
|
||||
}
|
||||
fixedHuffmanDecoder.init(bits[:])
|
||||
})
|
||||
}
|
||||
|
||||
func (f *decompressor) Reset(r io.Reader, dict []byte) error {
|
||||
*f = decompressor{
|
||||
r: makeReader(r),
|
||||
bits: f.bits,
|
||||
codebits: f.codebits,
|
||||
h1: f.h1,
|
||||
h2: f.h2,
|
||||
dict: f.dict,
|
||||
step: nextBlock,
|
||||
}
|
||||
f.dict.init(maxMatchOffset, dict)
|
||||
return nil
|
||||
}
|
||||
|
||||
// ResetCP will adjust the input to the provided checkpoint.
|
||||
// It is assumed the input stream is forwarded to cp.CompressedOffset.
|
||||
func (f *decompressor) ResetCP(r io.Reader, cp InflateCheckpoint) error {
|
||||
*f = decompressor{
|
||||
r: makeReader(r),
|
||||
bits: f.bits,
|
||||
codebits: f.codebits,
|
||||
h1: f.h1,
|
||||
h2: f.h2,
|
||||
dict: f.dict,
|
||||
step: nextBlock,
|
||||
cpBuf: f.cpBuf,
|
||||
}
|
||||
return f.applyCP(cp)
|
||||
}
|
||||
|
||||
// applyCP seeds the decompressor state from a resume checkpoint:
|
||||
// loads the sliding window, sets the absolute compressed/uncompressed
|
||||
// offsets, and skips cp.BitOffset bits into the first input byte so
|
||||
// the next decode aligns with the start of a deflate block.
|
||||
func (f *decompressor) applyCP(cp InflateCheckpoint) error {
|
||||
f.dict.init(maxMatchOffset, cp.Window)
|
||||
f.roffset = cp.CompressedOffset
|
||||
f.uncOffset = cp.UncompressedOffset
|
||||
f.final = cp.Final
|
||||
f.b = 0
|
||||
f.nb = 0
|
||||
if cp.BitOffset > 0 {
|
||||
c, err := f.r.ReadByte()
|
||||
if err != nil {
|
||||
return noEOF(err)
|
||||
}
|
||||
f.roffset++
|
||||
f.b = uint32(c) >> cp.BitOffset
|
||||
f.nb = 8 - uint(cp.BitOffset)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type ReaderOpt func(*decompressor)
|
||||
|
||||
// WithPartialBlock tells decompressor to return after each block,
|
||||
// so it can read data written with partial flush
|
||||
func WithPartialBlock() ReaderOpt {
|
||||
return func(f *decompressor) {
|
||||
f.flushMode = partialFlush
|
||||
}
|
||||
}
|
||||
|
||||
// WithDict initializes the reader with a preset dictionary
|
||||
func WithDict(dict []byte) ReaderOpt {
|
||||
return func(f *decompressor) {
|
||||
f.dict.init(maxMatchOffset, dict)
|
||||
}
|
||||
}
|
||||
|
||||
// InflateCheckpoint provides a resumable checkpoint for inflate.
|
||||
type InflateCheckpoint struct {
|
||||
UncompressedOffset int64 // Byte offset in the decompressed stream
|
||||
CompressedOffset int64 // Byte offset in the compressed stream
|
||||
Final bool // True if this is the final block
|
||||
BitOffset uint8 // 0-7 bits
|
||||
Window []byte // 32KB sliding window dictionary
|
||||
}
|
||||
|
||||
// WithEobCallback will call the provided function after each block
|
||||
// with the current gzip checkpoint.
|
||||
// After returning the provided window can no longer be referenced.
|
||||
// The callback will not be triggered after a block is marked "final".
|
||||
// The callback is not retained after Reset.
|
||||
func WithEobCallback(cb func(InflateCheckpoint)) ReaderOpt {
|
||||
return func(f *decompressor) {
|
||||
f.cb = cb
|
||||
}
|
||||
}
|
||||
|
||||
// WithResumeFrom will adjust the input to the provided checkpoint.
|
||||
// It is assumed the input stream is forwarded to the provided offset.
|
||||
// The checkpoint is removed when Reset is called.
|
||||
func WithResumeFrom(cp InflateCheckpoint) ReaderOpt {
|
||||
return func(f *decompressor) {
|
||||
f.cp = cp
|
||||
f.hasCP = true
|
||||
}
|
||||
}
|
||||
|
||||
// NewReaderOpts returns new reader with provided options
|
||||
func NewReaderOpts(r io.Reader, opts ...ReaderOpt) io.ReadCloser {
|
||||
fixedHuffmanDecoderInit()
|
||||
|
||||
var f decompressor
|
||||
f.r = makeReader(r)
|
||||
f.bits = new([maxNumLit + maxNumDist]int)
|
||||
f.codebits = new([numCodes]int)
|
||||
f.step = nextBlock
|
||||
f.dict.init(maxMatchOffset, nil)
|
||||
|
||||
for _, opt := range opts {
|
||||
opt(&f)
|
||||
}
|
||||
|
||||
if f.hasCP {
|
||||
if err := f.applyCP(f.cp); err != nil {
|
||||
f.err = err
|
||||
}
|
||||
}
|
||||
|
||||
return &f
|
||||
}
|
||||
|
||||
// NewReader returns a new ReadCloser that can be used
|
||||
// to read the uncompressed version of r.
|
||||
// If r does not also implement io.ByteReader,
|
||||
// the decompressor may read more data than necessary from r.
|
||||
// It is the caller's responsibility to call Close on the ReadCloser
|
||||
// when finished reading.
|
||||
//
|
||||
// The ReadCloser returned by NewReader also implements Resetter.
|
||||
func NewReader(r io.Reader) io.ReadCloser {
|
||||
return NewReaderOpts(r)
|
||||
}
|
||||
|
||||
// NewReaderDict is like NewReader but initializes the reader
|
||||
// with a preset dictionary. The returned Reader behaves as if
|
||||
// the uncompressed data stream started with the given dictionary,
|
||||
// which has already been read. NewReaderDict is typically used
|
||||
// to read data compressed by NewWriterDict.
|
||||
//
|
||||
// The ReadCloser returned by NewReader also implements Resetter.
|
||||
func NewReaderDict(r io.Reader, dict []byte) io.ReadCloser {
|
||||
return NewReaderOpts(r, WithDict(dict))
|
||||
}
|
||||
Generated
Vendored
-1339
File diff suppressed because it is too large
Load Diff
-215
@@ -1,215 +0,0 @@
|
||||
package flate
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"github.com/klauspost/compress/internal/le"
|
||||
)
|
||||
|
||||
// fastGen maintains the table for matches,
|
||||
// and the previous byte block for level 2.
|
||||
// This is the generic implementation.
|
||||
type fastEncL1 struct {
|
||||
fastGen
|
||||
table [tableSize]tableEntry
|
||||
}
|
||||
|
||||
// EncodeL1 uses a similar algorithm to level 1
|
||||
func (e *fastEncL1) Encode(dst *tokens, src []byte) {
|
||||
const (
|
||||
inputMargin = 12 - 1
|
||||
minNonLiteralBlockSize = 1 + 1 + inputMargin
|
||||
hashBytes = 5
|
||||
)
|
||||
if debugDeflate && e.cur < 0 {
|
||||
panic(fmt.Sprint("e.cur < 0: ", e.cur))
|
||||
}
|
||||
|
||||
// Protect against e.cur wraparound.
|
||||
for e.cur >= bufferReset {
|
||||
if len(e.hist) == 0 {
|
||||
for i := range e.table[:] {
|
||||
e.table[i] = tableEntry{}
|
||||
}
|
||||
e.cur = maxMatchOffset
|
||||
break
|
||||
}
|
||||
// Shift down everything in the table that isn't already too far away.
|
||||
minOff := e.cur + int32(len(e.hist)) - maxMatchOffset
|
||||
for i := range e.table[:] {
|
||||
v := e.table[i].offset
|
||||
if v <= minOff {
|
||||
v = 0
|
||||
} else {
|
||||
v = v - e.cur + maxMatchOffset
|
||||
}
|
||||
e.table[i].offset = v
|
||||
}
|
||||
e.cur = maxMatchOffset
|
||||
}
|
||||
|
||||
s := e.addBlock(src)
|
||||
|
||||
// This check isn't in the Snappy implementation, but there, the caller
|
||||
// instead of the callee handles this case.
|
||||
if len(src) < minNonLiteralBlockSize {
|
||||
// We do not fill the token table.
|
||||
// This will be picked up by caller.
|
||||
dst.n = uint16(len(src))
|
||||
return
|
||||
}
|
||||
|
||||
// Override src
|
||||
src = e.hist
|
||||
nextEmit := s
|
||||
|
||||
// sLimit is when to stop looking for offset/length copies. The inputMargin
|
||||
// lets us use a fast path for emitLiteral in the main loop, while we are
|
||||
// looking for copies.
|
||||
sLimit := int32(len(src) - inputMargin)
|
||||
|
||||
// nextEmit is where in src the next emitLiteral should start from.
|
||||
cv := load6432(src, s)
|
||||
|
||||
for {
|
||||
const skipLog = 5
|
||||
const doEvery = 2
|
||||
|
||||
nextS := s
|
||||
var candidate tableEntry
|
||||
var t int32
|
||||
for {
|
||||
nextHash := hashLen(cv, tableBits, hashBytes)
|
||||
candidate = e.table[nextHash]
|
||||
nextS = s + doEvery + (s-nextEmit)>>skipLog
|
||||
if nextS > sLimit {
|
||||
goto emitRemainder
|
||||
}
|
||||
|
||||
now := load6432(src, nextS)
|
||||
e.table[nextHash] = tableEntry{offset: s + e.cur}
|
||||
nextHash = hashLen(now, tableBits, hashBytes)
|
||||
t = candidate.offset - e.cur
|
||||
if s-t < maxMatchOffset && uint32(cv) == load3232(src, t) {
|
||||
e.table[nextHash] = tableEntry{offset: nextS + e.cur}
|
||||
break
|
||||
}
|
||||
|
||||
// Do one right away...
|
||||
cv = now
|
||||
s = nextS
|
||||
nextS++
|
||||
candidate = e.table[nextHash]
|
||||
now >>= 8
|
||||
e.table[nextHash] = tableEntry{offset: s + e.cur}
|
||||
|
||||
t = candidate.offset - e.cur
|
||||
if s-t < maxMatchOffset && uint32(cv) == load3232(src, t) {
|
||||
e.table[nextHash] = tableEntry{offset: nextS + e.cur}
|
||||
break
|
||||
}
|
||||
cv = now
|
||||
s = nextS
|
||||
}
|
||||
|
||||
// A 4-byte match has been found. We'll later see if more than 4 bytes
|
||||
// match. But, prior to the match, src[nextEmit:s] are unmatched. Emit
|
||||
// them as literal bytes.
|
||||
for {
|
||||
// Invariant: we have a 4-byte match at s, and no need to emit any
|
||||
// literal bytes prior to s.
|
||||
|
||||
// Extend the 4-byte match as long as possible.
|
||||
l := e.matchlenLong(int(s+4), int(t+4), src) + 4
|
||||
|
||||
// Extend backwards
|
||||
for t > 0 && s > nextEmit && le.Load8(src, t-1) == le.Load8(src, s-1) {
|
||||
s--
|
||||
t--
|
||||
l++
|
||||
}
|
||||
if nextEmit < s {
|
||||
if false {
|
||||
emitLiteral(dst, src[nextEmit:s])
|
||||
} else {
|
||||
for _, v := range src[nextEmit:s] {
|
||||
dst.tokens[dst.n] = token(v)
|
||||
dst.litHist[v]++
|
||||
dst.n++
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Save the match found
|
||||
if false {
|
||||
dst.AddMatchLong(l, uint32(s-t-baseMatchOffset))
|
||||
} else {
|
||||
// Inlined...
|
||||
xoffset := uint32(s - t - baseMatchOffset)
|
||||
xlength := l
|
||||
oc := offsetCode(xoffset)
|
||||
xoffset |= oc << 16
|
||||
for xlength > 0 {
|
||||
xl := xlength
|
||||
if xl > 258 {
|
||||
if xl > 258+baseMatchLength {
|
||||
xl = 258
|
||||
} else {
|
||||
xl = 258 - baseMatchLength
|
||||
}
|
||||
}
|
||||
xlength -= xl
|
||||
xl -= baseMatchLength
|
||||
dst.extraHist[lengthCodes1[uint8(xl)]]++
|
||||
dst.offHist[oc]++
|
||||
dst.tokens[dst.n] = token(matchType | uint32(xl)<<lengthShift | xoffset)
|
||||
dst.n++
|
||||
}
|
||||
}
|
||||
s += l
|
||||
nextEmit = s
|
||||
if nextS >= s {
|
||||
s = nextS + 1
|
||||
}
|
||||
if s >= sLimit {
|
||||
// Index first pair after match end.
|
||||
if int(s+l+8) < len(src) {
|
||||
cv := load6432(src, s)
|
||||
e.table[hashLen(cv, tableBits, hashBytes)] = tableEntry{offset: s + e.cur}
|
||||
}
|
||||
goto emitRemainder
|
||||
}
|
||||
|
||||
// We could immediately start working at s now, but to improve
|
||||
// compression we first update the hash table at s-2 and at s. If
|
||||
// another emitCopy is not our next move, also calculate nextHash
|
||||
// at s+1. At least on GOARCH=amd64, these three hash calculations
|
||||
// are faster as one load64 call (with some shifts) instead of
|
||||
// three load32 calls.
|
||||
x := load6432(src, s-2)
|
||||
o := e.cur + s - 2
|
||||
prevHash := hashLen(x, tableBits, hashBytes)
|
||||
e.table[prevHash] = tableEntry{offset: o}
|
||||
x >>= 16
|
||||
currHash := hashLen(x, tableBits, hashBytes)
|
||||
candidate = e.table[currHash]
|
||||
e.table[currHash] = tableEntry{offset: o + 2}
|
||||
|
||||
t = candidate.offset - e.cur
|
||||
if s-t > maxMatchOffset || uint32(x) != load3232(src, t) {
|
||||
cv = x >> 8
|
||||
s++
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
emitRemainder:
|
||||
if int(nextEmit) < len(src) {
|
||||
// If nothing was added, don't encode literals.
|
||||
if dst.n == 0 {
|
||||
return
|
||||
}
|
||||
emitLiteral(dst, src[nextEmit:])
|
||||
}
|
||||
}
|
||||
-214
@@ -1,214 +0,0 @@
|
||||
package flate
|
||||
|
||||
import "fmt"
|
||||
|
||||
// fastGen maintains the table for matches,
|
||||
// and the previous byte block for level 2.
|
||||
// This is the generic implementation.
|
||||
type fastEncL2 struct {
|
||||
fastGen
|
||||
table [bTableSize]tableEntry
|
||||
}
|
||||
|
||||
// EncodeL2 uses a similar algorithm to level 1, but is capable
|
||||
// of matching across blocks giving better compression at a small slowdown.
|
||||
func (e *fastEncL2) Encode(dst *tokens, src []byte) {
|
||||
const (
|
||||
inputMargin = 12 - 1
|
||||
minNonLiteralBlockSize = 1 + 1 + inputMargin
|
||||
hashBytes = 5
|
||||
)
|
||||
|
||||
if debugDeflate && e.cur < 0 {
|
||||
panic(fmt.Sprint("e.cur < 0: ", e.cur))
|
||||
}
|
||||
|
||||
// Protect against e.cur wraparound.
|
||||
for e.cur >= bufferReset {
|
||||
if len(e.hist) == 0 {
|
||||
for i := range e.table[:] {
|
||||
e.table[i] = tableEntry{}
|
||||
}
|
||||
e.cur = maxMatchOffset
|
||||
break
|
||||
}
|
||||
// Shift down everything in the table that isn't already too far away.
|
||||
minOff := e.cur + int32(len(e.hist)) - maxMatchOffset
|
||||
for i := range e.table[:] {
|
||||
v := e.table[i].offset
|
||||
if v <= minOff {
|
||||
v = 0
|
||||
} else {
|
||||
v = v - e.cur + maxMatchOffset
|
||||
}
|
||||
e.table[i].offset = v
|
||||
}
|
||||
e.cur = maxMatchOffset
|
||||
}
|
||||
|
||||
s := e.addBlock(src)
|
||||
|
||||
// This check isn't in the Snappy implementation, but there, the caller
|
||||
// instead of the callee handles this case.
|
||||
if len(src) < minNonLiteralBlockSize {
|
||||
// We do not fill the token table.
|
||||
// This will be picked up by caller.
|
||||
dst.n = uint16(len(src))
|
||||
return
|
||||
}
|
||||
|
||||
// Override src
|
||||
src = e.hist
|
||||
nextEmit := s
|
||||
|
||||
// sLimit is when to stop looking for offset/length copies. The inputMargin
|
||||
// lets us use a fast path for emitLiteral in the main loop, while we are
|
||||
// looking for copies.
|
||||
sLimit := int32(len(src) - inputMargin)
|
||||
|
||||
// nextEmit is where in src the next emitLiteral should start from.
|
||||
cv := load6432(src, s)
|
||||
for {
|
||||
// When should we start skipping if we haven't found matches in a long while.
|
||||
const skipLog = 5
|
||||
const doEvery = 2
|
||||
|
||||
nextS := s
|
||||
var candidate tableEntry
|
||||
for {
|
||||
nextHash := hashLen(cv, bTableBits, hashBytes)
|
||||
s = nextS
|
||||
nextS = s + doEvery + (s-nextEmit)>>skipLog
|
||||
if nextS > sLimit {
|
||||
goto emitRemainder
|
||||
}
|
||||
candidate = e.table[nextHash]
|
||||
now := load6432(src, nextS)
|
||||
e.table[nextHash] = tableEntry{offset: s + e.cur}
|
||||
nextHash = hashLen(now, bTableBits, hashBytes)
|
||||
|
||||
offset := s - (candidate.offset - e.cur)
|
||||
if offset < maxMatchOffset && uint32(cv) == load3232(src, candidate.offset-e.cur) {
|
||||
e.table[nextHash] = tableEntry{offset: nextS + e.cur}
|
||||
break
|
||||
}
|
||||
|
||||
// Do one right away...
|
||||
cv = now
|
||||
s = nextS
|
||||
nextS++
|
||||
candidate = e.table[nextHash]
|
||||
now >>= 8
|
||||
e.table[nextHash] = tableEntry{offset: s + e.cur}
|
||||
|
||||
offset = s - (candidate.offset - e.cur)
|
||||
if offset < maxMatchOffset && uint32(cv) == load3232(src, candidate.offset-e.cur) {
|
||||
break
|
||||
}
|
||||
cv = now
|
||||
}
|
||||
|
||||
// A 4-byte match has been found. We'll later see if more than 4 bytes
|
||||
// match. But, prior to the match, src[nextEmit:s] are unmatched. Emit
|
||||
// them as literal bytes.
|
||||
|
||||
// Call emitCopy, and then see if another emitCopy could be our next
|
||||
// move. Repeat until we find no match for the input immediately after
|
||||
// what was consumed by the last emitCopy call.
|
||||
//
|
||||
// If we exit this loop normally then we need to call emitLiteral next,
|
||||
// though we don't yet know how big the literal will be. We handle that
|
||||
// by proceeding to the next iteration of the main loop. We also can
|
||||
// exit this loop via goto if we get close to exhausting the input.
|
||||
for {
|
||||
// Invariant: we have a 4-byte match at s, and no need to emit any
|
||||
// literal bytes prior to s.
|
||||
|
||||
// Extend the 4-byte match as long as possible.
|
||||
t := candidate.offset - e.cur
|
||||
l := e.matchlenLong(int(s+4), int(t+4), src) + 4
|
||||
|
||||
// Extend backwards
|
||||
for t > 0 && s > nextEmit && src[t-1] == src[s-1] {
|
||||
s--
|
||||
t--
|
||||
l++
|
||||
}
|
||||
if nextEmit < s {
|
||||
if false {
|
||||
emitLiteral(dst, src[nextEmit:s])
|
||||
} else {
|
||||
for _, v := range src[nextEmit:s] {
|
||||
dst.tokens[dst.n] = token(v)
|
||||
dst.litHist[v]++
|
||||
dst.n++
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
dst.AddMatchLong(l, uint32(s-t-baseMatchOffset))
|
||||
s += l
|
||||
nextEmit = s
|
||||
if nextS >= s {
|
||||
s = nextS + 1
|
||||
}
|
||||
|
||||
if s >= sLimit {
|
||||
// Index first pair after match end.
|
||||
if int(s+l+8) < len(src) {
|
||||
cv := load6432(src, s)
|
||||
e.table[hashLen(cv, bTableBits, hashBytes)] = tableEntry{offset: s + e.cur}
|
||||
}
|
||||
goto emitRemainder
|
||||
}
|
||||
|
||||
// Store every second hash in-between, but offset by 1.
|
||||
for i := s - l + 2; i < s-5; i += 7 {
|
||||
x := load6432(src, i)
|
||||
nextHash := hashLen(x, bTableBits, hashBytes)
|
||||
e.table[nextHash] = tableEntry{offset: e.cur + i}
|
||||
// Skip one
|
||||
x >>= 16
|
||||
nextHash = hashLen(x, bTableBits, hashBytes)
|
||||
e.table[nextHash] = tableEntry{offset: e.cur + i + 2}
|
||||
// Skip one
|
||||
x >>= 16
|
||||
nextHash = hashLen(x, bTableBits, hashBytes)
|
||||
e.table[nextHash] = tableEntry{offset: e.cur + i + 4}
|
||||
}
|
||||
|
||||
// We could immediately start working at s now, but to improve
|
||||
// compression we first update the hash table at s-2 to s. If
|
||||
// another emitCopy is not our next move, also calculate nextHash
|
||||
// at s+1. At least on GOARCH=amd64, these three hash calculations
|
||||
// are faster as one load64 call (with some shifts) instead of
|
||||
// three load32 calls.
|
||||
x := load6432(src, s-2)
|
||||
o := e.cur + s - 2
|
||||
prevHash := hashLen(x, bTableBits, hashBytes)
|
||||
prevHash2 := hashLen(x>>8, bTableBits, hashBytes)
|
||||
e.table[prevHash] = tableEntry{offset: o}
|
||||
e.table[prevHash2] = tableEntry{offset: o + 1}
|
||||
currHash := hashLen(x>>16, bTableBits, hashBytes)
|
||||
candidate = e.table[currHash]
|
||||
e.table[currHash] = tableEntry{offset: o + 2}
|
||||
|
||||
offset := s - (candidate.offset - e.cur)
|
||||
if offset > maxMatchOffset || uint32(x>>16) != load3232(src, candidate.offset-e.cur) {
|
||||
cv = x >> 24
|
||||
s++
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
emitRemainder:
|
||||
if int(nextEmit) < len(src) {
|
||||
// If nothing was added, don't encode literals.
|
||||
if dst.n == 0 {
|
||||
return
|
||||
}
|
||||
|
||||
emitLiteral(dst, src[nextEmit:])
|
||||
}
|
||||
}
|
||||
-241
@@ -1,241 +0,0 @@
|
||||
package flate
|
||||
|
||||
import "fmt"
|
||||
|
||||
// fastEncL3
|
||||
type fastEncL3 struct {
|
||||
fastGen
|
||||
table [1 << 16]tableEntryPrev
|
||||
}
|
||||
|
||||
// Encode uses a similar algorithm to level 2, will check up to two candidates.
|
||||
func (e *fastEncL3) Encode(dst *tokens, src []byte) {
|
||||
const (
|
||||
inputMargin = 12 - 1
|
||||
minNonLiteralBlockSize = 1 + 1 + inputMargin
|
||||
tableBits = 16
|
||||
tableSize = 1 << tableBits
|
||||
hashBytes = 5
|
||||
)
|
||||
|
||||
if debugDeflate && e.cur < 0 {
|
||||
panic(fmt.Sprint("e.cur < 0: ", e.cur))
|
||||
}
|
||||
|
||||
// Protect against e.cur wraparound.
|
||||
for e.cur >= bufferReset {
|
||||
if len(e.hist) == 0 {
|
||||
for i := range e.table[:] {
|
||||
e.table[i] = tableEntryPrev{}
|
||||
}
|
||||
e.cur = maxMatchOffset
|
||||
break
|
||||
}
|
||||
// Shift down everything in the table that isn't already too far away.
|
||||
minOff := e.cur + int32(len(e.hist)) - maxMatchOffset
|
||||
for i := range e.table[:] {
|
||||
v := e.table[i]
|
||||
if v.Cur.offset <= minOff {
|
||||
v.Cur.offset = 0
|
||||
} else {
|
||||
v.Cur.offset = v.Cur.offset - e.cur + maxMatchOffset
|
||||
}
|
||||
if v.Prev.offset <= minOff {
|
||||
v.Prev.offset = 0
|
||||
} else {
|
||||
v.Prev.offset = v.Prev.offset - e.cur + maxMatchOffset
|
||||
}
|
||||
e.table[i] = v
|
||||
}
|
||||
e.cur = maxMatchOffset
|
||||
}
|
||||
|
||||
s := e.addBlock(src)
|
||||
|
||||
// Skip if too small.
|
||||
if len(src) < minNonLiteralBlockSize {
|
||||
// We do not fill the token table.
|
||||
// This will be picked up by caller.
|
||||
dst.n = uint16(len(src))
|
||||
return
|
||||
}
|
||||
|
||||
// Override src
|
||||
src = e.hist
|
||||
nextEmit := s
|
||||
|
||||
// sLimit is when to stop looking for offset/length copies. The inputMargin
|
||||
// lets us use a fast path for emitLiteral in the main loop, while we are
|
||||
// looking for copies.
|
||||
sLimit := int32(len(src) - inputMargin)
|
||||
|
||||
// nextEmit is where in src the next emitLiteral should start from.
|
||||
cv := load6432(src, s)
|
||||
for {
|
||||
const skipLog = 7
|
||||
nextS := s
|
||||
var candidate tableEntry
|
||||
for {
|
||||
nextHash := hashLen(cv, tableBits, hashBytes)
|
||||
s = nextS
|
||||
nextS = s + 1 + (s-nextEmit)>>skipLog
|
||||
if nextS > sLimit {
|
||||
goto emitRemainder
|
||||
}
|
||||
candidates := e.table[nextHash]
|
||||
now := load6432(src, nextS)
|
||||
|
||||
// Safe offset distance until s + 4...
|
||||
minOffset := e.cur + s - (maxMatchOffset - 4)
|
||||
e.table[nextHash] = tableEntryPrev{Prev: candidates.Cur, Cur: tableEntry{offset: s + e.cur}}
|
||||
|
||||
// Check both candidates
|
||||
candidate = candidates.Cur
|
||||
if candidate.offset < minOffset {
|
||||
cv = now
|
||||
// Previous will also be invalid, we have nothing.
|
||||
continue
|
||||
}
|
||||
|
||||
if uint32(cv) == load3232(src, candidate.offset-e.cur) {
|
||||
if candidates.Prev.offset < minOffset || uint32(cv) != load3232(src, candidates.Prev.offset-e.cur) {
|
||||
break
|
||||
}
|
||||
// Both match and are valid, pick longest.
|
||||
offset := s - (candidate.offset - e.cur)
|
||||
o2 := s - (candidates.Prev.offset - e.cur)
|
||||
l1, l2 := matchLen(src[s+4:], src[s-offset+4:]), matchLen(src[s+4:], src[s-o2+4:])
|
||||
if l2 > l1 {
|
||||
candidate = candidates.Prev
|
||||
}
|
||||
break
|
||||
} else {
|
||||
// We only check if value mismatches.
|
||||
// Offset will always be invalid in other cases.
|
||||
candidate = candidates.Prev
|
||||
if candidate.offset > minOffset && uint32(cv) == load3232(src, candidate.offset-e.cur) {
|
||||
break
|
||||
}
|
||||
}
|
||||
cv = now
|
||||
}
|
||||
|
||||
// Call emitCopy, and then see if another emitCopy could be our next
|
||||
// move. Repeat until we find no match for the input immediately after
|
||||
// what was consumed by the last emitCopy call.
|
||||
//
|
||||
// If we exit this loop normally then we need to call emitLiteral next,
|
||||
// though we don't yet know how big the literal will be. We handle that
|
||||
// by proceeding to the next iteration of the main loop. We also can
|
||||
// exit this loop via goto if we get close to exhausting the input.
|
||||
for {
|
||||
// Invariant: we have a 4-byte match at s, and no need to emit any
|
||||
// literal bytes prior to s.
|
||||
|
||||
// Extend the 4-byte match as long as possible.
|
||||
//
|
||||
t := candidate.offset - e.cur
|
||||
l := e.matchlenLong(int(s+4), int(t+4), src) + 4
|
||||
|
||||
// Extend backwards
|
||||
for t > 0 && s > nextEmit && src[t-1] == src[s-1] {
|
||||
s--
|
||||
t--
|
||||
l++
|
||||
}
|
||||
if nextEmit < s {
|
||||
if false {
|
||||
emitLiteral(dst, src[nextEmit:s])
|
||||
} else {
|
||||
for _, v := range src[nextEmit:s] {
|
||||
dst.tokens[dst.n] = token(v)
|
||||
dst.litHist[v]++
|
||||
dst.n++
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
dst.AddMatchLong(l, uint32(s-t-baseMatchOffset))
|
||||
s += l
|
||||
nextEmit = s
|
||||
if nextS >= s {
|
||||
s = nextS + 1
|
||||
}
|
||||
|
||||
if s >= sLimit {
|
||||
t += l
|
||||
// Index first pair after match end.
|
||||
if int(t+8) < len(src) && t > 0 {
|
||||
cv = load6432(src, t)
|
||||
nextHash := hashLen(cv, tableBits, hashBytes)
|
||||
e.table[nextHash] = tableEntryPrev{
|
||||
Prev: e.table[nextHash].Cur,
|
||||
Cur: tableEntry{offset: e.cur + t},
|
||||
}
|
||||
}
|
||||
goto emitRemainder
|
||||
}
|
||||
|
||||
// Store every 5th hash in-between.
|
||||
for i := s - l + 2; i < s-5; i += 6 {
|
||||
nextHash := hashLen(load6432(src, i), tableBits, hashBytes)
|
||||
e.table[nextHash] = tableEntryPrev{
|
||||
Prev: e.table[nextHash].Cur,
|
||||
Cur: tableEntry{offset: e.cur + i}}
|
||||
}
|
||||
// We could immediately start working at s now, but to improve
|
||||
// compression we first update the hash table at s-2 to s.
|
||||
x := load6432(src, s-2)
|
||||
prevHash := hashLen(x, tableBits, hashBytes)
|
||||
|
||||
e.table[prevHash] = tableEntryPrev{
|
||||
Prev: e.table[prevHash].Cur,
|
||||
Cur: tableEntry{offset: e.cur + s - 2},
|
||||
}
|
||||
x >>= 8
|
||||
prevHash = hashLen(x, tableBits, hashBytes)
|
||||
|
||||
e.table[prevHash] = tableEntryPrev{
|
||||
Prev: e.table[prevHash].Cur,
|
||||
Cur: tableEntry{offset: e.cur + s - 1},
|
||||
}
|
||||
x >>= 8
|
||||
currHash := hashLen(x, tableBits, hashBytes)
|
||||
candidates := e.table[currHash]
|
||||
cv = x
|
||||
e.table[currHash] = tableEntryPrev{
|
||||
Prev: candidates.Cur,
|
||||
Cur: tableEntry{offset: s + e.cur},
|
||||
}
|
||||
|
||||
// Check both candidates
|
||||
candidate = candidates.Cur
|
||||
minOffset := e.cur + s - (maxMatchOffset - 4)
|
||||
|
||||
if candidate.offset > minOffset {
|
||||
if uint32(cv) == load3232(src, candidate.offset-e.cur) {
|
||||
// Found a match...
|
||||
continue
|
||||
}
|
||||
candidate = candidates.Prev
|
||||
if candidate.offset > minOffset && uint32(cv) == load3232(src, candidate.offset-e.cur) {
|
||||
// Match at prev...
|
||||
continue
|
||||
}
|
||||
}
|
||||
cv = x >> 8
|
||||
s++
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
emitRemainder:
|
||||
if int(nextEmit) < len(src) {
|
||||
// If nothing was added, don't encode literals.
|
||||
if dst.n == 0 {
|
||||
return
|
||||
}
|
||||
|
||||
emitLiteral(dst, src[nextEmit:])
|
||||
}
|
||||
}
|
||||
-221
@@ -1,221 +0,0 @@
|
||||
package flate
|
||||
|
||||
import "fmt"
|
||||
|
||||
type fastEncL4 struct {
|
||||
fastGen
|
||||
table [tableSize]tableEntry
|
||||
bTable [tableSize]tableEntry
|
||||
}
|
||||
|
||||
func (e *fastEncL4) Encode(dst *tokens, src []byte) {
|
||||
const (
|
||||
inputMargin = 12 - 1
|
||||
minNonLiteralBlockSize = 1 + 1 + inputMargin
|
||||
hashShortBytes = 4
|
||||
)
|
||||
if debugDeflate && e.cur < 0 {
|
||||
panic(fmt.Sprint("e.cur < 0: ", e.cur))
|
||||
}
|
||||
// Protect against e.cur wraparound.
|
||||
for e.cur >= bufferReset {
|
||||
if len(e.hist) == 0 {
|
||||
for i := range e.table[:] {
|
||||
e.table[i] = tableEntry{}
|
||||
}
|
||||
for i := range e.bTable[:] {
|
||||
e.bTable[i] = tableEntry{}
|
||||
}
|
||||
e.cur = maxMatchOffset
|
||||
break
|
||||
}
|
||||
// Shift down everything in the table that isn't already too far away.
|
||||
minOff := e.cur + int32(len(e.hist)) - maxMatchOffset
|
||||
for i := range e.table[:] {
|
||||
v := e.table[i].offset
|
||||
if v <= minOff {
|
||||
v = 0
|
||||
} else {
|
||||
v = v - e.cur + maxMatchOffset
|
||||
}
|
||||
e.table[i].offset = v
|
||||
}
|
||||
for i := range e.bTable[:] {
|
||||
v := e.bTable[i].offset
|
||||
if v <= minOff {
|
||||
v = 0
|
||||
} else {
|
||||
v = v - e.cur + maxMatchOffset
|
||||
}
|
||||
e.bTable[i].offset = v
|
||||
}
|
||||
e.cur = maxMatchOffset
|
||||
}
|
||||
|
||||
s := e.addBlock(src)
|
||||
|
||||
// This check isn't in the Snappy implementation, but there, the caller
|
||||
// instead of the callee handles this case.
|
||||
if len(src) < minNonLiteralBlockSize {
|
||||
// We do not fill the token table.
|
||||
// This will be picked up by caller.
|
||||
dst.n = uint16(len(src))
|
||||
return
|
||||
}
|
||||
|
||||
// Override src
|
||||
src = e.hist
|
||||
nextEmit := s
|
||||
|
||||
// sLimit is when to stop looking for offset/length copies. The inputMargin
|
||||
// lets us use a fast path for emitLiteral in the main loop, while we are
|
||||
// looking for copies.
|
||||
sLimit := int32(len(src) - inputMargin)
|
||||
|
||||
// nextEmit is where in src the next emitLiteral should start from.
|
||||
cv := load6432(src, s)
|
||||
for {
|
||||
const skipLog = 6
|
||||
const doEvery = 1
|
||||
|
||||
nextS := s
|
||||
var t int32
|
||||
for {
|
||||
nextHashS := hashLen(cv, tableBits, hashShortBytes)
|
||||
nextHashL := hash7(cv, tableBits)
|
||||
|
||||
s = nextS
|
||||
nextS = s + doEvery + (s-nextEmit)>>skipLog
|
||||
if nextS > sLimit {
|
||||
goto emitRemainder
|
||||
}
|
||||
// Fetch a short+long candidate
|
||||
sCandidate := e.table[nextHashS]
|
||||
lCandidate := e.bTable[nextHashL]
|
||||
next := load6432(src, nextS)
|
||||
entry := tableEntry{offset: s + e.cur}
|
||||
e.table[nextHashS] = entry
|
||||
e.bTable[nextHashL] = entry
|
||||
|
||||
t = lCandidate.offset - e.cur
|
||||
if s-t < maxMatchOffset && uint32(cv) == load3232(src, t) {
|
||||
// We got a long match. Use that.
|
||||
break
|
||||
}
|
||||
|
||||
t = sCandidate.offset - e.cur
|
||||
if s-t < maxMatchOffset && uint32(cv) == load3232(src, t) {
|
||||
// Found a 4 match...
|
||||
lCandidate = e.bTable[hash7(next, tableBits)]
|
||||
|
||||
// If the next long is a candidate, check if we should use that instead...
|
||||
lOff := lCandidate.offset - e.cur
|
||||
if nextS-lOff < maxMatchOffset && load3232(src, lOff) == uint32(next) {
|
||||
l1, l2 := matchLen(src[s+4:], src[t+4:]), matchLen(src[nextS+4:], src[nextS-lOff+4:])
|
||||
if l2 > l1 {
|
||||
s = nextS
|
||||
t = lCandidate.offset - e.cur
|
||||
}
|
||||
}
|
||||
break
|
||||
}
|
||||
cv = next
|
||||
}
|
||||
|
||||
// A 4-byte match has been found. We'll later see if more than 4 bytes
|
||||
// match. But, prior to the match, src[nextEmit:s] are unmatched. Emit
|
||||
// them as literal bytes.
|
||||
|
||||
// Extend the 4-byte match as long as possible.
|
||||
l := e.matchlenLong(int(s+4), int(t+4), src) + 4
|
||||
|
||||
// Extend backwards
|
||||
for t > 0 && s > nextEmit && src[t-1] == src[s-1] {
|
||||
s--
|
||||
t--
|
||||
l++
|
||||
}
|
||||
if nextEmit < s {
|
||||
if false {
|
||||
emitLiteral(dst, src[nextEmit:s])
|
||||
} else {
|
||||
for _, v := range src[nextEmit:s] {
|
||||
dst.tokens[dst.n] = token(v)
|
||||
dst.litHist[v]++
|
||||
dst.n++
|
||||
}
|
||||
}
|
||||
}
|
||||
if debugDeflate {
|
||||
if t >= s {
|
||||
panic("s-t")
|
||||
}
|
||||
if (s - t) > maxMatchOffset {
|
||||
panic(fmt.Sprintln("mmo", t))
|
||||
}
|
||||
if l < baseMatchLength {
|
||||
panic("bml")
|
||||
}
|
||||
}
|
||||
|
||||
dst.AddMatchLong(l, uint32(s-t-baseMatchOffset))
|
||||
s += l
|
||||
nextEmit = s
|
||||
if nextS >= s {
|
||||
s = nextS + 1
|
||||
}
|
||||
|
||||
if s >= sLimit {
|
||||
// Index first pair after match end.
|
||||
if int(s+8) < len(src) {
|
||||
cv := load6432(src, s)
|
||||
e.table[hashLen(cv, tableBits, hashShortBytes)] = tableEntry{offset: s + e.cur}
|
||||
e.bTable[hash7(cv, tableBits)] = tableEntry{offset: s + e.cur}
|
||||
}
|
||||
goto emitRemainder
|
||||
}
|
||||
|
||||
// Store every 3rd hash in-between
|
||||
if true {
|
||||
i := nextS
|
||||
if i < s-1 {
|
||||
cv := load6432(src, i)
|
||||
t := tableEntry{offset: i + e.cur}
|
||||
t2 := tableEntry{offset: t.offset + 1}
|
||||
e.bTable[hash7(cv, tableBits)] = t
|
||||
e.bTable[hash7(cv>>8, tableBits)] = t2
|
||||
e.table[hashLen(cv>>8, tableBits, hashShortBytes)] = t2
|
||||
|
||||
i += 3
|
||||
for ; i < s-1; i += 3 {
|
||||
cv := load6432(src, i)
|
||||
t := tableEntry{offset: i + e.cur}
|
||||
t2 := tableEntry{offset: t.offset + 1}
|
||||
e.bTable[hash7(cv, tableBits)] = t
|
||||
e.bTable[hash7(cv>>8, tableBits)] = t2
|
||||
e.table[hashLen(cv>>8, tableBits, hashShortBytes)] = t2
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// We could immediately start working at s now, but to improve
|
||||
// compression we first update the hash table at s-1 and at s.
|
||||
x := load6432(src, s-1)
|
||||
o := e.cur + s - 1
|
||||
prevHashS := hashLen(x, tableBits, hashShortBytes)
|
||||
prevHashL := hash7(x, tableBits)
|
||||
e.table[prevHashS] = tableEntry{offset: o}
|
||||
e.bTable[prevHashL] = tableEntry{offset: o}
|
||||
cv = x >> 8
|
||||
}
|
||||
|
||||
emitRemainder:
|
||||
if int(nextEmit) < len(src) {
|
||||
// If nothing was added, don't encode literals.
|
||||
if dst.n == 0 {
|
||||
return
|
||||
}
|
||||
|
||||
emitLiteral(dst, src[nextEmit:])
|
||||
}
|
||||
}
|
||||
-705
@@ -1,705 +0,0 @@
|
||||
package flate
|
||||
|
||||
import "fmt"
|
||||
|
||||
type fastEncL5 struct {
|
||||
fastGen
|
||||
table [tableSize]tableEntry
|
||||
bTable [tableSize]tableEntryPrev
|
||||
}
|
||||
|
||||
func (e *fastEncL5) Encode(dst *tokens, src []byte) {
|
||||
const (
|
||||
inputMargin = 12 - 1
|
||||
minNonLiteralBlockSize = 1 + 1 + inputMargin
|
||||
hashShortBytes = 4
|
||||
)
|
||||
if debugDeflate && e.cur < 0 {
|
||||
panic(fmt.Sprint("e.cur < 0: ", e.cur))
|
||||
}
|
||||
|
||||
// Protect against e.cur wraparound.
|
||||
for e.cur >= bufferReset {
|
||||
if len(e.hist) == 0 {
|
||||
for i := range e.table[:] {
|
||||
e.table[i] = tableEntry{}
|
||||
}
|
||||
for i := range e.bTable[:] {
|
||||
e.bTable[i] = tableEntryPrev{}
|
||||
}
|
||||
e.cur = maxMatchOffset
|
||||
break
|
||||
}
|
||||
// Shift down everything in the table that isn't already too far away.
|
||||
minOff := e.cur + int32(len(e.hist)) - maxMatchOffset
|
||||
for i := range e.table[:] {
|
||||
v := e.table[i].offset
|
||||
if v <= minOff {
|
||||
v = 0
|
||||
} else {
|
||||
v = v - e.cur + maxMatchOffset
|
||||
}
|
||||
e.table[i].offset = v
|
||||
}
|
||||
for i := range e.bTable[:] {
|
||||
v := e.bTable[i]
|
||||
if v.Cur.offset <= minOff {
|
||||
v.Cur.offset = 0
|
||||
v.Prev.offset = 0
|
||||
} else {
|
||||
v.Cur.offset = v.Cur.offset - e.cur + maxMatchOffset
|
||||
if v.Prev.offset <= minOff {
|
||||
v.Prev.offset = 0
|
||||
} else {
|
||||
v.Prev.offset = v.Prev.offset - e.cur + maxMatchOffset
|
||||
}
|
||||
}
|
||||
e.bTable[i] = v
|
||||
}
|
||||
e.cur = maxMatchOffset
|
||||
}
|
||||
|
||||
s := e.addBlock(src)
|
||||
|
||||
// This check isn't in the Snappy implementation, but there, the caller
|
||||
// instead of the callee handles this case.
|
||||
if len(src) < minNonLiteralBlockSize {
|
||||
// We do not fill the token table.
|
||||
// This will be picked up by caller.
|
||||
dst.n = uint16(len(src))
|
||||
return
|
||||
}
|
||||
|
||||
// Override src
|
||||
src = e.hist
|
||||
nextEmit := s
|
||||
|
||||
// sLimit is when to stop looking for offset/length copies. The inputMargin
|
||||
// lets us use a fast path for emitLiteral in the main loop, while we are
|
||||
// looking for copies.
|
||||
sLimit := int32(len(src) - inputMargin)
|
||||
|
||||
// nextEmit is where in src the next emitLiteral should start from.
|
||||
cv := load6432(src, s)
|
||||
for {
|
||||
const skipLog = 6
|
||||
const doEvery = 1
|
||||
|
||||
nextS := s
|
||||
var l int32
|
||||
var t int32
|
||||
for {
|
||||
nextHashS := hashLen(cv, tableBits, hashShortBytes)
|
||||
nextHashL := hash7(cv, tableBits)
|
||||
|
||||
s = nextS
|
||||
nextS = s + doEvery + (s-nextEmit)>>skipLog
|
||||
if nextS > sLimit {
|
||||
goto emitRemainder
|
||||
}
|
||||
// Fetch a short+long candidate
|
||||
sCandidate := e.table[nextHashS]
|
||||
lCandidate := e.bTable[nextHashL]
|
||||
next := load6432(src, nextS)
|
||||
entry := tableEntry{offset: s + e.cur}
|
||||
e.table[nextHashS] = entry
|
||||
eLong := &e.bTable[nextHashL]
|
||||
eLong.Cur, eLong.Prev = entry, eLong.Cur
|
||||
|
||||
nextHashS = hashLen(next, tableBits, hashShortBytes)
|
||||
nextHashL = hash7(next, tableBits)
|
||||
|
||||
t = lCandidate.Cur.offset - e.cur
|
||||
if s-t < maxMatchOffset {
|
||||
if uint32(cv) == load3232(src, t) {
|
||||
// Store the next match
|
||||
e.table[nextHashS] = tableEntry{offset: nextS + e.cur}
|
||||
eLong := &e.bTable[nextHashL]
|
||||
eLong.Cur, eLong.Prev = tableEntry{offset: nextS + e.cur}, eLong.Cur
|
||||
|
||||
t2 := lCandidate.Prev.offset - e.cur
|
||||
if s-t2 < maxMatchOffset && uint32(cv) == load3232(src, t2) {
|
||||
l = e.matchlen(int(s+4), int(t+4), src) + 4
|
||||
ml1 := e.matchlen(int(s+4), int(t2+4), src) + 4
|
||||
if ml1 > l {
|
||||
t = t2
|
||||
l = ml1
|
||||
break
|
||||
}
|
||||
}
|
||||
break
|
||||
}
|
||||
t = lCandidate.Prev.offset - e.cur
|
||||
if s-t < maxMatchOffset && uint32(cv) == load3232(src, t) {
|
||||
// Store the next match
|
||||
e.table[nextHashS] = tableEntry{offset: nextS + e.cur}
|
||||
eLong := &e.bTable[nextHashL]
|
||||
eLong.Cur, eLong.Prev = tableEntry{offset: nextS + e.cur}, eLong.Cur
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
t = sCandidate.offset - e.cur
|
||||
if s-t < maxMatchOffset && uint32(cv) == load3232(src, t) {
|
||||
// Found a 4 match...
|
||||
l = e.matchlen(int(s+4), int(t+4), src) + 4
|
||||
lCandidate = e.bTable[nextHashL]
|
||||
// Store the next match
|
||||
|
||||
e.table[nextHashS] = tableEntry{offset: nextS + e.cur}
|
||||
eLong := &e.bTable[nextHashL]
|
||||
eLong.Cur, eLong.Prev = tableEntry{offset: nextS + e.cur}, eLong.Cur
|
||||
|
||||
// If the next long is a candidate, use that...
|
||||
t2 := lCandidate.Cur.offset - e.cur
|
||||
if nextS-t2 < maxMatchOffset {
|
||||
if load3232(src, t2) == uint32(next) {
|
||||
ml := e.matchlen(int(nextS+4), int(t2+4), src) + 4
|
||||
if ml > l {
|
||||
t = t2
|
||||
s = nextS
|
||||
l = ml
|
||||
break
|
||||
}
|
||||
}
|
||||
// If the previous long is a candidate, use that...
|
||||
t2 = lCandidate.Prev.offset - e.cur
|
||||
if nextS-t2 < maxMatchOffset && load3232(src, t2) == uint32(next) {
|
||||
ml := e.matchlen(int(nextS+4), int(t2+4), src) + 4
|
||||
if ml > l {
|
||||
t = t2
|
||||
s = nextS
|
||||
l = ml
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
break
|
||||
}
|
||||
cv = next
|
||||
}
|
||||
|
||||
// A 4-byte match has been found. We'll later see if more than 4 bytes
|
||||
// match. But, prior to the match, src[nextEmit:s] are unmatched. Emit
|
||||
// them as literal bytes.
|
||||
|
||||
if l == 0 {
|
||||
// Extend the 4-byte match as long as possible.
|
||||
l = e.matchlenLong(int(s+4), int(t+4), src) + 4
|
||||
} else if l == maxMatchLength {
|
||||
l += e.matchlenLong(int(s+l), int(t+l), src)
|
||||
}
|
||||
|
||||
// Try to locate a better match by checking the end of best match...
|
||||
if sAt := s + l; l < 30 && sAt < sLimit {
|
||||
// Allow some bytes at the beginning to mismatch.
|
||||
// Sweet spot is 2/3 bytes depending on input.
|
||||
// 3 is only a little better when it is but sometimes a lot worse.
|
||||
// The skipped bytes are tested in Extend backwards,
|
||||
// and still picked up as part of the match if they do.
|
||||
const skipBeginning = 2
|
||||
eLong := e.bTable[hash7(load6432(src, sAt), tableBits)].Cur.offset
|
||||
t2 := eLong - e.cur - l + skipBeginning
|
||||
s2 := s + skipBeginning
|
||||
off := s2 - t2
|
||||
if t2 >= 0 && off < maxMatchOffset && off > 0 {
|
||||
if l2 := e.matchlenLong(int(s2), int(t2), src); l2 > l {
|
||||
t = t2
|
||||
l = l2
|
||||
s = s2
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Extend backwards
|
||||
for t > 0 && s > nextEmit && src[t-1] == src[s-1] {
|
||||
s--
|
||||
t--
|
||||
l++
|
||||
}
|
||||
if nextEmit < s {
|
||||
if false {
|
||||
emitLiteral(dst, src[nextEmit:s])
|
||||
} else {
|
||||
for _, v := range src[nextEmit:s] {
|
||||
dst.tokens[dst.n] = token(v)
|
||||
dst.litHist[v]++
|
||||
dst.n++
|
||||
}
|
||||
}
|
||||
}
|
||||
if debugDeflate {
|
||||
if t >= s {
|
||||
panic(fmt.Sprintln("s-t", s, t))
|
||||
}
|
||||
if (s - t) > maxMatchOffset {
|
||||
panic(fmt.Sprintln("mmo", s-t))
|
||||
}
|
||||
if l < baseMatchLength {
|
||||
panic("bml")
|
||||
}
|
||||
}
|
||||
|
||||
dst.AddMatchLong(l, uint32(s-t-baseMatchOffset))
|
||||
s += l
|
||||
nextEmit = s
|
||||
if nextS >= s {
|
||||
s = nextS + 1
|
||||
}
|
||||
|
||||
if s >= sLimit {
|
||||
goto emitRemainder
|
||||
}
|
||||
|
||||
// Store every 3rd hash in-between.
|
||||
if true {
|
||||
const hashEvery = 3
|
||||
i := s - l + 1
|
||||
if i < s-1 {
|
||||
cv := load6432(src, i)
|
||||
t := tableEntry{offset: i + e.cur}
|
||||
e.table[hashLen(cv, tableBits, hashShortBytes)] = t
|
||||
eLong := &e.bTable[hash7(cv, tableBits)]
|
||||
eLong.Cur, eLong.Prev = t, eLong.Cur
|
||||
|
||||
// Do an long at i+1
|
||||
cv >>= 8
|
||||
t = tableEntry{offset: t.offset + 1}
|
||||
eLong = &e.bTable[hash7(cv, tableBits)]
|
||||
eLong.Cur, eLong.Prev = t, eLong.Cur
|
||||
|
||||
// We only have enough bits for a short entry at i+2
|
||||
cv >>= 8
|
||||
t = tableEntry{offset: t.offset + 1}
|
||||
e.table[hashLen(cv, tableBits, hashShortBytes)] = t
|
||||
|
||||
// Skip one - otherwise we risk hitting 's'
|
||||
i += 4
|
||||
for ; i < s-1; i += hashEvery {
|
||||
cv := load6432(src, i)
|
||||
t := tableEntry{offset: i + e.cur}
|
||||
t2 := tableEntry{offset: t.offset + 1}
|
||||
eLong := &e.bTable[hash7(cv, tableBits)]
|
||||
eLong.Cur, eLong.Prev = t, eLong.Cur
|
||||
e.table[hashLen(cv>>8, tableBits, hashShortBytes)] = t2
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// We could immediately start working at s now, but to improve
|
||||
// compression we first update the hash table at s-1 and at s.
|
||||
x := load6432(src, s-1)
|
||||
o := e.cur + s - 1
|
||||
prevHashS := hashLen(x, tableBits, hashShortBytes)
|
||||
prevHashL := hash7(x, tableBits)
|
||||
e.table[prevHashS] = tableEntry{offset: o}
|
||||
eLong := &e.bTable[prevHashL]
|
||||
eLong.Cur, eLong.Prev = tableEntry{offset: o}, eLong.Cur
|
||||
cv = x >> 8
|
||||
}
|
||||
|
||||
emitRemainder:
|
||||
if int(nextEmit) < len(src) {
|
||||
// If nothing was added, don't encode literals.
|
||||
if dst.n == 0 {
|
||||
return
|
||||
}
|
||||
|
||||
emitLiteral(dst, src[nextEmit:])
|
||||
}
|
||||
}
|
||||
|
||||
// fastEncL5Window is a level 5 encoder,
|
||||
// but with a custom window size.
|
||||
type fastEncL5Window struct {
|
||||
hist []byte
|
||||
cur int32
|
||||
maxOffset int32
|
||||
table [tableSize]tableEntry
|
||||
bTable [tableSize]tableEntryPrev
|
||||
}
|
||||
|
||||
func (e *fastEncL5Window) Encode(dst *tokens, src []byte) {
|
||||
const (
|
||||
inputMargin = 12 - 1
|
||||
minNonLiteralBlockSize = 1 + 1 + inputMargin
|
||||
hashShortBytes = 4
|
||||
)
|
||||
maxMatchOffset := e.maxOffset
|
||||
if debugDeflate && e.cur < 0 {
|
||||
panic(fmt.Sprint("e.cur < 0: ", e.cur))
|
||||
}
|
||||
|
||||
// Protect against e.cur wraparound.
|
||||
for e.cur >= bufferReset {
|
||||
if len(e.hist) == 0 {
|
||||
for i := range e.table[:] {
|
||||
e.table[i] = tableEntry{}
|
||||
}
|
||||
for i := range e.bTable[:] {
|
||||
e.bTable[i] = tableEntryPrev{}
|
||||
}
|
||||
e.cur = maxMatchOffset
|
||||
break
|
||||
}
|
||||
// Shift down everything in the table that isn't already too far away.
|
||||
minOff := e.cur + int32(len(e.hist)) - maxMatchOffset
|
||||
for i := range e.table[:] {
|
||||
v := e.table[i].offset
|
||||
if v <= minOff {
|
||||
v = 0
|
||||
} else {
|
||||
v = v - e.cur + maxMatchOffset
|
||||
}
|
||||
e.table[i].offset = v
|
||||
}
|
||||
for i := range e.bTable[:] {
|
||||
v := e.bTable[i]
|
||||
if v.Cur.offset <= minOff {
|
||||
v.Cur.offset = 0
|
||||
v.Prev.offset = 0
|
||||
} else {
|
||||
v.Cur.offset = v.Cur.offset - e.cur + maxMatchOffset
|
||||
if v.Prev.offset <= minOff {
|
||||
v.Prev.offset = 0
|
||||
} else {
|
||||
v.Prev.offset = v.Prev.offset - e.cur + maxMatchOffset
|
||||
}
|
||||
}
|
||||
e.bTable[i] = v
|
||||
}
|
||||
e.cur = maxMatchOffset
|
||||
}
|
||||
|
||||
s := e.addBlock(src)
|
||||
|
||||
// This check isn't in the Snappy implementation, but there, the caller
|
||||
// instead of the callee handles this case.
|
||||
if len(src) < minNonLiteralBlockSize {
|
||||
// We do not fill the token table.
|
||||
// This will be picked up by caller.
|
||||
dst.n = uint16(len(src))
|
||||
return
|
||||
}
|
||||
|
||||
// Override src
|
||||
src = e.hist
|
||||
nextEmit := s
|
||||
|
||||
// sLimit is when to stop looking for offset/length copies. The inputMargin
|
||||
// lets us use a fast path for emitLiteral in the main loop, while we are
|
||||
// looking for copies.
|
||||
sLimit := int32(len(src) - inputMargin)
|
||||
|
||||
// nextEmit is where in src the next emitLiteral should start from.
|
||||
cv := load6432(src, s)
|
||||
for {
|
||||
const skipLog = 6
|
||||
const doEvery = 1
|
||||
|
||||
nextS := s
|
||||
var l int32
|
||||
var t int32
|
||||
for {
|
||||
nextHashS := hashLen(cv, tableBits, hashShortBytes)
|
||||
nextHashL := hash7(cv, tableBits)
|
||||
|
||||
s = nextS
|
||||
nextS = s + doEvery + (s-nextEmit)>>skipLog
|
||||
if nextS > sLimit {
|
||||
goto emitRemainder
|
||||
}
|
||||
// Fetch a short+long candidate
|
||||
sCandidate := e.table[nextHashS]
|
||||
lCandidate := e.bTable[nextHashL]
|
||||
next := load6432(src, nextS)
|
||||
entry := tableEntry{offset: s + e.cur}
|
||||
e.table[nextHashS] = entry
|
||||
eLong := &e.bTable[nextHashL]
|
||||
eLong.Cur, eLong.Prev = entry, eLong.Cur
|
||||
|
||||
nextHashS = hashLen(next, tableBits, hashShortBytes)
|
||||
nextHashL = hash7(next, tableBits)
|
||||
|
||||
t = lCandidate.Cur.offset - e.cur
|
||||
if s-t < maxMatchOffset {
|
||||
if uint32(cv) == load3232(src, t) {
|
||||
// Store the next match
|
||||
e.table[nextHashS] = tableEntry{offset: nextS + e.cur}
|
||||
eLong := &e.bTable[nextHashL]
|
||||
eLong.Cur, eLong.Prev = tableEntry{offset: nextS + e.cur}, eLong.Cur
|
||||
|
||||
t2 := lCandidate.Prev.offset - e.cur
|
||||
if s-t2 < maxMatchOffset && uint32(cv) == load3232(src, t2) {
|
||||
l = e.matchlen(s+4, t+4, src) + 4
|
||||
ml1 := e.matchlen(s+4, t2+4, src) + 4
|
||||
if ml1 > l {
|
||||
t = t2
|
||||
l = ml1
|
||||
break
|
||||
}
|
||||
}
|
||||
break
|
||||
}
|
||||
t = lCandidate.Prev.offset - e.cur
|
||||
if s-t < maxMatchOffset && uint32(cv) == load3232(src, t) {
|
||||
// Store the next match
|
||||
e.table[nextHashS] = tableEntry{offset: nextS + e.cur}
|
||||
eLong := &e.bTable[nextHashL]
|
||||
eLong.Cur, eLong.Prev = tableEntry{offset: nextS + e.cur}, eLong.Cur
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
t = sCandidate.offset - e.cur
|
||||
if s-t < maxMatchOffset && uint32(cv) == load3232(src, t) {
|
||||
// Found a 4 match...
|
||||
l = e.matchlen(s+4, t+4, src) + 4
|
||||
lCandidate = e.bTable[nextHashL]
|
||||
// Store the next match
|
||||
|
||||
e.table[nextHashS] = tableEntry{offset: nextS + e.cur}
|
||||
eLong := &e.bTable[nextHashL]
|
||||
eLong.Cur, eLong.Prev = tableEntry{offset: nextS + e.cur}, eLong.Cur
|
||||
|
||||
// If the next long is a candidate, use that...
|
||||
t2 := lCandidate.Cur.offset - e.cur
|
||||
if nextS-t2 < maxMatchOffset {
|
||||
if load3232(src, t2) == uint32(next) {
|
||||
ml := e.matchlen(nextS+4, t2+4, src) + 4
|
||||
if ml > l {
|
||||
t = t2
|
||||
s = nextS
|
||||
l = ml
|
||||
break
|
||||
}
|
||||
}
|
||||
// If the previous long is a candidate, use that...
|
||||
t2 = lCandidate.Prev.offset - e.cur
|
||||
if nextS-t2 < maxMatchOffset && load3232(src, t2) == uint32(next) {
|
||||
ml := e.matchlen(nextS+4, t2+4, src) + 4
|
||||
if ml > l {
|
||||
t = t2
|
||||
s = nextS
|
||||
l = ml
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
break
|
||||
}
|
||||
cv = next
|
||||
}
|
||||
|
||||
// A 4-byte match has been found. We'll later see if more than 4 bytes
|
||||
// match. But, prior to the match, src[nextEmit:s] are unmatched. Emit
|
||||
// them as literal bytes.
|
||||
|
||||
if l == 0 {
|
||||
// Extend the 4-byte match as long as possible.
|
||||
l = e.matchlenLong(s+4, t+4, src) + 4
|
||||
} else if l == maxMatchLength {
|
||||
l += e.matchlenLong(s+l, t+l, src)
|
||||
}
|
||||
|
||||
// Try to locate a better match by checking the end of best match...
|
||||
if sAt := s + l; l < 30 && sAt < sLimit {
|
||||
// Allow some bytes at the beginning to mismatch.
|
||||
// Sweet spot is 2/3 bytes depending on input.
|
||||
// 3 is only a little better when it is but sometimes a lot worse.
|
||||
// The skipped bytes are tested in Extend backwards,
|
||||
// and still picked up as part of the match if they do.
|
||||
const skipBeginning = 2
|
||||
eLong := e.bTable[hash7(load6432(src, sAt), tableBits)].Cur.offset
|
||||
t2 := eLong - e.cur - l + skipBeginning
|
||||
s2 := s + skipBeginning
|
||||
off := s2 - t2
|
||||
if t2 >= 0 && off < maxMatchOffset && off > 0 {
|
||||
if l2 := e.matchlenLong(s2, t2, src); l2 > l {
|
||||
t = t2
|
||||
l = l2
|
||||
s = s2
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Extend backwards
|
||||
for t > 0 && s > nextEmit && src[t-1] == src[s-1] {
|
||||
s--
|
||||
t--
|
||||
l++
|
||||
}
|
||||
if nextEmit < s {
|
||||
if false {
|
||||
emitLiteral(dst, src[nextEmit:s])
|
||||
} else {
|
||||
for _, v := range src[nextEmit:s] {
|
||||
dst.tokens[dst.n] = token(v)
|
||||
dst.litHist[v]++
|
||||
dst.n++
|
||||
}
|
||||
}
|
||||
}
|
||||
if debugDeflate {
|
||||
if t >= s {
|
||||
panic(fmt.Sprintln("s-t", s, t))
|
||||
}
|
||||
if (s - t) > maxMatchOffset {
|
||||
panic(fmt.Sprintln("mmo", s-t))
|
||||
}
|
||||
if l < baseMatchLength {
|
||||
panic("bml")
|
||||
}
|
||||
}
|
||||
|
||||
dst.AddMatchLong(l, uint32(s-t-baseMatchOffset))
|
||||
s += l
|
||||
nextEmit = s
|
||||
if nextS >= s {
|
||||
s = nextS + 1
|
||||
}
|
||||
|
||||
if s >= sLimit {
|
||||
goto emitRemainder
|
||||
}
|
||||
|
||||
// Store every 3rd hash in-between.
|
||||
if true {
|
||||
const hashEvery = 3
|
||||
i := s - l + 1
|
||||
if i < s-1 {
|
||||
cv := load6432(src, i)
|
||||
t := tableEntry{offset: i + e.cur}
|
||||
e.table[hashLen(cv, tableBits, hashShortBytes)] = t
|
||||
eLong := &e.bTable[hash7(cv, tableBits)]
|
||||
eLong.Cur, eLong.Prev = t, eLong.Cur
|
||||
|
||||
// Do an long at i+1
|
||||
cv >>= 8
|
||||
t = tableEntry{offset: t.offset + 1}
|
||||
eLong = &e.bTable[hash7(cv, tableBits)]
|
||||
eLong.Cur, eLong.Prev = t, eLong.Cur
|
||||
|
||||
// We only have enough bits for a short entry at i+2
|
||||
cv >>= 8
|
||||
t = tableEntry{offset: t.offset + 1}
|
||||
e.table[hashLen(cv, tableBits, hashShortBytes)] = t
|
||||
|
||||
// Skip one - otherwise we risk hitting 's'
|
||||
i += 4
|
||||
for ; i < s-1; i += hashEvery {
|
||||
cv := load6432(src, i)
|
||||
t := tableEntry{offset: i + e.cur}
|
||||
t2 := tableEntry{offset: t.offset + 1}
|
||||
eLong := &e.bTable[hash7(cv, tableBits)]
|
||||
eLong.Cur, eLong.Prev = t, eLong.Cur
|
||||
e.table[hashLen(cv>>8, tableBits, hashShortBytes)] = t2
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// We could immediately start working at s now, but to improve
|
||||
// compression we first update the hash table at s-1 and at s.
|
||||
x := load6432(src, s-1)
|
||||
o := e.cur + s - 1
|
||||
prevHashS := hashLen(x, tableBits, hashShortBytes)
|
||||
prevHashL := hash7(x, tableBits)
|
||||
e.table[prevHashS] = tableEntry{offset: o}
|
||||
eLong := &e.bTable[prevHashL]
|
||||
eLong.Cur, eLong.Prev = tableEntry{offset: o}, eLong.Cur
|
||||
cv = x >> 8
|
||||
}
|
||||
|
||||
emitRemainder:
|
||||
if int(nextEmit) < len(src) {
|
||||
// If nothing was added, don't encode literals.
|
||||
if dst.n == 0 {
|
||||
return
|
||||
}
|
||||
|
||||
emitLiteral(dst, src[nextEmit:])
|
||||
}
|
||||
}
|
||||
|
||||
// Reset the encoding table.
|
||||
func (e *fastEncL5Window) Reset() {
|
||||
// We keep the same allocs, since we are compressing the same block sizes.
|
||||
if cap(e.hist) < allocHistory {
|
||||
e.hist = make([]byte, 0, allocHistory)
|
||||
}
|
||||
|
||||
// We offset current position so everything will be out of reach.
|
||||
// If we are above the buffer reset it will be cleared anyway since len(hist) == 0.
|
||||
if e.cur <= int32(bufferReset) {
|
||||
e.cur += e.maxOffset + int32(len(e.hist))
|
||||
}
|
||||
e.hist = e.hist[:0]
|
||||
}
|
||||
|
||||
func (e *fastEncL5Window) addBlock(src []byte) int32 {
|
||||
// check if we have space already
|
||||
maxMatchOffset := e.maxOffset
|
||||
|
||||
if len(e.hist)+len(src) > cap(e.hist) {
|
||||
if cap(e.hist) == 0 {
|
||||
e.hist = make([]byte, 0, allocHistory)
|
||||
} else {
|
||||
if cap(e.hist) < int(maxMatchOffset*2) {
|
||||
panic("unexpected buffer size")
|
||||
}
|
||||
// Move down
|
||||
offset := int32(len(e.hist)) - maxMatchOffset
|
||||
copy(e.hist[0:maxMatchOffset], e.hist[offset:])
|
||||
e.cur += offset
|
||||
e.hist = e.hist[:maxMatchOffset]
|
||||
}
|
||||
}
|
||||
s := int32(len(e.hist))
|
||||
e.hist = append(e.hist, src...)
|
||||
return s
|
||||
}
|
||||
|
||||
// matchlen will return the match length between offsets and t in src.
|
||||
// The maximum length returned is maxMatchLength - 4.
|
||||
// It is assumed that s > t, that t >=0 and s < len(src).
|
||||
func (e *fastEncL5Window) matchlen(s, t int32, src []byte) int32 {
|
||||
if debugDecode {
|
||||
if t >= s {
|
||||
panic(fmt.Sprint("t >=s:", t, s))
|
||||
}
|
||||
if int(s) >= len(src) {
|
||||
panic(fmt.Sprint("s >= len(src):", s, len(src)))
|
||||
}
|
||||
if t < 0 {
|
||||
panic(fmt.Sprint("t < 0:", t))
|
||||
}
|
||||
if s-t > e.maxOffset {
|
||||
panic(fmt.Sprint(s, "-", t, "(", s-t, ") > maxMatchLength (", maxMatchOffset, ")"))
|
||||
}
|
||||
}
|
||||
s1 := min(int(s)+maxMatchLength-4, len(src))
|
||||
|
||||
// Extend the match to be as long as possible.
|
||||
return int32(matchLen(src[s:s1], src[t:]))
|
||||
}
|
||||
|
||||
// matchlenLong will return the match length between offsets and t in src.
|
||||
// It is assumed that s > t, that t >=0 and s < len(src).
|
||||
func (e *fastEncL5Window) matchlenLong(s, t int32, src []byte) int32 {
|
||||
if debugDeflate {
|
||||
if t >= s {
|
||||
panic(fmt.Sprint("t >=s:", t, s))
|
||||
}
|
||||
if int(s) >= len(src) {
|
||||
panic(fmt.Sprint("s >= len(src):", s, len(src)))
|
||||
}
|
||||
if t < 0 {
|
||||
panic(fmt.Sprint("t < 0:", t))
|
||||
}
|
||||
if s-t > e.maxOffset {
|
||||
panic(fmt.Sprint(s, "-", t, "(", s-t, ") > maxMatchLength (", maxMatchOffset, ")"))
|
||||
}
|
||||
}
|
||||
// Extend the match to be as long as possible.
|
||||
return int32(matchLen(src[s:], src[t:]))
|
||||
}
|
||||
-325
@@ -1,325 +0,0 @@
|
||||
package flate
|
||||
|
||||
import "fmt"
|
||||
|
||||
type fastEncL6 struct {
|
||||
fastGen
|
||||
table [tableSize]tableEntry
|
||||
bTable [tableSize]tableEntryPrev
|
||||
}
|
||||
|
||||
func (e *fastEncL6) Encode(dst *tokens, src []byte) {
|
||||
const (
|
||||
inputMargin = 12 - 1
|
||||
minNonLiteralBlockSize = 1 + 1 + inputMargin
|
||||
hashShortBytes = 4
|
||||
)
|
||||
if debugDeflate && e.cur < 0 {
|
||||
panic(fmt.Sprint("e.cur < 0: ", e.cur))
|
||||
}
|
||||
|
||||
// Protect against e.cur wraparound.
|
||||
for e.cur >= bufferReset {
|
||||
if len(e.hist) == 0 {
|
||||
for i := range e.table[:] {
|
||||
e.table[i] = tableEntry{}
|
||||
}
|
||||
for i := range e.bTable[:] {
|
||||
e.bTable[i] = tableEntryPrev{}
|
||||
}
|
||||
e.cur = maxMatchOffset
|
||||
break
|
||||
}
|
||||
// Shift down everything in the table that isn't already too far away.
|
||||
minOff := e.cur + int32(len(e.hist)) - maxMatchOffset
|
||||
for i := range e.table[:] {
|
||||
v := e.table[i].offset
|
||||
if v <= minOff {
|
||||
v = 0
|
||||
} else {
|
||||
v = v - e.cur + maxMatchOffset
|
||||
}
|
||||
e.table[i].offset = v
|
||||
}
|
||||
for i := range e.bTable[:] {
|
||||
v := e.bTable[i]
|
||||
if v.Cur.offset <= minOff {
|
||||
v.Cur.offset = 0
|
||||
v.Prev.offset = 0
|
||||
} else {
|
||||
v.Cur.offset = v.Cur.offset - e.cur + maxMatchOffset
|
||||
if v.Prev.offset <= minOff {
|
||||
v.Prev.offset = 0
|
||||
} else {
|
||||
v.Prev.offset = v.Prev.offset - e.cur + maxMatchOffset
|
||||
}
|
||||
}
|
||||
e.bTable[i] = v
|
||||
}
|
||||
e.cur = maxMatchOffset
|
||||
}
|
||||
|
||||
s := e.addBlock(src)
|
||||
|
||||
// This check isn't in the Snappy implementation, but there, the caller
|
||||
// instead of the callee handles this case.
|
||||
if len(src) < minNonLiteralBlockSize {
|
||||
// We do not fill the token table.
|
||||
// This will be picked up by caller.
|
||||
dst.n = uint16(len(src))
|
||||
return
|
||||
}
|
||||
|
||||
// Override src
|
||||
src = e.hist
|
||||
nextEmit := s
|
||||
|
||||
// sLimit is when to stop looking for offset/length copies. The inputMargin
|
||||
// lets us use a fast path for emitLiteral in the main loop, while we are
|
||||
// looking for copies.
|
||||
sLimit := int32(len(src) - inputMargin)
|
||||
|
||||
// nextEmit is where in src the next emitLiteral should start from.
|
||||
cv := load6432(src, s)
|
||||
// Repeat MUST be > 1 and within range
|
||||
repeat := int32(1)
|
||||
for {
|
||||
const skipLog = 7
|
||||
const doEvery = 1
|
||||
|
||||
nextS := s
|
||||
var l int32
|
||||
var t int32
|
||||
for {
|
||||
nextHashS := hashLen(cv, tableBits, hashShortBytes)
|
||||
nextHashL := hash7(cv, tableBits)
|
||||
s = nextS
|
||||
nextS = s + doEvery + (s-nextEmit)>>skipLog
|
||||
if nextS > sLimit {
|
||||
goto emitRemainder
|
||||
}
|
||||
// Fetch a short+long candidate
|
||||
sCandidate := e.table[nextHashS]
|
||||
lCandidate := e.bTable[nextHashL]
|
||||
next := load6432(src, nextS)
|
||||
entry := tableEntry{offset: s + e.cur}
|
||||
e.table[nextHashS] = entry
|
||||
eLong := &e.bTable[nextHashL]
|
||||
eLong.Cur, eLong.Prev = entry, eLong.Cur
|
||||
|
||||
// Calculate hashes of 'next'
|
||||
nextHashS = hashLen(next, tableBits, hashShortBytes)
|
||||
nextHashL = hash7(next, tableBits)
|
||||
|
||||
t = lCandidate.Cur.offset - e.cur
|
||||
if s-t < maxMatchOffset {
|
||||
if uint32(cv) == load3232(src, t) {
|
||||
// Long candidate matches at least 4 bytes.
|
||||
|
||||
// Store the next match
|
||||
e.table[nextHashS] = tableEntry{offset: nextS + e.cur}
|
||||
eLong := &e.bTable[nextHashL]
|
||||
eLong.Cur, eLong.Prev = tableEntry{offset: nextS + e.cur}, eLong.Cur
|
||||
|
||||
// Check the previous long candidate as well.
|
||||
t2 := lCandidate.Prev.offset - e.cur
|
||||
if s-t2 < maxMatchOffset && uint32(cv) == load3232(src, t2) {
|
||||
l = e.matchlen(int(s+4), int(t+4), src) + 4
|
||||
ml1 := e.matchlen(int(s+4), int(t2+4), src) + 4
|
||||
if ml1 > l {
|
||||
t = t2
|
||||
l = ml1
|
||||
break
|
||||
}
|
||||
}
|
||||
break
|
||||
}
|
||||
// Current value did not match, but check if previous long value does.
|
||||
t = lCandidate.Prev.offset - e.cur
|
||||
if s-t < maxMatchOffset && uint32(cv) == load3232(src, t) {
|
||||
// Store the next match
|
||||
e.table[nextHashS] = tableEntry{offset: nextS + e.cur}
|
||||
eLong := &e.bTable[nextHashL]
|
||||
eLong.Cur, eLong.Prev = tableEntry{offset: nextS + e.cur}, eLong.Cur
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
t = sCandidate.offset - e.cur
|
||||
if s-t < maxMatchOffset && uint32(cv) == load3232(src, t) {
|
||||
// Found a 4 match...
|
||||
l = e.matchlen(int(s+4), int(t+4), src) + 4
|
||||
|
||||
// Look up next long candidate (at nextS)
|
||||
lCandidate = e.bTable[nextHashL]
|
||||
|
||||
// Store the next match
|
||||
e.table[nextHashS] = tableEntry{offset: nextS + e.cur}
|
||||
eLong := &e.bTable[nextHashL]
|
||||
eLong.Cur, eLong.Prev = tableEntry{offset: nextS + e.cur}, eLong.Cur
|
||||
|
||||
// Check repeat at s + repOff
|
||||
const repOff = 1
|
||||
t2 := s - repeat + repOff
|
||||
if load3232(src, t2) == uint32(cv>>(8*repOff)) {
|
||||
ml := e.matchlen(int(s+4+repOff), int(t2+4), src) + 4
|
||||
if ml > l {
|
||||
t = t2
|
||||
l = ml
|
||||
s += repOff
|
||||
// Not worth checking more.
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
// If the next long is a candidate, use that...
|
||||
t2 = lCandidate.Cur.offset - e.cur
|
||||
if nextS-t2 < maxMatchOffset {
|
||||
if load3232(src, t2) == uint32(next) {
|
||||
ml := e.matchlen(int(nextS+4), int(t2+4), src) + 4
|
||||
if ml > l {
|
||||
t = t2
|
||||
s = nextS
|
||||
l = ml
|
||||
// This is ok, but check previous as well.
|
||||
}
|
||||
}
|
||||
// If the previous long is a candidate, use that...
|
||||
t2 = lCandidate.Prev.offset - e.cur
|
||||
if nextS-t2 < maxMatchOffset && load3232(src, t2) == uint32(next) {
|
||||
ml := e.matchlen(int(nextS+4), int(t2+4), src) + 4
|
||||
if ml > l {
|
||||
t = t2
|
||||
s = nextS
|
||||
l = ml
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
break
|
||||
}
|
||||
cv = next
|
||||
}
|
||||
|
||||
// A 4-byte match has been found. We'll later see if more than 4 bytes
|
||||
// match. But, prior to the match, src[nextEmit:s] are unmatched. Emit
|
||||
// them as literal bytes.
|
||||
|
||||
// Extend the 4-byte match as long as possible.
|
||||
if l == 0 {
|
||||
l = e.matchlenLong(int(s+4), int(t+4), src) + 4
|
||||
} else if l == maxMatchLength {
|
||||
l += e.matchlenLong(int(s+l), int(t+l), src)
|
||||
}
|
||||
|
||||
// Try to locate a better match by checking the end-of-match...
|
||||
if sAt := s + l; sAt < sLimit {
|
||||
// Allow some bytes at the beginning to mismatch.
|
||||
// Sweet spot is 2/3 bytes depending on input.
|
||||
// 3 is only a little better when it is but sometimes a lot worse.
|
||||
// The skipped bytes are tested in Extend backwards,
|
||||
// and still picked up as part of the match if they do.
|
||||
const skipBeginning = 2
|
||||
eLong := &e.bTable[hash7(load6432(src, sAt), tableBits)]
|
||||
// Test current
|
||||
t2 := eLong.Cur.offset - e.cur - l + skipBeginning
|
||||
s2 := s + skipBeginning
|
||||
off := s2 - t2
|
||||
if off < maxMatchOffset {
|
||||
if off > 0 && t2 >= 0 {
|
||||
if l2 := e.matchlenLong(int(s2), int(t2), src); l2 > l {
|
||||
t = t2
|
||||
l = l2
|
||||
s = s2
|
||||
}
|
||||
}
|
||||
// Test next:
|
||||
t2 = eLong.Prev.offset - e.cur - l + skipBeginning
|
||||
off := s2 - t2
|
||||
if off > 0 && off < maxMatchOffset && t2 >= 0 {
|
||||
if l2 := e.matchlenLong(int(s2), int(t2), src); l2 > l {
|
||||
t = t2
|
||||
l = l2
|
||||
s = s2
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Extend backwards
|
||||
for t > 0 && s > nextEmit && src[t-1] == src[s-1] {
|
||||
s--
|
||||
t--
|
||||
l++
|
||||
}
|
||||
if nextEmit < s {
|
||||
if false {
|
||||
emitLiteral(dst, src[nextEmit:s])
|
||||
} else {
|
||||
for _, v := range src[nextEmit:s] {
|
||||
dst.tokens[dst.n] = token(v)
|
||||
dst.litHist[v]++
|
||||
dst.n++
|
||||
}
|
||||
}
|
||||
}
|
||||
if false {
|
||||
if t >= s {
|
||||
panic(fmt.Sprintln("s-t", s, t))
|
||||
}
|
||||
if (s - t) > maxMatchOffset {
|
||||
panic(fmt.Sprintln("mmo", s-t))
|
||||
}
|
||||
if l < baseMatchLength {
|
||||
panic("bml")
|
||||
}
|
||||
}
|
||||
|
||||
dst.AddMatchLong(l, uint32(s-t-baseMatchOffset))
|
||||
repeat = s - t
|
||||
s += l
|
||||
nextEmit = s
|
||||
if nextS >= s {
|
||||
s = nextS + 1
|
||||
}
|
||||
|
||||
if s >= sLimit {
|
||||
// Index after match end.
|
||||
for i := nextS + 1; i < int32(len(src))-8; i += 2 {
|
||||
cv := load6432(src, i)
|
||||
e.table[hashLen(cv, tableBits, hashShortBytes)] = tableEntry{offset: i + e.cur}
|
||||
eLong := &e.bTable[hash7(cv, tableBits)]
|
||||
eLong.Cur, eLong.Prev = tableEntry{offset: i + e.cur}, eLong.Cur
|
||||
}
|
||||
goto emitRemainder
|
||||
}
|
||||
|
||||
// Store every long hash in-between and every second short.
|
||||
if true {
|
||||
for i := nextS + 1; i < s-1; i += 2 {
|
||||
cv := load6432(src, i)
|
||||
t := tableEntry{offset: i + e.cur}
|
||||
t2 := tableEntry{offset: t.offset + 1}
|
||||
eLong := &e.bTable[hash7(cv, tableBits)]
|
||||
eLong2 := &e.bTable[hash7(cv>>8, tableBits)]
|
||||
e.table[hashLen(cv, tableBits, hashShortBytes)] = t
|
||||
eLong.Cur, eLong.Prev = t, eLong.Cur
|
||||
eLong2.Cur, eLong2.Prev = t2, eLong2.Cur
|
||||
}
|
||||
}
|
||||
|
||||
// We could immediately start working at s now, but to improve
|
||||
// compression we first update the hash table at s-1 and at s.
|
||||
cv = load6432(src, s)
|
||||
}
|
||||
|
||||
emitRemainder:
|
||||
if int(nextEmit) < len(src) {
|
||||
// If nothing was added, don't encode literals.
|
||||
if dst.n == 0 {
|
||||
return
|
||||
}
|
||||
|
||||
emitLiteral(dst, src[nextEmit:])
|
||||
}
|
||||
}
|
||||
Generated
Vendored
-34
@@ -1,34 +0,0 @@
|
||||
// Copyright 2019+ Klaus Post. All rights reserved.
|
||||
// License information can be found in the LICENSE file.
|
||||
|
||||
package flate
|
||||
|
||||
import (
|
||||
"math/bits"
|
||||
|
||||
"github.com/klauspost/compress/internal/le"
|
||||
)
|
||||
|
||||
// matchLen returns the maximum common prefix length of a and b.
|
||||
// a must be the shortest of the two.
|
||||
func matchLen(a, b []byte) (n int) {
|
||||
left := len(a)
|
||||
for left >= 8 {
|
||||
diff := le.Load64(a, n) ^ le.Load64(b, n)
|
||||
if diff != 0 {
|
||||
return n + bits.TrailingZeros64(diff)>>3
|
||||
}
|
||||
n += 8
|
||||
left -= 8
|
||||
}
|
||||
|
||||
a = a[n:]
|
||||
b = b[n:]
|
||||
for i := range a {
|
||||
if a[i] != b[i] {
|
||||
break
|
||||
}
|
||||
n++
|
||||
}
|
||||
return n
|
||||
}
|
||||
Generated
Vendored
-325
@@ -1,325 +0,0 @@
|
||||
package flate
|
||||
|
||||
import (
|
||||
"io"
|
||||
"math"
|
||||
"sync"
|
||||
|
||||
"github.com/klauspost/compress/internal/le"
|
||||
)
|
||||
|
||||
const (
|
||||
maxStatelessBlock = math.MaxInt16
|
||||
// dictionary will be taken from maxStatelessBlock, so limit it.
|
||||
maxStatelessDict = 8 << 10
|
||||
|
||||
slTableBits = 13
|
||||
slTableSize = 1 << slTableBits
|
||||
slTableShift = 32 - slTableBits
|
||||
)
|
||||
|
||||
type statelessWriter struct {
|
||||
dst io.Writer
|
||||
closed bool
|
||||
}
|
||||
|
||||
func (s *statelessWriter) Close() error {
|
||||
if s.closed {
|
||||
return nil
|
||||
}
|
||||
s.closed = true
|
||||
// Emit EOF block
|
||||
return StatelessDeflate(s.dst, nil, true, nil)
|
||||
}
|
||||
|
||||
func (s *statelessWriter) Write(p []byte) (n int, err error) {
|
||||
err = StatelessDeflate(s.dst, p, false, nil)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return len(p), nil
|
||||
}
|
||||
|
||||
func (s *statelessWriter) Reset(w io.Writer) {
|
||||
s.dst = w
|
||||
s.closed = false
|
||||
}
|
||||
|
||||
// NewStatelessWriter will do compression but without maintaining any state
|
||||
// between Write calls.
|
||||
// There will be no memory kept between Write calls,
|
||||
// but compression and speed will be suboptimal.
|
||||
// Because of this, the size of actual Write calls will affect output size.
|
||||
func NewStatelessWriter(dst io.Writer) io.WriteCloser {
|
||||
return &statelessWriter{dst: dst}
|
||||
}
|
||||
|
||||
// bitWriterPool contains bit writers that can be reused.
|
||||
var bitWriterPool = sync.Pool{
|
||||
New: func() any {
|
||||
return newHuffmanBitWriter(nil)
|
||||
},
|
||||
}
|
||||
|
||||
// tokensPool contains tokens struct objects that can be reused
|
||||
var tokensPool = sync.Pool{
|
||||
New: func() any {
|
||||
return &tokens{}
|
||||
},
|
||||
}
|
||||
|
||||
// StatelessDeflate allows compressing directly to a Writer without retaining state.
|
||||
// When returning everything will be flushed.
|
||||
// Up to 8KB of an optional dictionary can be given which is presumed to precede the block.
|
||||
// Longer dictionaries will be truncated and will still produce valid output.
|
||||
// Sending nil dictionary is perfectly fine.
|
||||
func StatelessDeflate(out io.Writer, in []byte, eof bool, dict []byte) error {
|
||||
bw := bitWriterPool.Get().(*huffmanBitWriter)
|
||||
bw.reset(out)
|
||||
defer func() {
|
||||
// don't keep a reference to our output
|
||||
bw.reset(nil)
|
||||
bitWriterPool.Put(bw)
|
||||
}()
|
||||
if eof && len(in) == 0 {
|
||||
// Just write an EOF block.
|
||||
// Could be faster...
|
||||
bw.writeStoredHeader(0, true)
|
||||
bw.flush()
|
||||
return bw.err
|
||||
}
|
||||
|
||||
// Truncate dict
|
||||
if len(dict) > maxStatelessDict {
|
||||
dict = dict[len(dict)-maxStatelessDict:]
|
||||
}
|
||||
|
||||
// For subsequent loops, keep shallow dict reference to avoid alloc+copy.
|
||||
var inDict []byte
|
||||
|
||||
dst := tokensPool.Get().(*tokens)
|
||||
dst.Reset()
|
||||
defer func() {
|
||||
tokensPool.Put(dst)
|
||||
}()
|
||||
|
||||
for len(in) > 0 {
|
||||
todo := in
|
||||
if len(inDict) > 0 {
|
||||
if len(todo) > maxStatelessBlock-maxStatelessDict {
|
||||
todo = todo[:maxStatelessBlock-maxStatelessDict]
|
||||
}
|
||||
} else if len(todo) > maxStatelessBlock-len(dict) {
|
||||
todo = todo[:maxStatelessBlock-len(dict)]
|
||||
}
|
||||
inOrg := in
|
||||
in = in[len(todo):]
|
||||
uncompressed := todo
|
||||
if len(dict) > 0 {
|
||||
// combine dict and source
|
||||
bufLen := len(todo) + len(dict)
|
||||
combined := make([]byte, bufLen)
|
||||
copy(combined, dict)
|
||||
copy(combined[len(dict):], todo)
|
||||
todo = combined
|
||||
}
|
||||
// Compress
|
||||
if len(inDict) == 0 {
|
||||
statelessEnc(dst, todo, int16(len(dict)))
|
||||
} else {
|
||||
statelessEnc(dst, inDict[:maxStatelessDict+len(todo)], maxStatelessDict)
|
||||
}
|
||||
isEof := eof && len(in) == 0
|
||||
|
||||
if dst.n == 0 {
|
||||
bw.writeStoredHeader(len(uncompressed), isEof)
|
||||
if bw.err != nil {
|
||||
return bw.err
|
||||
}
|
||||
bw.writeBytes(uncompressed)
|
||||
} else if int(dst.n) > len(uncompressed)-len(uncompressed)>>4 {
|
||||
// If we removed less than 1/16th, huffman compress the block.
|
||||
bw.writeBlockHuff(isEof, uncompressed, len(in) == 0)
|
||||
} else {
|
||||
bw.writeBlockDynamic(dst, isEof, uncompressed, len(in) == 0)
|
||||
}
|
||||
if len(in) > 0 {
|
||||
// Retain a dict if we have more
|
||||
inDict = inOrg[len(uncompressed)-maxStatelessDict:]
|
||||
dict = nil
|
||||
dst.Reset()
|
||||
}
|
||||
if bw.err != nil {
|
||||
return bw.err
|
||||
}
|
||||
}
|
||||
if !eof {
|
||||
// Align, only a stored block can do that.
|
||||
bw.writeStoredHeader(0, false)
|
||||
}
|
||||
bw.flush()
|
||||
return bw.err
|
||||
}
|
||||
|
||||
func hashSL(u uint32) uint32 {
|
||||
return (u * 0x1e35a7bd) >> slTableShift
|
||||
}
|
||||
|
||||
func load3216(b []byte, i int16) uint32 {
|
||||
return le.Load32(b, i)
|
||||
}
|
||||
|
||||
func load6416(b []byte, i int16) uint64 {
|
||||
return le.Load64(b, i)
|
||||
}
|
||||
|
||||
func statelessEnc(dst *tokens, src []byte, startAt int16) {
|
||||
const (
|
||||
inputMargin = 12 - 1
|
||||
minNonLiteralBlockSize = 1 + 1 + inputMargin
|
||||
)
|
||||
|
||||
type tableEntry struct {
|
||||
offset int16
|
||||
}
|
||||
|
||||
var table [slTableSize]tableEntry
|
||||
|
||||
// This check isn't in the Snappy implementation, but there, the caller
|
||||
// instead of the callee handles this case.
|
||||
if len(src)-int(startAt) < minNonLiteralBlockSize {
|
||||
// We do not fill the token table.
|
||||
// This will be picked up by caller.
|
||||
dst.n = 0
|
||||
return
|
||||
}
|
||||
// Index until startAt
|
||||
if startAt > 0 {
|
||||
cv := load3232(src, 0)
|
||||
for i := range startAt {
|
||||
table[hashSL(cv)] = tableEntry{offset: i}
|
||||
cv = (cv >> 8) | (uint32(src[i+4]) << 24)
|
||||
}
|
||||
}
|
||||
|
||||
s := startAt + 1
|
||||
nextEmit := startAt
|
||||
// sLimit is when to stop looking for offset/length copies. The inputMargin
|
||||
// lets us use a fast path for emitLiteral in the main loop, while we are
|
||||
// looking for copies.
|
||||
sLimit := int16(len(src) - inputMargin)
|
||||
|
||||
// nextEmit is where in src the next emitLiteral should start from.
|
||||
cv := load3216(src, s)
|
||||
|
||||
for {
|
||||
const skipLog = 5
|
||||
const doEvery = 2
|
||||
|
||||
nextS := s
|
||||
var candidate tableEntry
|
||||
for {
|
||||
nextHash := hashSL(cv)
|
||||
candidate = table[nextHash]
|
||||
nextS = s + doEvery + (s-nextEmit)>>skipLog
|
||||
if nextS > sLimit || nextS <= 0 {
|
||||
goto emitRemainder
|
||||
}
|
||||
|
||||
now := load6416(src, nextS)
|
||||
table[nextHash] = tableEntry{offset: s}
|
||||
nextHash = hashSL(uint32(now))
|
||||
|
||||
if cv == load3216(src, candidate.offset) {
|
||||
table[nextHash] = tableEntry{offset: nextS}
|
||||
break
|
||||
}
|
||||
|
||||
// Do one right away...
|
||||
cv = uint32(now)
|
||||
s = nextS
|
||||
nextS++
|
||||
candidate = table[nextHash]
|
||||
now >>= 8
|
||||
table[nextHash] = tableEntry{offset: s}
|
||||
|
||||
if cv == load3216(src, candidate.offset) {
|
||||
table[nextHash] = tableEntry{offset: nextS}
|
||||
break
|
||||
}
|
||||
cv = uint32(now)
|
||||
s = nextS
|
||||
}
|
||||
|
||||
// A 4-byte match has been found. We'll later see if more than 4 bytes
|
||||
// match. But, prior to the match, src[nextEmit:s] are unmatched. Emit
|
||||
// them as literal bytes.
|
||||
for {
|
||||
// Invariant: we have a 4-byte match at s, and no need to emit any
|
||||
// literal bytes prior to s.
|
||||
|
||||
// Extend the 4-byte match as long as possible.
|
||||
t := candidate.offset
|
||||
l := int16(matchLen(src[s+4:], src[t+4:]) + 4)
|
||||
|
||||
// Extend backwards
|
||||
for t > 0 && s > nextEmit && src[t-1] == src[s-1] {
|
||||
s--
|
||||
t--
|
||||
l++
|
||||
}
|
||||
if nextEmit < s {
|
||||
if false {
|
||||
emitLiteral(dst, src[nextEmit:s])
|
||||
} else {
|
||||
for _, v := range src[nextEmit:s] {
|
||||
dst.tokens[dst.n] = token(v)
|
||||
dst.litHist[v]++
|
||||
dst.n++
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Save the match found
|
||||
dst.AddMatchLong(int32(l), uint32(s-t-baseMatchOffset))
|
||||
s += l
|
||||
nextEmit = s
|
||||
if nextS >= s {
|
||||
s = nextS + 1
|
||||
}
|
||||
if s >= sLimit {
|
||||
goto emitRemainder
|
||||
}
|
||||
|
||||
// We could immediately start working at s now, but to improve
|
||||
// compression we first update the hash table at s-2 and at s. If
|
||||
// another emitCopy is not our next move, also calculate nextHash
|
||||
// at s+1. At least on GOARCH=amd64, these three hash calculations
|
||||
// are faster as one load64 call (with some shifts) instead of
|
||||
// three load32 calls.
|
||||
x := load6416(src, s-2)
|
||||
o := s - 2
|
||||
prevHash := hashSL(uint32(x))
|
||||
table[prevHash] = tableEntry{offset: o}
|
||||
x >>= 16
|
||||
currHash := hashSL(uint32(x))
|
||||
candidate = table[currHash]
|
||||
table[currHash] = tableEntry{offset: o + 2}
|
||||
|
||||
if uint32(x) != load3216(src, candidate.offset) {
|
||||
cv = uint32(x >> 8)
|
||||
s++
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
emitRemainder:
|
||||
if int(nextEmit) < len(src) {
|
||||
// If nothing was added, don't encode literals.
|
||||
if dst.n == 0 {
|
||||
return
|
||||
}
|
||||
emitLiteral(dst, src[nextEmit:])
|
||||
}
|
||||
}
|
||||
-379
@@ -1,379 +0,0 @@
|
||||
// Copyright 2009 The Go Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package flate
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/binary"
|
||||
"fmt"
|
||||
"io"
|
||||
"math"
|
||||
)
|
||||
|
||||
const (
|
||||
// bits 0-16 xoffset = offset - MIN_OFFSET_SIZE, or literal - 16 bits
|
||||
// bits 16-22 offsetcode - 5 bits
|
||||
// bits 22-30 xlength = length - MIN_MATCH_LENGTH - 8 bits
|
||||
// bits 30-32 type 0 = literal 1=EOF 2=Match 3=Unused - 2 bits
|
||||
lengthShift = 22
|
||||
offsetMask = 1<<lengthShift - 1
|
||||
typeMask = 3 << 30
|
||||
literalType = 0 << 30
|
||||
matchType = 1 << 30
|
||||
matchOffsetOnlyMask = 0xffff
|
||||
)
|
||||
|
||||
// The length code for length X (MIN_MATCH_LENGTH <= X <= MAX_MATCH_LENGTH)
|
||||
// is lengthCodes[length - MIN_MATCH_LENGTH]
|
||||
var lengthCodes = [256]uint8{
|
||||
0, 1, 2, 3, 4, 5, 6, 7, 8, 8,
|
||||
9, 9, 10, 10, 11, 11, 12, 12, 12, 12,
|
||||
13, 13, 13, 13, 14, 14, 14, 14, 15, 15,
|
||||
15, 15, 16, 16, 16, 16, 16, 16, 16, 16,
|
||||
17, 17, 17, 17, 17, 17, 17, 17, 18, 18,
|
||||
18, 18, 18, 18, 18, 18, 19, 19, 19, 19,
|
||||
19, 19, 19, 19, 20, 20, 20, 20, 20, 20,
|
||||
20, 20, 20, 20, 20, 20, 20, 20, 20, 20,
|
||||
21, 21, 21, 21, 21, 21, 21, 21, 21, 21,
|
||||
21, 21, 21, 21, 21, 21, 22, 22, 22, 22,
|
||||
22, 22, 22, 22, 22, 22, 22, 22, 22, 22,
|
||||
22, 22, 23, 23, 23, 23, 23, 23, 23, 23,
|
||||
23, 23, 23, 23, 23, 23, 23, 23, 24, 24,
|
||||
24, 24, 24, 24, 24, 24, 24, 24, 24, 24,
|
||||
24, 24, 24, 24, 24, 24, 24, 24, 24, 24,
|
||||
24, 24, 24, 24, 24, 24, 24, 24, 24, 24,
|
||||
25, 25, 25, 25, 25, 25, 25, 25, 25, 25,
|
||||
25, 25, 25, 25, 25, 25, 25, 25, 25, 25,
|
||||
25, 25, 25, 25, 25, 25, 25, 25, 25, 25,
|
||||
25, 25, 26, 26, 26, 26, 26, 26, 26, 26,
|
||||
26, 26, 26, 26, 26, 26, 26, 26, 26, 26,
|
||||
26, 26, 26, 26, 26, 26, 26, 26, 26, 26,
|
||||
26, 26, 26, 26, 27, 27, 27, 27, 27, 27,
|
||||
27, 27, 27, 27, 27, 27, 27, 27, 27, 27,
|
||||
27, 27, 27, 27, 27, 27, 27, 27, 27, 27,
|
||||
27, 27, 27, 27, 27, 28,
|
||||
}
|
||||
|
||||
// lengthCodes1 is length codes, but starting at 1.
|
||||
var lengthCodes1 = [256]uint8{
|
||||
1, 2, 3, 4, 5, 6, 7, 8, 9, 9,
|
||||
10, 10, 11, 11, 12, 12, 13, 13, 13, 13,
|
||||
14, 14, 14, 14, 15, 15, 15, 15, 16, 16,
|
||||
16, 16, 17, 17, 17, 17, 17, 17, 17, 17,
|
||||
18, 18, 18, 18, 18, 18, 18, 18, 19, 19,
|
||||
19, 19, 19, 19, 19, 19, 20, 20, 20, 20,
|
||||
20, 20, 20, 20, 21, 21, 21, 21, 21, 21,
|
||||
21, 21, 21, 21, 21, 21, 21, 21, 21, 21,
|
||||
22, 22, 22, 22, 22, 22, 22, 22, 22, 22,
|
||||
22, 22, 22, 22, 22, 22, 23, 23, 23, 23,
|
||||
23, 23, 23, 23, 23, 23, 23, 23, 23, 23,
|
||||
23, 23, 24, 24, 24, 24, 24, 24, 24, 24,
|
||||
24, 24, 24, 24, 24, 24, 24, 24, 25, 25,
|
||||
25, 25, 25, 25, 25, 25, 25, 25, 25, 25,
|
||||
25, 25, 25, 25, 25, 25, 25, 25, 25, 25,
|
||||
25, 25, 25, 25, 25, 25, 25, 25, 25, 25,
|
||||
26, 26, 26, 26, 26, 26, 26, 26, 26, 26,
|
||||
26, 26, 26, 26, 26, 26, 26, 26, 26, 26,
|
||||
26, 26, 26, 26, 26, 26, 26, 26, 26, 26,
|
||||
26, 26, 27, 27, 27, 27, 27, 27, 27, 27,
|
||||
27, 27, 27, 27, 27, 27, 27, 27, 27, 27,
|
||||
27, 27, 27, 27, 27, 27, 27, 27, 27, 27,
|
||||
27, 27, 27, 27, 28, 28, 28, 28, 28, 28,
|
||||
28, 28, 28, 28, 28, 28, 28, 28, 28, 28,
|
||||
28, 28, 28, 28, 28, 28, 28, 28, 28, 28,
|
||||
28, 28, 28, 28, 28, 29,
|
||||
}
|
||||
|
||||
var offsetCodes = [256]uint32{
|
||||
0, 1, 2, 3, 4, 4, 5, 5, 6, 6, 6, 6, 7, 7, 7, 7,
|
||||
8, 8, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9,
|
||||
10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10,
|
||||
11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11,
|
||||
12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12,
|
||||
12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12,
|
||||
13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13,
|
||||
13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13,
|
||||
14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14,
|
||||
14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14,
|
||||
14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14,
|
||||
14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14,
|
||||
15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15,
|
||||
15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15,
|
||||
15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15,
|
||||
15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15,
|
||||
}
|
||||
|
||||
// offsetCodes14 are offsetCodes, but with 14 added.
|
||||
var offsetCodes14 = [256]uint32{
|
||||
14, 15, 16, 17, 18, 18, 19, 19, 20, 20, 20, 20, 21, 21, 21, 21,
|
||||
22, 22, 22, 22, 22, 22, 22, 22, 23, 23, 23, 23, 23, 23, 23, 23,
|
||||
24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24,
|
||||
25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25,
|
||||
26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26,
|
||||
26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26,
|
||||
27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27,
|
||||
27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27,
|
||||
28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28,
|
||||
28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28,
|
||||
28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28,
|
||||
28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28,
|
||||
29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29,
|
||||
29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29,
|
||||
29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29,
|
||||
29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29,
|
||||
}
|
||||
|
||||
type token uint32
|
||||
|
||||
type tokens struct {
|
||||
extraHist [32]uint16 // codes 256->maxnumlit
|
||||
offHist [32]uint16 // offset codes
|
||||
litHist [256]uint16 // codes 0->255
|
||||
nFilled int
|
||||
n uint16 // Must be able to contain maxStoreBlockSize
|
||||
tokens [maxStoreBlockSize + 1]token
|
||||
}
|
||||
|
||||
func (t *tokens) Reset() {
|
||||
if t.n == 0 {
|
||||
return
|
||||
}
|
||||
t.n = 0
|
||||
t.nFilled = 0
|
||||
for i := range t.litHist[:] {
|
||||
t.litHist[i] = 0
|
||||
}
|
||||
for i := range t.extraHist[:] {
|
||||
t.extraHist[i] = 0
|
||||
}
|
||||
for i := range t.offHist[:] {
|
||||
t.offHist[i] = 0
|
||||
}
|
||||
}
|
||||
|
||||
func (t *tokens) Fill() {
|
||||
if t.n == 0 {
|
||||
return
|
||||
}
|
||||
for i, v := range t.litHist[:] {
|
||||
if v == 0 {
|
||||
t.litHist[i] = 1
|
||||
t.nFilled++
|
||||
}
|
||||
}
|
||||
for i, v := range t.extraHist[:literalCount-256] {
|
||||
if v == 0 {
|
||||
t.nFilled++
|
||||
t.extraHist[i] = 1
|
||||
}
|
||||
}
|
||||
for i, v := range t.offHist[:offsetCodeCount] {
|
||||
if v == 0 {
|
||||
t.offHist[i] = 1
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func indexTokens(in []token) tokens {
|
||||
var t tokens
|
||||
t.indexTokens(in)
|
||||
return t
|
||||
}
|
||||
|
||||
func (t *tokens) indexTokens(in []token) {
|
||||
t.Reset()
|
||||
for _, tok := range in {
|
||||
if tok < matchType {
|
||||
t.AddLiteral(tok.literal())
|
||||
continue
|
||||
}
|
||||
t.AddMatch(uint32(tok.length()), tok.offset()&matchOffsetOnlyMask)
|
||||
}
|
||||
}
|
||||
|
||||
// emitLiteral writes a literal chunk and returns the number of bytes written.
|
||||
func emitLiteral(dst *tokens, lit []byte) {
|
||||
for _, v := range lit {
|
||||
dst.tokens[dst.n] = token(v)
|
||||
dst.litHist[v]++
|
||||
dst.n++
|
||||
}
|
||||
}
|
||||
|
||||
func (t *tokens) AddLiteral(lit byte) {
|
||||
t.tokens[t.n] = token(lit)
|
||||
t.litHist[lit]++
|
||||
t.n++
|
||||
}
|
||||
|
||||
// from https://stackoverflow.com/a/28730362
|
||||
func mFastLog2(val float32) float32 {
|
||||
ux := int32(math.Float32bits(val))
|
||||
log2 := (float32)(((ux >> 23) & 255) - 128)
|
||||
ux &= -0x7f800001
|
||||
ux += 127 << 23
|
||||
uval := math.Float32frombits(uint32(ux))
|
||||
log2 += ((-0.34484843)*uval+2.02466578)*uval - 0.67487759
|
||||
return log2
|
||||
}
|
||||
|
||||
// EstimatedBits will return an minimum size estimated by an *optimal*
|
||||
// compression of the block.
|
||||
// The size of the block
|
||||
func (t *tokens) EstimatedBits() int {
|
||||
shannon := float32(0)
|
||||
bits := int(0)
|
||||
nMatches := 0
|
||||
total := int(t.n) + t.nFilled
|
||||
if total > 0 {
|
||||
invTotal := 1.0 / float32(total)
|
||||
for _, v := range t.litHist[:] {
|
||||
if v > 0 {
|
||||
n := float32(v)
|
||||
shannon += atLeastOne(-mFastLog2(n*invTotal)) * n
|
||||
}
|
||||
}
|
||||
// Just add 15 for EOB
|
||||
shannon += 15
|
||||
for i, v := range t.extraHist[1 : literalCount-256] {
|
||||
if v > 0 {
|
||||
n := float32(v)
|
||||
shannon += atLeastOne(-mFastLog2(n*invTotal)) * n
|
||||
bits += int(lengthExtraBits[i&31]) * int(v)
|
||||
nMatches += int(v)
|
||||
}
|
||||
}
|
||||
}
|
||||
if nMatches > 0 {
|
||||
invTotal := 1.0 / float32(nMatches)
|
||||
for i, v := range t.offHist[:offsetCodeCount] {
|
||||
if v > 0 {
|
||||
n := float32(v)
|
||||
shannon += atLeastOne(-mFastLog2(n*invTotal)) * n
|
||||
bits += int(offsetExtraBits[i&31]) * int(v)
|
||||
}
|
||||
}
|
||||
}
|
||||
return int(shannon) + bits
|
||||
}
|
||||
|
||||
// AddMatch adds a match to the tokens.
|
||||
// This function is very sensitive to inlining and right on the border.
|
||||
func (t *tokens) AddMatch(xlength uint32, xoffset uint32) {
|
||||
if debugDeflate {
|
||||
if xlength >= maxMatchLength+baseMatchLength {
|
||||
panic(fmt.Errorf("invalid length: %v", xlength))
|
||||
}
|
||||
if xoffset >= maxMatchOffset+baseMatchOffset {
|
||||
panic(fmt.Errorf("invalid offset: %v", xoffset))
|
||||
}
|
||||
}
|
||||
oCode := offsetCode(xoffset)
|
||||
xoffset |= oCode << 16
|
||||
|
||||
t.extraHist[lengthCodes1[uint8(xlength)]]++
|
||||
t.offHist[oCode&31]++
|
||||
t.tokens[t.n] = token(matchType | xlength<<lengthShift | xoffset)
|
||||
t.n++
|
||||
}
|
||||
|
||||
// AddMatchLong adds a match to the tokens, potentially longer than max match length.
|
||||
// Length should NOT have the base subtracted, only offset should.
|
||||
func (t *tokens) AddMatchLong(xlength int32, xoffset uint32) {
|
||||
if debugDeflate {
|
||||
if xoffset >= maxMatchOffset+baseMatchOffset {
|
||||
panic(fmt.Errorf("invalid offset: %v", xoffset))
|
||||
}
|
||||
}
|
||||
oc := offsetCode(xoffset)
|
||||
xoffset |= oc << 16
|
||||
for xlength > 0 {
|
||||
xl := xlength
|
||||
if xl > 258 {
|
||||
// We need to have at least baseMatchLength left over for next loop.
|
||||
if xl > 258+baseMatchLength {
|
||||
xl = 258
|
||||
} else {
|
||||
xl = 258 - baseMatchLength
|
||||
}
|
||||
}
|
||||
xlength -= xl
|
||||
xl -= baseMatchLength
|
||||
t.extraHist[lengthCodes1[uint8(xl)]]++
|
||||
t.offHist[oc&31]++
|
||||
t.tokens[t.n] = token(matchType | uint32(xl)<<lengthShift | xoffset)
|
||||
t.n++
|
||||
}
|
||||
}
|
||||
|
||||
func (t *tokens) AddEOB() {
|
||||
t.tokens[t.n] = token(endBlockMarker)
|
||||
t.extraHist[0]++
|
||||
t.n++
|
||||
}
|
||||
|
||||
func (t *tokens) Slice() []token {
|
||||
return t.tokens[:t.n]
|
||||
}
|
||||
|
||||
// VarInt returns the tokens as varint encoded bytes.
|
||||
func (t *tokens) VarInt() []byte {
|
||||
var b = make([]byte, binary.MaxVarintLen32*int(t.n))
|
||||
var off int
|
||||
for _, v := range t.tokens[:t.n] {
|
||||
off += binary.PutUvarint(b[off:], uint64(v))
|
||||
}
|
||||
return b[:off]
|
||||
}
|
||||
|
||||
// FromVarInt restores t to the varint encoded tokens provided.
|
||||
// Any data in t is removed.
|
||||
func (t *tokens) FromVarInt(b []byte) error {
|
||||
var buf = bytes.NewReader(b)
|
||||
var toks []token
|
||||
for {
|
||||
r, err := binary.ReadUvarint(buf)
|
||||
if err == io.EOF {
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
toks = append(toks, token(r))
|
||||
}
|
||||
t.indexTokens(toks)
|
||||
return nil
|
||||
}
|
||||
|
||||
// Returns the type of a token
|
||||
func (t token) typ() uint32 { return uint32(t) & typeMask }
|
||||
|
||||
// Returns the literal of a literal token
|
||||
func (t token) literal() uint8 { return uint8(t) }
|
||||
|
||||
// Returns the extra offset of a match token
|
||||
func (t token) offset() uint32 { return uint32(t) & offsetMask }
|
||||
|
||||
func (t token) length() uint8 { return uint8(t >> lengthShift) }
|
||||
|
||||
// Convert length to code.
|
||||
func lengthCode(len uint8) uint8 { return lengthCodes[len] }
|
||||
|
||||
// Returns the offset code corresponding to a specific offset
|
||||
func offsetCode(off uint32) uint32 {
|
||||
if false {
|
||||
if off < uint32(len(offsetCodes)) {
|
||||
return offsetCodes[off&255]
|
||||
} else if off>>7 < uint32(len(offsetCodes)) {
|
||||
return offsetCodes[(off>>7)&255] + 14
|
||||
} else {
|
||||
return offsetCodes[(off>>14)&255] + 28
|
||||
}
|
||||
}
|
||||
if off < uint32(len(offsetCodes)) {
|
||||
return offsetCodes[uint8(off)]
|
||||
}
|
||||
return offsetCodes14[uint8(off>>7)]
|
||||
}
|
||||
Generated
Vendored
-5
@@ -1,5 +0,0 @@
|
||||
package le
|
||||
|
||||
type Indexer interface {
|
||||
int | int8 | int16 | int32 | int64 | uint | uint8 | uint16 | uint32 | uint64
|
||||
}
|
||||
Generated
Vendored
-42
@@ -1,42 +0,0 @@
|
||||
//go:build !(amd64 || arm64 || ppc64le || riscv64) || nounsafe || purego || appengine
|
||||
|
||||
package le
|
||||
|
||||
import (
|
||||
"encoding/binary"
|
||||
)
|
||||
|
||||
// Load8 will load from b at index i.
|
||||
func Load8[I Indexer](b []byte, i I) byte {
|
||||
return b[i]
|
||||
}
|
||||
|
||||
// Load16 will load from b at index i.
|
||||
func Load16[I Indexer](b []byte, i I) uint16 {
|
||||
return binary.LittleEndian.Uint16(b[i:])
|
||||
}
|
||||
|
||||
// Load32 will load from b at index i.
|
||||
func Load32[I Indexer](b []byte, i I) uint32 {
|
||||
return binary.LittleEndian.Uint32(b[i:])
|
||||
}
|
||||
|
||||
// Load64 will load from b at index i.
|
||||
func Load64[I Indexer](b []byte, i I) uint64 {
|
||||
return binary.LittleEndian.Uint64(b[i:])
|
||||
}
|
||||
|
||||
// Store16 will store v at b.
|
||||
func Store16(b []byte, v uint16) {
|
||||
binary.LittleEndian.PutUint16(b, v)
|
||||
}
|
||||
|
||||
// Store32 will store v at b.
|
||||
func Store32(b []byte, v uint32) {
|
||||
binary.LittleEndian.PutUint32(b, v)
|
||||
}
|
||||
|
||||
// Store64 will store v at b.
|
||||
func Store64[I Indexer](b []byte, i I, v uint64) {
|
||||
binary.LittleEndian.PutUint64(b[i:], v)
|
||||
}
|
||||
Generated
Vendored
-52
@@ -1,52 +0,0 @@
|
||||
// We enable 64 bit LE platforms:
|
||||
|
||||
//go:build (amd64 || arm64 || ppc64le || riscv64) && !nounsafe && !purego && !appengine
|
||||
|
||||
package le
|
||||
|
||||
import (
|
||||
"unsafe"
|
||||
)
|
||||
|
||||
// Load8 will load from b at index i.
|
||||
func Load8[I Indexer](b []byte, i I) byte {
|
||||
//return binary.LittleEndian.Uint16(b[i:])
|
||||
//return *(*uint16)(unsafe.Pointer(&b[i]))
|
||||
return *(*byte)(unsafe.Add(unsafe.Pointer(unsafe.SliceData(b)), i))
|
||||
}
|
||||
|
||||
// Load16 will load from b at index i.
|
||||
func Load16[I Indexer](b []byte, i I) uint16 {
|
||||
//return binary.LittleEndian.Uint16(b[i:])
|
||||
//return *(*uint16)(unsafe.Pointer(&b[i]))
|
||||
return *(*uint16)(unsafe.Add(unsafe.Pointer(unsafe.SliceData(b)), i))
|
||||
}
|
||||
|
||||
// Load32 will load from b at index i.
|
||||
func Load32[I Indexer](b []byte, i I) uint32 {
|
||||
//return binary.LittleEndian.Uint32(b[i:])
|
||||
//return *(*uint32)(unsafe.Pointer(&b[i]))
|
||||
return *(*uint32)(unsafe.Add(unsafe.Pointer(unsafe.SliceData(b)), i))
|
||||
}
|
||||
|
||||
// Load64 will load from b at index i.
|
||||
func Load64[I Indexer](b []byte, i I) uint64 {
|
||||
//return binary.LittleEndian.Uint64(b[i:])
|
||||
//return *(*uint64)(unsafe.Pointer(&b[i]))
|
||||
return *(*uint64)(unsafe.Add(unsafe.Pointer(unsafe.SliceData(b)), i))
|
||||
}
|
||||
|
||||
// Store16 will store v at b.
|
||||
func Store16(b []byte, v uint16) {
|
||||
*(*uint16)(unsafe.Pointer(unsafe.SliceData(b))) = v
|
||||
}
|
||||
|
||||
// Store32 will store v at b.
|
||||
func Store32(b []byte, v uint32) {
|
||||
*(*uint32)(unsafe.Pointer(unsafe.SliceData(b))) = v
|
||||
}
|
||||
|
||||
// Store64 will store v at b[i:].
|
||||
func Store64[I Indexer](b []byte, i I, v uint64) {
|
||||
*(*uint64)(unsafe.Add(unsafe.Pointer(unsafe.SliceData(b)), i)) = v
|
||||
}
|
||||
Generated
Vendored
-13
@@ -1,13 +0,0 @@
|
||||
// Copyright 2015 The Go Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
//go:build !race
|
||||
|
||||
package race
|
||||
|
||||
func ReadSlice[T any](s []T) {
|
||||
}
|
||||
|
||||
func WriteSlice[T any](s []T) {
|
||||
}
|
||||
Generated
Vendored
-26
@@ -1,26 +0,0 @@
|
||||
// Copyright 2015 The Go Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
//go:build race
|
||||
|
||||
package race
|
||||
|
||||
import (
|
||||
"runtime"
|
||||
"unsafe"
|
||||
)
|
||||
|
||||
func ReadSlice[T any](s []T) {
|
||||
if len(s) == 0 {
|
||||
return
|
||||
}
|
||||
runtime.RaceReadRange(unsafe.Pointer(&s[0]), len(s)*int(unsafe.Sizeof(s[0])))
|
||||
}
|
||||
|
||||
func WriteSlice[T any](s []T) {
|
||||
if len(s) == 0 {
|
||||
return
|
||||
}
|
||||
runtime.RaceWriteRange(unsafe.Pointer(&s[0]), len(s)*int(unsafe.Sizeof(s[0])))
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user