Merge pull request 'R4e (mesh-sdk): the narrowed tree, the build modules' shared steps and layouts, ParseManifestForInputs, and the vendor directory's leaving' (#70) from feat/r4e into main
This commit was merged in pull request #70.
This commit is contained in:
@@ -1,2 +1,3 @@
|
||||
node_modules/
|
||||
dist/
|
||||
vendor/
|
||||
|
||||
@@ -0,0 +1,126 @@
|
||||
// Package announce is how a module says on the bus which states it keeps and which of them others may read (novox/hq
|
||||
// ADR 0495 clauses 2 and 3; issue 1790): the subject it says it on, the value it says, and where the controller keeps
|
||||
// the last one said by each module. What a module announced is the truth of what it keeps; a reader is granted a read
|
||||
// from it, never from a list or from the owner's manifest.
|
||||
//
|
||||
// A module announces from its code (process.Conn.Keeps, stdio.Keeps, the TypeScript SDK's keeps), never from its
|
||||
// manifest, which only the core reads (clause 1). A module that keeps no state on the bus announces nothing.
|
||||
package announce
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"regexp"
|
||||
"sort"
|
||||
)
|
||||
|
||||
// Version is the version of the value this package writes.
|
||||
const Version = 1
|
||||
|
||||
// Event is the event type of an announcement, under the module's own namespace.
|
||||
const Event = "states"
|
||||
|
||||
// Subject is where module announces what it keeps: its own event subject, mesh.mod.<module>.event.states, which its
|
||||
// account may already publish to and the events stream keeps.
|
||||
func Subject(module string) string { return "mesh.mod." + module + ".event." + Event }
|
||||
|
||||
// SubjectAll is every module's announcement, what the controller consumes.
|
||||
const SubjectAll = "mesh.mod.*.event." + Event
|
||||
|
||||
// KeptBucket is the controller's bucket of the last announcement of each module, by Key.
|
||||
const KeptBucket = "mesh-controller_states-announced"
|
||||
|
||||
var name = regexp.MustCompile(`^[a-z0-9][a-z0-9-]*$`)
|
||||
|
||||
// Key is module's key in KeptBucket: the module itself, which must be one plain name, since a module's name is part of
|
||||
// its buckets' names (catalogue.StateProblems).
|
||||
func Key(module string) (string, error) {
|
||||
if !name.MatchString(module) {
|
||||
return "", fmt.Errorf("announce: %q is not a module that keeps state: lower-case letters, digits and hyphens", module)
|
||||
}
|
||||
return module, nil
|
||||
}
|
||||
|
||||
// State is one state a module keeps, by its local name, and whether others may read it. Others never write it: a
|
||||
// state another module writes is the owner's own code's business.
|
||||
type State struct {
|
||||
Name string `json:"name"`
|
||||
Readable bool `json:"readable,omitempty"`
|
||||
}
|
||||
|
||||
// States is an announcement: the module, the machine whose instance said it, and every state the module keeps, sorted
|
||||
// by name. An empty list says the module keeps nothing.
|
||||
type States struct {
|
||||
V int `json:"v"`
|
||||
Module string `json:"module"`
|
||||
Node string `json:"node,omitempty"`
|
||||
States []State `json:"states"`
|
||||
}
|
||||
|
||||
// Keeps says whether the module announced keeping the state.
|
||||
func (s States) Keeps(state string) bool {
|
||||
for _, k := range s.States {
|
||||
if k.Name == state {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// Readable says whether the module announced the state readable by others.
|
||||
func (s States) Readable(state string) bool {
|
||||
for _, k := range s.States {
|
||||
if k.Name == state {
|
||||
return k.Readable
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func (s *States) check() error {
|
||||
if s.Module == "" {
|
||||
return fmt.Errorf("announce: an announcement names no module")
|
||||
}
|
||||
if _, err := Key(s.Module); err != nil {
|
||||
return err
|
||||
}
|
||||
seen := map[string]bool{}
|
||||
for _, k := range s.States {
|
||||
if !name.MatchString(k.Name) {
|
||||
return fmt.Errorf("announce: %s announces the state %q, and a state is named locally: lower-case letters, digits and hyphens", s.Module, k.Name)
|
||||
}
|
||||
if seen[k.Name] {
|
||||
return fmt.Errorf("announce: %s announces the state %q twice", s.Module, k.Name)
|
||||
}
|
||||
seen[k.Name] = true
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Encode is s as JSON at Version, its states sorted; it refuses a module or a state no bucket could be named by and a
|
||||
// state announced twice, naming it.
|
||||
func Encode(s States) ([]byte, error) {
|
||||
if err := s.check(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
s.V = Version
|
||||
s.States = append([]State{}, s.States...)
|
||||
sort.Slice(s.States, func(i, j int) bool { return s.States[i].Name < s.States[j].Name })
|
||||
return json.Marshal(s)
|
||||
}
|
||||
|
||||
// Decode reads an announcement; it refuses another version and what Encode refuses.
|
||||
func Decode(data []byte) (States, error) {
|
||||
var s States
|
||||
if err := json.Unmarshal(data, &s); err != nil {
|
||||
return States{}, fmt.Errorf("announce: the announcement is not JSON: %w", err)
|
||||
}
|
||||
if s.V != Version {
|
||||
return States{}, fmt.Errorf("announce: the announcement is version %d, and this reads version %d", s.V, Version)
|
||||
}
|
||||
if err := s.check(); err != nil {
|
||||
return States{}, err
|
||||
}
|
||||
sort.Slice(s.States, func(i, j int) bool { return s.States[i].Name < s.States[j].Name })
|
||||
return s, nil
|
||||
}
|
||||
@@ -0,0 +1,71 @@
|
||||
package announce
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// A module says on the bus which states it keeps and which of them others may read (novox/hq ADR 0495 clause 2), on its
|
||||
// own event subject, in a versioned value the controller keeps by module.
|
||||
func TestAnAnnouncementRoundTripsSortedAndSaysWhatIsReadable(t *testing.T) {
|
||||
data, err := Encode(States{Module: "mesh-controller", Node: "novox", States: []State{
|
||||
{Name: "plans-building", Readable: true}, {Name: "assignments", Readable: true}, {Name: "own"}}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
back, err := Decode(data)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if back.V != Version || back.Module != "mesh-controller" || back.Node != "novox" {
|
||||
t.Fatalf("decoded %+v", back)
|
||||
}
|
||||
var names []string
|
||||
for _, s := range back.States {
|
||||
names = append(names, s.Name)
|
||||
}
|
||||
if strings.Join(names, ",") != "assignments,own,plans-building" {
|
||||
t.Fatalf("the states are not sorted: %v", names)
|
||||
}
|
||||
if !back.Readable("plans-building") || back.Readable("own") || back.Readable("absent") {
|
||||
t.Fatal("Readable does not say what was announced readable")
|
||||
}
|
||||
if !back.Keeps("own") || back.Keeps("absent") {
|
||||
t.Fatal("Keeps does not say what was announced")
|
||||
}
|
||||
if Subject("mesh-controller") != "mesh.mod.mesh-controller.event.states" || SubjectAll != "mesh.mod.*.event.states" {
|
||||
t.Fatalf("the subjects are %q and %q", Subject("mesh-controller"), SubjectAll)
|
||||
}
|
||||
if KeptBucket != "mesh-controller_states-announced" {
|
||||
t.Fatalf("the kept bucket is %q", KeptBucket)
|
||||
}
|
||||
if k, err := Key("mesh-delivery-builder"); err != nil || k != "mesh-delivery-builder" {
|
||||
t.Fatalf("Key = %q, %v", k, err)
|
||||
}
|
||||
if nothing, err := Encode(States{Module: "quiet"}); err != nil || !strings.Contains(string(nothing), `"states":[]`) {
|
||||
t.Fatalf("a module keeping nothing: %s, %v", nothing, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnAnnouncementRefusesWhatNoBucketCouldBe(t *testing.T) {
|
||||
for _, c := range []struct {
|
||||
s States
|
||||
says string
|
||||
}{
|
||||
{States{Module: ""}, "no module"},
|
||||
{States{Module: "a.b"}, `"a.b"`},
|
||||
{States{Module: "a", States: []State{{Name: "x_y"}}}, `"x_y"`},
|
||||
{States{Module: "a", States: []State{{Name: ""}}}, `""`},
|
||||
{States{Module: "a", States: []State{{Name: "x"}, {Name: "x", Readable: true}}}, "twice"},
|
||||
} {
|
||||
if _, err := Encode(c.s); err == nil || !strings.Contains(err.Error(), c.says) {
|
||||
t.Errorf("%+v: %v, wanted a refusal saying %s", c.s, err, c.says)
|
||||
}
|
||||
}
|
||||
if _, err := Decode([]byte(`{"v":2,"module":"a","states":[]}`)); err == nil {
|
||||
t.Error("an unknown version was read")
|
||||
}
|
||||
if _, err := Key("a.b"); err == nil {
|
||||
t.Error("a module with a dot became a key")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,91 @@
|
||||
// Package announcetest is the test each module runs with its own checks to hold that it announces on the bus every
|
||||
// state it keeps (novox/hq ADR 0495 clause 5; issue 1790): it starts the module against a bus of the test's own and
|
||||
// compares what the module announced with what its code declares. With the bind guard of process.Conn.Keeps,
|
||||
// stdio.Keeps and the TypeScript keeps, which refuses a state the code did not announce, announcement and code agree.
|
||||
//
|
||||
// It replaces the merge gate's check of one module's reads against another's manifest (clause 5); a test using it
|
||||
// carries `//go:build integration`, since it starts a bus.
|
||||
package announcetest
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"reflect"
|
||||
"sort"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/announce"
|
||||
"git.novox.be/novox/mesh-sdk/go/testbus"
|
||||
)
|
||||
|
||||
// Wait is how long Check waits for the module's announcement once it started it.
|
||||
var Wait = 10 * time.Second
|
||||
|
||||
// Check starts a bus of the test's own, with a stream taking module's events, calls start with its address (start
|
||||
// runs the module, or its part that connects, against it, and answers how to stop it), waits for the module's
|
||||
// announcement and fails the test unless it is module's and names exactly want (in any order), readable as want says.
|
||||
// It answers the announcement.
|
||||
func Check(t testing.TB, module string, want []announce.State, start func(url string) (stop func())) announce.States {
|
||||
t.Helper()
|
||||
url := testbus.URL(t)
|
||||
nc, err := nats.Connect(url)
|
||||
if err != nil {
|
||||
t.Fatalf("announcetest: connecting to the test bus: %v", err)
|
||||
}
|
||||
defer nc.Close()
|
||||
js, err := jetstream.New(nc)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), Wait)
|
||||
defer cancel()
|
||||
stream, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: "ANNOUNCETEST", Subjects: []string{"mesh.mod." + module + ".event.>"}})
|
||||
if err != nil {
|
||||
t.Fatalf("announcetest: making the events stream: %v", err)
|
||||
}
|
||||
stop := start(url)
|
||||
if stop != nil {
|
||||
defer stop()
|
||||
}
|
||||
got, err := waitFor(ctx, stream, module)
|
||||
if err != nil {
|
||||
t.Fatalf("announcetest: %v", err)
|
||||
}
|
||||
if problem := differs(got, module, want); problem != "" {
|
||||
t.Fatalf("announcetest: %s", problem)
|
||||
}
|
||||
return got
|
||||
}
|
||||
|
||||
// waitFor reads the stream until module's announcement is in it.
|
||||
func waitFor(ctx context.Context, stream jetstream.Stream, module string) (announce.States, error) {
|
||||
for {
|
||||
msg, err := stream.GetLastMsgForSubject(ctx, announce.Subject(module))
|
||||
if err == nil {
|
||||
return announce.Decode(msg.Data)
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return announce.States{}, fmt.Errorf("%s announced nothing on %s within %s", module, announce.Subject(module), Wait)
|
||||
case <-time.After(50 * time.Millisecond):
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// differs says how got is not module's announcement of want, or "".
|
||||
func differs(got announce.States, module string, want []announce.State) string {
|
||||
if got.Module != module {
|
||||
return fmt.Sprintf("the announcement names the module %q, not %q", got.Module, module)
|
||||
}
|
||||
w := append([]announce.State{}, want...)
|
||||
sort.Slice(w, func(i, j int) bool { return w[i].Name < w[j].Name })
|
||||
g := append([]announce.State{}, got.States...)
|
||||
if !reflect.DeepEqual(g, w) && !(len(g) == 0 && len(w) == 0) {
|
||||
return fmt.Sprintf("%s announced %v, and its code declares %v", module, g, w)
|
||||
}
|
||||
return ""
|
||||
}
|
||||
@@ -0,0 +1,29 @@
|
||||
//go:build integration
|
||||
|
||||
package announcetest
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/announce"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
)
|
||||
|
||||
// 1790/S11: a module started against the test bus announces through process.Conn.Keeps, and Check reads that
|
||||
// announcement back against what the code declares.
|
||||
func TestCheckReadsTheModulesAnnouncementOffARealBus(t *testing.T) {
|
||||
declared := []announce.State{{Name: "accounts", Readable: true}, {Name: "drafts"}}
|
||||
got := Check(t, "mesh-mail", declared, func(url string) func() {
|
||||
conn, err := process.Connect(process.Account{URL: url, Module: "mesh-mail", Node: "novox", User: "u", Password: "p"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := conn.Keeps(declared...); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return conn.Close
|
||||
})
|
||||
if got.Node != "novox" {
|
||||
t.Fatalf("announced from %q", got.Node)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,32 @@
|
||||
package announcetest
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/announce"
|
||||
)
|
||||
|
||||
// The comparison is exact up to order: a state missing, one extra, a readability that differs or another module's
|
||||
// announcement each fail, naming what differs.
|
||||
func TestAnAnnouncementDiffersFromTheCodeByNameAndReadability(t *testing.T) {
|
||||
got := announce.States{Module: "m", States: []announce.State{{Name: "a", Readable: true}, {Name: "b"}}}
|
||||
if p := differs(got, "m", []announce.State{{Name: "b"}, {Name: "a", Readable: true}}); p != "" {
|
||||
t.Fatalf("an equal announcement differs: %s", p)
|
||||
}
|
||||
for _, want := range [][]announce.State{
|
||||
{{Name: "a", Readable: true}},
|
||||
{{Name: "a", Readable: true}, {Name: "b"}, {Name: "c"}},
|
||||
{{Name: "a"}, {Name: "b"}},
|
||||
} {
|
||||
if p := differs(got, "m", want); p == "" || !strings.Contains(p, "declares") {
|
||||
t.Errorf("want %v: %q", want, p)
|
||||
}
|
||||
}
|
||||
if p := differs(got, "other", nil); !strings.Contains(p, `"other"`) {
|
||||
t.Errorf("another module's announcement: %q", p)
|
||||
}
|
||||
if p := differs(announce.States{Module: "m"}, "m", nil); p != "" {
|
||||
t.Errorf("a module keeping nothing: %q", p)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,170 @@
|
||||
package buildstep
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strings"
|
||||
"syscall"
|
||||
"time"
|
||||
)
|
||||
|
||||
// RegistryWait is how long a push waits for an artifact store that refuses connections, longer than its nightly
|
||||
// collection holds it still (about a minute and a half, issue 457); RegistryFirstPause is the first pause, doubling
|
||||
// to RegistryMostPause. Variables so a test shortens them.
|
||||
var (
|
||||
RegistryWait = 5 * time.Minute
|
||||
RegistryFirstPause = time.Second
|
||||
RegistryMostPause = 10 * time.Second
|
||||
)
|
||||
|
||||
// BlobReference is where a pushed bundle or archive is, as a done key names it and the controller delivers from
|
||||
// (novox/hq ADR 0498 clause 1): artifact-store://<module>/<artifact>/blobs/<digest>, the store's address left out
|
||||
// (issue 102).
|
||||
func BlobReference(module, artifact, digest string) string {
|
||||
return "artifact-store://" + module + "/" + artifact + "/blobs/" + digest
|
||||
}
|
||||
|
||||
// PushBlob puts body into the artifact store at registry (host:port) as a blob of repository (<module>/<artifact>) by
|
||||
// its digest (`sha256:` and hex), from the build module's own process, after the build step ended (novox/hq ADR 0498
|
||||
// clauses 1 and 2): a blob the store already holds is done without an upload; otherwise it is uploaded (POST
|
||||
// blobs/uploads/, then PUT with the digest). A store refusing connections is waited out up to RegistryWait; any other
|
||||
// refusal fails, naming the store's status and its own words, for the caller's `not stored:` error. Bytes that are not
|
||||
// their digest are refused before anything is sent.
|
||||
func PushBlob(ctx context.Context, client *http.Client, registry, repository, digest string, body []byte) error {
|
||||
sum := sha256.Sum256(body)
|
||||
if got := "sha256:" + hex.EncodeToString(sum[:]); got != digest {
|
||||
return fmt.Errorf("the blob is %s, not the digest %s it was to be pushed as", got, digest)
|
||||
}
|
||||
if client == nil {
|
||||
client = http.DefaultClient
|
||||
}
|
||||
base := "http://" + registry + "/v2/" + repository + "/blobs/"
|
||||
var present bool
|
||||
if err := waitForStore(ctx, registry, func() error {
|
||||
var err error
|
||||
present, err = blobPresent(ctx, client, base+digest)
|
||||
return err
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
if present {
|
||||
return nil
|
||||
}
|
||||
var location string
|
||||
if err := waitForStore(ctx, registry, func() error {
|
||||
var err error
|
||||
location, err = startUpload(ctx, client, base+"uploads/")
|
||||
return err
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
target, err := url.Parse(base)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
loc, err := target.Parse(location)
|
||||
if err != nil {
|
||||
return fmt.Errorf("the store answered the upload's location %q, which is not a URL: %w", location, err)
|
||||
}
|
||||
q := loc.Query()
|
||||
q.Set("digest", digest)
|
||||
loc.RawQuery = q.Encode()
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodPut, loc.String(), bytes.NewReader(body))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/octet-stream")
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
return fmt.Errorf("putting %s of %s: %w", digest, repository, err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusCreated {
|
||||
return storeSaid(resp, "putting "+digest+" of "+repository)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func blobPresent(ctx context.Context, client *http.Client, at string) (bool, error) {
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodHead, at, nil)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
resp.Body.Close()
|
||||
switch resp.StatusCode {
|
||||
case http.StatusOK:
|
||||
return true, nil
|
||||
case http.StatusNotFound:
|
||||
return false, nil
|
||||
}
|
||||
return false, fmt.Errorf("asking the store for %s: it answered %s", at, resp.Status)
|
||||
}
|
||||
|
||||
func startUpload(ctx context.Context, client *http.Client, at string) (string, error) {
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodPost, at, nil)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusAccepted {
|
||||
return "", storeSaid(resp, "starting an upload at "+at)
|
||||
}
|
||||
location := resp.Header.Get("Location")
|
||||
if location == "" {
|
||||
return "", fmt.Errorf("starting an upload at %s: the store named no location", at)
|
||||
}
|
||||
return location, nil
|
||||
}
|
||||
|
||||
// storeSaid is the store's refusal: its status and the first of its own words.
|
||||
func storeSaid(resp *http.Response, what string) error {
|
||||
words, _ := io.ReadAll(io.LimitReader(resp.Body, 2048))
|
||||
return fmt.Errorf("%s: the store answered %s: %s", what, resp.Status, strings.TrimSpace(string(words)))
|
||||
}
|
||||
|
||||
// refused is whether an error is a connection refused.
|
||||
func refused(err error) bool {
|
||||
return err != nil && (errors.Is(err, syscall.ECONNREFUSED) || strings.Contains(err.Error(), "connection refused"))
|
||||
}
|
||||
|
||||
// waitForStore runs try, and while it fails because the store refuses connections, tries again with a growing pause
|
||||
// until RegistryWait has passed. A refusal is the one error waited for: nothing was sent, so trying again does nothing
|
||||
// twice.
|
||||
func waitForStore(ctx context.Context, registry string, try func() error) error {
|
||||
err := try()
|
||||
if !refused(err) {
|
||||
return err
|
||||
}
|
||||
started, pause := time.Now(), RegistryFirstPause
|
||||
for {
|
||||
left := RegistryWait - time.Since(started)
|
||||
if left <= 0 {
|
||||
return fmt.Errorf("the artifact store at %s refused every connection for %s, longer than its nightly collection holds it still, so it is down, not paused: %w",
|
||||
registry, time.Since(started).Round(time.Millisecond), err)
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return fmt.Errorf("stopped while waiting for the artifact store at %s: %w (last: %v)", registry, ctx.Err(), err)
|
||||
case <-time.After(min(pause, left)):
|
||||
}
|
||||
pause = min(pause*2, RegistryMostPause)
|
||||
if err = try(); !refused(err) {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,151 @@
|
||||
package buildstep
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
func digestOfBytes(b []byte) string {
|
||||
h := sha256.Sum256(b)
|
||||
return "sha256:" + hex.EncodeToString(h[:])
|
||||
}
|
||||
|
||||
// registry is an artifact store as far as a blob push needs one: HEAD a blob, start an upload, PUT it with its digest.
|
||||
type registry struct {
|
||||
mu sync.Mutex
|
||||
blobs map[string][]byte
|
||||
calls []string
|
||||
put int // the status a PUT answers, 201 when 0
|
||||
}
|
||||
|
||||
func (r *registry) ServeHTTP(w http.ResponseWriter, req *http.Request) {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
r.calls = append(r.calls, req.Method+" "+req.URL.Path)
|
||||
switch {
|
||||
case req.Method == http.MethodHead && strings.Contains(req.URL.Path, "/blobs/sha256:"):
|
||||
if _, ok := r.blobs[req.URL.Path[strings.LastIndex(req.URL.Path, "/")+1:]]; ok {
|
||||
w.WriteHeader(http.StatusOK)
|
||||
return
|
||||
}
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
case req.Method == http.MethodPost && strings.HasSuffix(req.URL.Path, "/blobs/uploads/"):
|
||||
w.Header().Set("Location", strings.TrimSuffix(req.URL.Path, "/")+"/u1?state=x")
|
||||
w.WriteHeader(http.StatusAccepted)
|
||||
case req.Method == http.MethodPut && strings.Contains(req.URL.Path, "/blobs/uploads/u1"):
|
||||
if r.put != 0 {
|
||||
w.WriteHeader(r.put)
|
||||
_, _ = io.WriteString(w, `{"errors":[{"code":"DIGEST_INVALID","message":"provided digest did not match uploaded content"}]}`)
|
||||
return
|
||||
}
|
||||
body, _ := io.ReadAll(req.Body)
|
||||
d := req.URL.Query().Get("digest")
|
||||
if req.URL.Query().Get("state") != "x" || d != digestOfBytes(body) {
|
||||
w.WriteHeader(http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
r.blobs[d] = body
|
||||
w.WriteHeader(http.StatusCreated)
|
||||
default:
|
||||
w.WriteHeader(http.StatusMethodNotAllowed)
|
||||
}
|
||||
}
|
||||
|
||||
// A done build's bundle or archive is pushed to the artifact store by its build module (novox/hq ADR 0498 clauses 1
|
||||
// and 2): a blob the store already holds is done with no upload; one it lacks is uploaded by digest; its reference is
|
||||
// the artifact-store form the builder's key and the controller check.
|
||||
func TestABlobIsPushedOnceByItsDigest(t *testing.T) {
|
||||
reg := ®istry{blobs: map[string][]byte{}}
|
||||
srv := httptest.NewServer(reg)
|
||||
defer srv.Close()
|
||||
addr := strings.TrimPrefix(srv.URL, "http://")
|
||||
body := []byte("the packed bundle")
|
||||
d := digestOfBytes(body)
|
||||
if err := PushBlob(context.Background(), srv.Client(), addr, "mail/code", d, body); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if string(reg.blobs[d]) != string(body) {
|
||||
t.Fatalf("the store holds %q", reg.blobs[d])
|
||||
}
|
||||
reg.calls = nil
|
||||
if err := PushBlob(context.Background(), srv.Client(), addr, "mail/code", d, body); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(reg.calls) != 1 || !strings.HasPrefix(reg.calls[0], "HEAD ") {
|
||||
t.Fatalf("a blob the store holds was uploaded again: %v", reg.calls)
|
||||
}
|
||||
ref := BlobReference("mail", "code", d)
|
||||
if ref != "artifact-store://mail/code/blobs/"+d {
|
||||
t.Fatalf("BlobReference = %q", ref)
|
||||
}
|
||||
b := delivery.BuilderBuild{State: delivery.BuildDone, Manifest: strings.Repeat("4", 64),
|
||||
Artifacts: []delivery.BuiltArtifact{{Name: "code", Kind: "bundle", Digest: d, Reference: ref}}}
|
||||
if _, err := delivery.EncodeBuilderBuild(b); err != nil {
|
||||
t.Fatalf("the key refuses the reference BlobReference gives: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// A store that refuses the upload fails the push naming the store's own words; bytes that are not the digest are
|
||||
// refused before anything is sent.
|
||||
func TestAPushTheStoreRefusesSaysTheStoresWords(t *testing.T) {
|
||||
reg := ®istry{blobs: map[string][]byte{}, put: http.StatusBadRequest}
|
||||
srv := httptest.NewServer(reg)
|
||||
defer srv.Close()
|
||||
addr := strings.TrimPrefix(srv.URL, "http://")
|
||||
body := []byte("x")
|
||||
err := PushBlob(context.Background(), srv.Client(), addr, "mail/code", digestOfBytes(body), body)
|
||||
if err == nil || !strings.Contains(err.Error(), "provided digest did not match") || !strings.Contains(err.Error(), "400") {
|
||||
t.Fatalf("a refused upload said %v", err)
|
||||
}
|
||||
reg.calls = nil
|
||||
err = PushBlob(context.Background(), srv.Client(), addr, "mail/code", digestOfBytes([]byte("other")), body)
|
||||
if err == nil || len(reg.calls) != 0 {
|
||||
t.Fatalf("bytes that are not their digest: %v, calls %v", err, reg.calls)
|
||||
}
|
||||
}
|
||||
|
||||
// A store refusing connections (its nightly collection holds it still) is waited out, up to RegistryWait (issue 457).
|
||||
func TestAPushWaitsOutAStoreRefusingConnections(t *testing.T) {
|
||||
l, err := net.Listen("tcp", "127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
addr := l.Addr().String()
|
||||
l.Close() // refusing now
|
||||
reg := ®istry{blobs: map[string][]byte{}}
|
||||
go func() {
|
||||
time.Sleep(300 * time.Millisecond)
|
||||
l2, err := net.Listen("tcp", addr)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
_ = http.Serve(l2, reg)
|
||||
}()
|
||||
beforeWait, beforePause := RegistryWait, RegistryFirstPause
|
||||
RegistryWait, RegistryFirstPause = 3*time.Second, 100*time.Millisecond
|
||||
defer func() { RegistryWait, RegistryFirstPause = beforeWait, beforePause }()
|
||||
body := []byte("late")
|
||||
if err := PushBlob(context.Background(), http.DefaultClient, addr, "mail/code", digestOfBytes(body), body); err != nil {
|
||||
t.Fatalf("a store that came back was not waited for: %v", err)
|
||||
}
|
||||
|
||||
l3, _ := net.Listen("tcp", "127.0.0.1:0")
|
||||
gone := l3.Addr().String()
|
||||
l3.Close()
|
||||
RegistryWait = 300 * time.Millisecond
|
||||
err = PushBlob(context.Background(), http.DefaultClient, gone, "mail/code", digestOfBytes(body), body)
|
||||
if err == nil || !strings.Contains(err.Error(), "refused every connection") {
|
||||
t.Fatalf("a store that stayed down said %v", err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,175 @@
|
||||
package buildstep
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/forgekey"
|
||||
)
|
||||
|
||||
// Bubblewrap is the program a run step runs in (novox/hq ADR 0448): a build module declares its package, which needs
|
||||
// unprivileged user namespaces (issue 1308).
|
||||
const Bubblewrap = "bwrap"
|
||||
|
||||
// Isolation is how a run step's command is shut off (novox/hq ADR 0448; issue 1309, the lead's decision L2 of R4e: one
|
||||
// runner in the SDK for every build module). The fetch step holds the forge key and the network and runs none of the
|
||||
// change's code; the run step runs the change's build and tests in bubblewrap with:
|
||||
//
|
||||
// - no network: its own network namespace, with only a loopback, so a test may start a bus or a store of its own but
|
||||
// resolves no name and reaches no socket of the machine;
|
||||
// - none of the account: not its home, not the forge key, not a bus login, not the runtime directory (/run), since
|
||||
// none of them is bound; HOME is a directory in the step's own /tmp unless Env names one;
|
||||
// - an environment of exactly Env and the runner's env: nothing of the build module's own passes;
|
||||
// - the system's programs and libraries read-only, and of /etc only what a program needs to run (users, libraries,
|
||||
// certificates, time zone), with a hosts file naming only the loopback;
|
||||
// - the Workspace and Writable directories read-write, the ReadOnly ones (the toolchain, the modules the fetch step
|
||||
// downloaded) read-only;
|
||||
// - the shared warm Caches read-only under a throwaway writable layer (S18), or read-write for a trunk build alone
|
||||
// (WriteCaches), so a candidate's code never changes what a later build reads.
|
||||
type Isolation struct {
|
||||
Workspace string
|
||||
Writable []string
|
||||
ReadOnly []string
|
||||
Caches []string
|
||||
WriteCaches bool
|
||||
Env []string
|
||||
// Hosts is the file bound as /etc/hosts; Isolated writes one naming only the loopback.
|
||||
Hosts string
|
||||
}
|
||||
|
||||
// etcNeeded is what of /etc a program in the step may read: who the users are, where the libraries are, the
|
||||
// certificates, the time zone. Not resolv.conf, not hosts: the step resolves no name.
|
||||
var etcNeeded = []string{"/etc/passwd", "/etc/group", "/etc/nsswitch.conf", "/etc/ld.so.cache", "/etc/ld.so.conf",
|
||||
"/etc/ld.so.conf.d", "/etc/ssl", "/etc/ca-certificates", "/etc/pki", "/etc/localtime", "/etc/os-release", "/etc/alternatives"}
|
||||
|
||||
// systemArgs binds the system's programs and libraries read-only: /usr, and /bin, /sbin, /lib, /lib64 as the symlinks
|
||||
// they are on a merged-usr system, or bound when they are directories. A variable so a unit test fixes it.
|
||||
var systemArgs = func() []string {
|
||||
args := []string{"--ro-bind", "/usr", "/usr"}
|
||||
for _, d := range []string{"/bin", "/sbin", "/lib", "/lib64"} {
|
||||
info, err := os.Lstat(d)
|
||||
switch {
|
||||
case err != nil:
|
||||
case info.Mode()&os.ModeSymlink != 0:
|
||||
if target, err := os.Readlink(d); err == nil {
|
||||
args = append(args, "--symlink", target, d)
|
||||
}
|
||||
case info.IsDir():
|
||||
args = append(args, "--ro-bind", d, d)
|
||||
}
|
||||
}
|
||||
for _, e := range etcNeeded {
|
||||
args = append(args, "--ro-bind-try", e, e)
|
||||
}
|
||||
return args
|
||||
}
|
||||
|
||||
// stepHome is the HOME of a step that is given none: inside its own /tmp, gone with it.
|
||||
const stepHome = "/tmp/home"
|
||||
|
||||
func checkDir(what, dir string) error {
|
||||
if dir == "" || !strings.HasPrefix(dir, "/") || path.Clean(dir) != dir || dir == "/" {
|
||||
return fmt.Errorf("buildstep: the %s %q is not an absolute, clean directory other than the root", what, dir)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Command is the bubblewrap command line that runs name with args shut off as iso says, env added to iso.Env. It
|
||||
// refuses a directory that is not absolute and clean, or is the root, and an environment word without "=".
|
||||
func (iso Isolation) Command(name string, args ...string) ([]string, error) {
|
||||
return iso.command(nil, name, args...)
|
||||
}
|
||||
|
||||
func (iso Isolation) command(env []string, name string, args ...string) ([]string, error) {
|
||||
if err := checkDir("workspace", iso.Workspace); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, group := range []struct {
|
||||
what string
|
||||
dirs []string
|
||||
}{{"writable directory", iso.Writable}, {"read-only directory", iso.ReadOnly}, {"cache", iso.Caches}} {
|
||||
for _, d := range group.dirs {
|
||||
if err := checkDir(group.what, d); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
}
|
||||
words := append(append([]string{}, iso.Env...), env...)
|
||||
home := false
|
||||
for _, w := range words {
|
||||
k, _, ok := strings.Cut(w, "=")
|
||||
if !ok || k == "" {
|
||||
return nil, fmt.Errorf("buildstep: the environment word %q is not NAME=value", w)
|
||||
}
|
||||
home = home || k == "HOME"
|
||||
}
|
||||
argv := []string{"--unshare-all", "--die-with-parent", "--new-session", "--clearenv",
|
||||
"--proc", "/proc", "--dev", "/dev", "--tmpfs", "/tmp", "--dir", stepHome}
|
||||
argv = append(argv, systemArgs()...)
|
||||
for _, d := range iso.ReadOnly {
|
||||
argv = append(argv, "--ro-bind", d, d)
|
||||
}
|
||||
argv = append(argv, cacheArgs(iso.Caches, iso.WriteCaches)...)
|
||||
argv = append(argv, "--bind", iso.Workspace, iso.Workspace)
|
||||
for _, d := range iso.Writable {
|
||||
argv = append(argv, "--bind", d, d)
|
||||
}
|
||||
if iso.Hosts != "" {
|
||||
argv = append(argv, "--ro-bind", iso.Hosts, "/etc/hosts")
|
||||
}
|
||||
for _, w := range words {
|
||||
k, v, _ := strings.Cut(w, "=")
|
||||
argv = append(argv, "--setenv", k, v)
|
||||
}
|
||||
if !home {
|
||||
argv = append(argv, "--setenv", "HOME", stepHome)
|
||||
}
|
||||
argv = append(argv, "--chdir", iso.Workspace, "--", name)
|
||||
return append(argv, args...), nil
|
||||
}
|
||||
|
||||
// cacheArgs binds the shared warm caches (S18): for a trunk build read-write, so the next build is warm; for any other
|
||||
// read-only beneath an overlay whose writes go to a tmpfs of the step's own, so the step writes as a build does and
|
||||
// nothing it wrote outlives it (bubblewrap's --tmp-overlay, an overlay mount in the step's user namespace).
|
||||
func cacheArgs(caches []string, write bool) []string {
|
||||
var argv []string
|
||||
for _, c := range caches {
|
||||
if write {
|
||||
argv = append(argv, "--bind", c, c)
|
||||
continue
|
||||
}
|
||||
argv = append(argv, "--overlay-src", c, "--tmp-overlay", c)
|
||||
}
|
||||
return argv
|
||||
}
|
||||
|
||||
// loopbackHosts is the hosts file of a step: the loopback and nothing else.
|
||||
const loopbackHosts = "127.0.0.1 localhost\n::1 localhost\n"
|
||||
|
||||
// Isolated is run with every command shut off as iso says (Command), run's env going inside the step: the build modules'
|
||||
// run steps call their test, build and digest commands through it, never run itself. Each command gets a hosts file of
|
||||
// its own, naming only the loopback, removed when it ends.
|
||||
func Isolated(run forgekey.Runner, iso Isolation) forgekey.Runner {
|
||||
return func(ctx context.Context, env []string, name string, args ...string) forgekey.Ran {
|
||||
dir, err := os.MkdirTemp("", "step-hosts-")
|
||||
if err != nil {
|
||||
return forgekey.Ran{Status: 1, Err: "the step's hosts file: " + err.Error()}
|
||||
}
|
||||
defer os.RemoveAll(dir)
|
||||
hosts := filepath.Join(dir, "hosts")
|
||||
if err := os.WriteFile(hosts, []byte(loopbackHosts), 0o644); err != nil {
|
||||
return forgekey.Ran{Status: 1, Err: "the step's hosts file: " + err.Error()}
|
||||
}
|
||||
iso := iso
|
||||
iso.Hosts = hosts
|
||||
argv, err := iso.command(env, name, args...)
|
||||
if err != nil {
|
||||
return forgekey.Ran{Status: 1, Err: err.Error()}
|
||||
}
|
||||
return run(ctx, nil, Bubblewrap, argv...)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,30 @@
|
||||
package buildstep
|
||||
|
||||
import "testing"
|
||||
|
||||
// The shared warm caches are read under a throwaway writable layer (novox/hq ADR 0448; issue 1309): a candidate's step
|
||||
// writes into them as a build does, and nothing it writes outlives it or reaches a later build; only a trunk build
|
||||
// writes them.
|
||||
func TestACandidatesStepWritesTheCachesOnlyIntoAThrowawayLayer(t *testing.T) {
|
||||
fixedSystem(t)
|
||||
iso := Isolation{Workspace: "/w", Caches: []string{"/var/cache/go-build", "/var/cache/npm"}}
|
||||
argv, err := iso.Command("go", "test")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, c := range iso.Caches {
|
||||
if !holds(argv, "--overlay-src", c, "--tmp-overlay", c) {
|
||||
t.Errorf("the cache %s is not under a throwaway layer: %v", c, argv)
|
||||
}
|
||||
if holds(argv, "--bind", c, c) {
|
||||
t.Errorf("a candidate's step writes the cache %s: %v", c, argv)
|
||||
}
|
||||
}
|
||||
iso.WriteCaches = true
|
||||
argv, _ = iso.Command("go", "build")
|
||||
for _, c := range iso.Caches {
|
||||
if !holds(argv, "--bind", c, c) || holds(argv, "--tmp-overlay", c) {
|
||||
t.Errorf("a trunk build does not write the cache %s: %v", c, argv)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,66 @@
|
||||
//go:build integration && isolation
|
||||
|
||||
package buildstep
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/forgekey"
|
||||
)
|
||||
|
||||
// 1309/S19, the issue's proof: a run step, in bubblewrap as Isolated runs it, reads no key and no login, resolves no
|
||||
// name, reaches no TCP or unix socket of the machine, finds no home of the account, and leaves the shared cache it
|
||||
// wrote unchanged; the workspace it writes keeps what it wrote. It needs bubblewrap and unprivileged user namespaces
|
||||
// (issue 1308) on the machine that runs it, and fails naming them when they are not there.
|
||||
func TestARunStepReachesNothingAndLeavesTheCachesUnchanged(t *testing.T) {
|
||||
if _, err := exec.LookPath(Bubblewrap); err != nil {
|
||||
t.Fatalf("bubblewrap is not installed here, so the isolation of a run step cannot be proved (issue 1309): %v", err)
|
||||
}
|
||||
root := t.TempDir()
|
||||
account := filepath.Join(root, "account")
|
||||
key, login := filepath.Join(account, ".ssh", "id_ed25519"), filepath.Join(account, "bus-login.json")
|
||||
writeFile(t, filepath.Dir(key), filepath.Base(key), "PRIVATE KEY")
|
||||
writeFile(t, account, "bus-login.json", `{"password":"secret"}`)
|
||||
cacheDir := filepath.Join(root, "cache")
|
||||
writeFile(t, cacheDir, "warm", "warm\n")
|
||||
workspace := filepath.Join(root, "work", "tree")
|
||||
if err := os.MkdirAll(workspace, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
probe := filepath.Join(workspace, "probe")
|
||||
if out, err := exec.Command("go", "build", "-o", probe, "./testdata/isolated-probe").CombinedOutput(); err != nil {
|
||||
t.Fatalf("building the probe: %v\n%s", err, out)
|
||||
}
|
||||
tcp, err := net.Listen("tcp", "127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer tcp.Close()
|
||||
socket := filepath.Join(account, "bus.sock")
|
||||
unix, err := net.Listen("unix", socket)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer unix.Close()
|
||||
|
||||
run := Isolated(forgekey.ExecRunnerFor(0), Isolation{Workspace: workspace, Caches: []string{cacheDir}})
|
||||
r := run(context.Background(), []string{"PATH=/usr/bin:/bin"}, probe, key, login, tcp.Addr().String(), socket, "localhost.localdomain.invalid", filepath.Join(cacheDir, "warm"))
|
||||
if r.Status != 0 || r.Err != "" || strings.TrimSpace(r.Stdout) != "shut off" {
|
||||
t.Fatalf("the run step was not shut off (status %d, %s):\n%s%s", r.Status, r.Err, r.Stdout, r.Stderr)
|
||||
}
|
||||
if b, _ := os.ReadFile(filepath.Join(cacheDir, "warm")); string(b) != "warm\n" {
|
||||
t.Fatalf("the step changed the shared cache: %q", b)
|
||||
}
|
||||
|
||||
// The workspace is the step's own: what it writes there stays.
|
||||
r = run(context.Background(), nil, "/bin/sh", "-c", "echo built > out")
|
||||
if b, err := os.ReadFile(filepath.Join(workspace, "out")); r.Status != 0 || err != nil || string(b) != "built\n" {
|
||||
t.Fatalf("the step's write to its workspace did not stay (status %d, %s%s): %q %v", r.Status, r.Stderr, r.Err, b, err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,115 @@
|
||||
package buildstep
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/forgekey"
|
||||
)
|
||||
|
||||
// pairs says whether argv holds the words of want one after another.
|
||||
func holds(argv []string, want ...string) bool {
|
||||
for i := 0; i+len(want) <= len(argv); i++ {
|
||||
match := true
|
||||
for j, w := range want {
|
||||
if argv[i+j] != w {
|
||||
match = false
|
||||
break
|
||||
}
|
||||
}
|
||||
if match {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func fixedSystem(t *testing.T) {
|
||||
t.Helper()
|
||||
before := systemArgs
|
||||
systemArgs = func() []string { return []string{"--ro-bind", "/usr", "/usr", "--symlink", "usr/bin", "/bin"} }
|
||||
t.Cleanup(func() { systemArgs = before })
|
||||
}
|
||||
|
||||
// A run step's command runs in bubblewrap (novox/hq ADR 0448; issue 1309): no network, no namespace shared, an
|
||||
// environment of exactly what the step is given, the workspace the one tree it writes, the toolchain read-only, and
|
||||
// nothing else of the machine's: not the account's home, not the forge key or a bus login, not the runtime directory,
|
||||
// and no name service.
|
||||
func TestARunStepRunsShutOffInBubblewrap(t *testing.T) {
|
||||
fixedSystem(t)
|
||||
iso := Isolation{Workspace: "/work/ab12/tree", ReadOnly: []string{"/usr/lib/go"}, Writable: []string{"/work/ab12/out"},
|
||||
Env: []string{"GOFLAGS=-mod=mod", "PATH=/usr/bin"}, Hosts: "/work/ab12/hosts"}
|
||||
argv, err := iso.Command("go", "test", "./...")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, want := range [][]string{
|
||||
{"--unshare-all"}, {"--die-with-parent"}, {"--new-session"}, {"--clearenv"},
|
||||
{"--proc", "/proc"}, {"--dev", "/dev"}, {"--tmpfs", "/tmp"},
|
||||
{"--ro-bind", "/usr", "/usr"}, {"--symlink", "usr/bin", "/bin"},
|
||||
{"--ro-bind", "/usr/lib/go", "/usr/lib/go"},
|
||||
{"--bind", "/work/ab12/tree", "/work/ab12/tree"}, {"--bind", "/work/ab12/out", "/work/ab12/out"},
|
||||
{"--ro-bind", "/work/ab12/hosts", "/etc/hosts"},
|
||||
{"--setenv", "GOFLAGS", "-mod=mod"}, {"--setenv", "PATH", "/usr/bin"}, {"--setenv", "HOME", "/tmp/home"},
|
||||
{"--chdir", "/work/ab12/tree"},
|
||||
{"--", "go", "test", "./..."},
|
||||
} {
|
||||
if !holds(argv, want...) {
|
||||
t.Errorf("the command has no %v: %v", want, argv)
|
||||
}
|
||||
}
|
||||
if holds(argv, "--share-net") || strings.Contains(strings.Join(argv, " "), "resolv.conf") || strings.Contains(strings.Join(argv, " "), "/run") {
|
||||
t.Errorf("the command reaches the network, the name service or the runtime directory: %v", argv)
|
||||
}
|
||||
// The workspace is mounted after /tmp's tmpfs, so a workspace under /tmp is not hidden by it.
|
||||
if strings.Index(strings.Join(argv, "\x00"), "--tmpfs\x00/tmp") > strings.Index(strings.Join(argv, "\x00"), "--bind\x00/work/ab12/tree") {
|
||||
t.Errorf("the tmpfs of /tmp comes after the workspace: %v", argv)
|
||||
}
|
||||
// A HOME the step is given is its own.
|
||||
iso.Env = append(iso.Env, "HOME=/work/ab12/home")
|
||||
argv, _ = iso.Command("true")
|
||||
if holds(argv, "--setenv", "HOME", "/tmp/home") || !holds(argv, "--setenv", "HOME", "/work/ab12/home") {
|
||||
t.Errorf("the step's own HOME was replaced: %v", argv)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnIsolationNamesNoPathItCannotPin(t *testing.T) {
|
||||
for _, iso := range []Isolation{
|
||||
{Workspace: ""},
|
||||
{Workspace: "tree"},
|
||||
{Workspace: "/"},
|
||||
{Workspace: "/w", ReadOnly: []string{"relative"}},
|
||||
{Workspace: "/w", Writable: []string{"/"}},
|
||||
{Workspace: "/w", Caches: []string{"/a/../b"}},
|
||||
{Workspace: "/w", Env: []string{"NOEQUALS"}},
|
||||
} {
|
||||
if _, err := iso.Command("true"); err == nil {
|
||||
t.Errorf("%+v was accepted", iso)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Isolated runs every command through bwrap, with the step's environment inside and a hosts file that names only the
|
||||
// loopback, removed after.
|
||||
func TestIsolatedRunsEachCommandThroughBubblewrap(t *testing.T) {
|
||||
fixedSystem(t)
|
||||
var ran []string
|
||||
var hosts string
|
||||
inner := func(ctx context.Context, env []string, name string, args ...string) forgekey.Ran {
|
||||
ran = append([]string{name}, args...)
|
||||
for i, a := range args {
|
||||
if a == "/etc/hosts" && i > 0 {
|
||||
hosts = args[i-1]
|
||||
}
|
||||
}
|
||||
return forgekey.Ran{Stdout: "ok"}
|
||||
}
|
||||
r := Isolated(inner, Isolation{Workspace: "/w"})(context.Background(), []string{"A=1"}, "go", "build")
|
||||
if r.Stdout != "ok" || ran[0] != Bubblewrap || !holds(ran, "--setenv", "A", "1") || !holds(ran, "--", "go", "build") {
|
||||
t.Fatalf("ran %v: %+v", ran, r)
|
||||
}
|
||||
if hosts == "" {
|
||||
t.Fatalf("no hosts file was bound: %v", ran)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
package buildstep
|
||||
|
||||
import "strings"
|
||||
|
||||
// LauncherSuffix is what a TypeScript entrypoint's launcher is called beside it: index.js is started as
|
||||
// index.serve.mjs (novox/hq ADR 0193), an ES module by its own extension whatever the bundle's package.json says.
|
||||
const LauncherSuffix = ".serve.mjs"
|
||||
|
||||
// Launchers are the launchers a bundle's build writes, by entrypoint, relative to the bundle's root: for TypeScript
|
||||
// every entrypoint `x.js` is started by `x.serve.mjs` beside it; an entrypoint that is not JavaScript and every other
|
||||
// language, whose build is already executable, get none. The one rule mesh-build-node, build-agent and the controller
|
||||
// share (R4e), so the name a build writes is the name a reader expects.
|
||||
func Launchers(language string, entrypoints []string) map[string]string {
|
||||
out := map[string]string{}
|
||||
if language != "typescript" {
|
||||
return out
|
||||
}
|
||||
for _, entry := range entrypoints {
|
||||
if strings.HasSuffix(entry, ".js") {
|
||||
out[entry] = strings.TrimSuffix(entry, ".js") + LauncherSuffix
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
package buildstep
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// A TypeScript bundle's entrypoint x.js is started as x.serve.mjs beside it (novox/hq ADR 0193), the rule mesh-build-node
|
||||
// and build-agent both apply; an entrypoint that is not .js gets none, and another language gets none.
|
||||
func TestATypeScriptEntrypointIsStartedByTheLauncherBesideIt(t *testing.T) {
|
||||
got := Launchers("typescript", []string{"index.js", "tools/x.js", "data.json"})
|
||||
want := map[string]string{"index.js": "index.serve.mjs", "tools/x.js": "tools/x.serve.mjs"}
|
||||
if !reflect.DeepEqual(got, want) {
|
||||
t.Fatalf("Launchers(typescript) = %v, want %v", got, want)
|
||||
}
|
||||
if got := Launchers("go", []string{"index.js"}); len(got) != 0 {
|
||||
t.Fatalf("a Go bundle got launchers: %v", got)
|
||||
}
|
||||
if got := Launchers("typescript", nil); len(got) != 0 {
|
||||
t.Fatalf("a bundle with no entrypoint got launchers: %v", got)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,71 @@
|
||||
package buildstep
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// narrowKept are the files of a module's directory a narrowed build keeps beside its named paths: its manifest and
|
||||
// its lock files, each an input of the module's key (ModuleTree's Manifest and Locks). package.json and go.mod are
|
||||
// not kept: neither is in the named tree, so a build that needs one names it among its artifact's paths, which puts
|
||||
// it in the key (novox/hq ADR 0499 clause 4).
|
||||
var narrowKept = append([]string{"module.json"}, lockFiles...)
|
||||
|
||||
// Narrow leaves in the fetched tree only what a build of an artifact that names its paths may see (novox/hq ADR 0499
|
||||
// clause 4): r.Dir's named paths, its manifest and lock files, and .git; everything else is removed, so a file outside
|
||||
// them cannot change what is built without changing its key. A named path absent at the tree is an error naming it,
|
||||
// and nothing is removed then. With no paths in r it changes nothing. Both build modules' fetch steps call it.
|
||||
func Narrow(tree string, r FetchRequest) error {
|
||||
if len(r.Paths) == 0 {
|
||||
return nil
|
||||
}
|
||||
keep := map[string]bool{".git": true}
|
||||
for _, p := range r.Paths {
|
||||
rel := path.Join(r.Dir, p)
|
||||
if strings.HasPrefix(rel, "../") || rel == ".." || path.IsAbs(p) {
|
||||
return fmt.Errorf("the named path %q leaves the module's directory", p)
|
||||
}
|
||||
if _, err := os.Lstat(filepath.Join(tree, filepath.FromSlash(rel))); err != nil {
|
||||
return fmt.Errorf("the named path %q is not in the fetched tree: %w", p, err)
|
||||
}
|
||||
keep[rel] = true
|
||||
}
|
||||
for _, f := range narrowKept {
|
||||
keep[path.Join(r.Dir, f)] = true
|
||||
}
|
||||
return prune(tree, "", keep)
|
||||
}
|
||||
|
||||
// prune removes from dir (rel within the tree) every entry that is neither kept nor on the way to one.
|
||||
func prune(root, rel string, keep map[string]bool) error {
|
||||
entries, err := os.ReadDir(filepath.Join(root, filepath.FromSlash(rel)))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, e := range entries {
|
||||
p := path.Join(rel, e.Name())
|
||||
if keep[p] {
|
||||
continue
|
||||
}
|
||||
ancestor := false
|
||||
for k := range keep {
|
||||
if strings.HasPrefix(k, p+"/") {
|
||||
ancestor = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if ancestor && e.IsDir() {
|
||||
if err := prune(root, p, keep); err != nil {
|
||||
return err
|
||||
}
|
||||
continue
|
||||
}
|
||||
if err := os.RemoveAll(filepath.Join(root, filepath.FromSlash(p))); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,82 @@
|
||||
package buildstep
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func filesUnder(t *testing.T, root string) []string {
|
||||
t.Helper()
|
||||
var out []string
|
||||
_ = filepath.Walk(root, func(p string, info os.FileInfo, err error) error {
|
||||
if err == nil && !info.IsDir() {
|
||||
rel, _ := filepath.Rel(root, p)
|
||||
out = append(out, filepath.ToSlash(rel))
|
||||
}
|
||||
return nil
|
||||
})
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// A build of an artifact that names its paths sees only those paths, its manifest and its lock files (novox/hq ADR 0499
|
||||
// clause 4): Narrow removes everything else of the fetched tree but .git, package.json and go.mod among it, since
|
||||
// neither is in the named tree nor among the lock files; with no paths it changes nothing.
|
||||
func TestNarrowLeavesOnlyTheNamedPathsTheManifestAndTheLocks(t *testing.T) {
|
||||
tree := t.TempDir()
|
||||
for _, f := range []string{".git/HEAD", "module.json", "package.json", "package-lock.json", "go.sum", "src/index.ts",
|
||||
"src/deep/a.ts", "test/a.test.ts", "tsconfig.json", "go/delivery/x.go", "README.md", "other/module.json"} {
|
||||
writeTreeFile(t, tree, f, "x")
|
||||
}
|
||||
if err := Narrow(tree, FetchRequest{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := len(filesUnder(t, tree)); got != 12 {
|
||||
t.Fatalf("a request without paths changed the tree: %d files", got)
|
||||
}
|
||||
if err := Narrow(tree, FetchRequest{Dir: ".", Paths: []string{"src", "test", "tsconfig.json"}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := ".git/HEAD,go.sum,module.json,package-lock.json,src/deep/a.ts,src/index.ts,test/a.test.ts,tsconfig.json"
|
||||
if got := strings.Join(filesUnder(t, tree), ","); got != want {
|
||||
t.Fatalf("the narrowed tree holds %s\nwant %s", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNarrowKeepsAModuleDirectorysPathsAndFailsOnAnAbsentOne(t *testing.T) {
|
||||
tree := t.TempDir()
|
||||
for _, f := range []string{".git/config", "modules/a/module.json", "modules/a/go.mod", "modules/a/src/x.ts",
|
||||
"modules/a/notes.md", "modules/b/module.json", "root.txt"} {
|
||||
writeTreeFile(t, tree, f, "x")
|
||||
}
|
||||
err := Narrow(tree, FetchRequest{Dir: "modules/a", Paths: []string{"src", "gone"}})
|
||||
if err == nil || !strings.Contains(err.Error(), `"gone"`) {
|
||||
t.Fatalf("an absent named path gave %v", err)
|
||||
}
|
||||
if len(filesUnder(t, tree)) != 7 {
|
||||
t.Fatal("a refused narrowing removed files")
|
||||
}
|
||||
if err := Narrow(tree, FetchRequest{Dir: "modules/a", Paths: []string{"src"}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := strings.Join(filesUnder(t, tree), ","); got != ".git/config,modules/a/module.json,modules/a/src/x.ts" {
|
||||
t.Fatalf("the narrowed tree holds %s", got)
|
||||
}
|
||||
}
|
||||
|
||||
// A build that needs package.json or go.mod names it among its paths, and so keys on it (ADR 0499 clause 4).
|
||||
func TestNarrowKeepsAManifestOfTheLanguageOnlyWhenNamed(t *testing.T) {
|
||||
tree := t.TempDir()
|
||||
for _, f := range []string{"module.json", "package.json", "go.mod", "src/a.ts"} {
|
||||
writeTreeFile(t, tree, f, "x")
|
||||
}
|
||||
if err := Narrow(tree, FetchRequest{Dir: ".", Paths: []string{"src", "package.json"}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := strings.Join(filesUnder(t, tree), ","); got != "module.json,package.json,src/a.ts" {
|
||||
t.Fatalf("the narrowed tree holds %s", got)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,115 @@
|
||||
package buildstep
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"path"
|
||||
"regexp"
|
||||
"strings"
|
||||
)
|
||||
|
||||
var (
|
||||
requestCommitOK = regexp.MustCompile(`^[0-9a-f]{40}$`)
|
||||
requestRepositoryOK = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._-]*/[A-Za-z0-9][A-Za-z0-9._-]*$`)
|
||||
)
|
||||
|
||||
// FetchRequest is what a build module's fetch tool is asked: the repository and the full commit, the pull request's
|
||||
// ref when only its head carries the commit, the trunk commit to merge onto for a merge preview, and, for a build of a
|
||||
// module whose every artifact names its paths, that module's directory and its named paths (novox/hq ADR 0499 clause 4).
|
||||
//
|
||||
// **The paths come from the builder, not from a manifest** (ADR 0495 clause 1; the lead's decision L8 of R4e): the
|
||||
// builder fills Dir and Paths from the indexer's Module value (Path, and KeptInputs2.Paths' keys), and the build module
|
||||
// gives the build and test steps a workspace holding those paths, the manifest and its lock files. A request without
|
||||
// them builds from the module's directory, as before.
|
||||
type FetchRequest struct {
|
||||
Repository string `json:"repository"`
|
||||
Commit string `json:"commit"`
|
||||
Ref string `json:"ref,omitempty"`
|
||||
Base string `json:"base,omitempty"`
|
||||
Dir string `json:"dir,omitempty"`
|
||||
Paths []string `json:"paths,omitempty"`
|
||||
}
|
||||
|
||||
// FetchRequestProperties are the JSON schema properties of FetchRequest's dir and paths, for a fetch tool's Input.
|
||||
func FetchRequestProperties() map[string]any {
|
||||
return map[string]any{
|
||||
"dir": map[string]any{"type": "string", "description": "the module's directory in the repository (\".\" for its root), with paths"},
|
||||
"paths": map[string]any{"type": "array", "items": map[string]any{"type": "string"},
|
||||
"description": "the paths the module names, relative to dir, from the indexer's kept inputs: the build sees only these, the manifest and its lock files"},
|
||||
}
|
||||
}
|
||||
|
||||
// Args is r as a tool call's arguments.
|
||||
func (r FetchRequest) Args() map[string]any {
|
||||
args := map[string]any{"repository": r.Repository, "commit": r.Commit}
|
||||
for k, v := range map[string]string{"ref": r.Ref, "base": r.Base, "dir": r.Dir} {
|
||||
if v != "" {
|
||||
args[k] = v
|
||||
}
|
||||
}
|
||||
if len(r.Paths) > 0 {
|
||||
args["paths"] = append([]string(nil), r.Paths...)
|
||||
}
|
||||
return args
|
||||
}
|
||||
|
||||
// ParseFetchRequest reads a fetch tool's arguments; arguments it does not know (a build module's own) are left to the
|
||||
// caller. It refuses a repository that is not <owner>/<name>, a commit or a base that is not 40 hex digits, paths with no
|
||||
// module directory or a directory with no path, a directory leaving the repository and a path leaving the directory,
|
||||
// naming each.
|
||||
func ParseFetchRequest(args map[string]any) (FetchRequest, error) {
|
||||
var r FetchRequest
|
||||
r.Repository, _ = args["repository"].(string)
|
||||
r.Commit, _ = args["commit"].(string)
|
||||
r.Ref, _ = args["ref"].(string)
|
||||
r.Base, _ = args["base"].(string)
|
||||
r.Dir, _ = args["dir"].(string)
|
||||
switch v := args["paths"].(type) {
|
||||
case nil:
|
||||
case []string:
|
||||
r.Paths = append([]string(nil), v...)
|
||||
case []any:
|
||||
for _, p := range v {
|
||||
s, ok := p.(string)
|
||||
if !ok {
|
||||
return FetchRequest{}, fmt.Errorf("paths is a list of strings")
|
||||
}
|
||||
r.Paths = append(r.Paths, s)
|
||||
}
|
||||
default:
|
||||
return FetchRequest{}, fmt.Errorf("paths is a list of strings")
|
||||
}
|
||||
return r, r.Check()
|
||||
}
|
||||
|
||||
// Check is what ParseFetchRequest refuses.
|
||||
func (r FetchRequest) Check() error {
|
||||
if !requestRepositoryOK.MatchString(r.Repository) || strings.Contains(r.Repository, "..") {
|
||||
return fmt.Errorf("%q is not a repository, <owner>/<name>", r.Repository)
|
||||
}
|
||||
if !requestCommitOK.MatchString(r.Commit) {
|
||||
return fmt.Errorf("%q is not a full commit (40 hex digits)", r.Commit)
|
||||
}
|
||||
if r.Base != "" && !requestCommitOK.MatchString(r.Base) {
|
||||
return fmt.Errorf("%q is not a trunk commit (40 hex digits)", r.Base)
|
||||
}
|
||||
switch {
|
||||
case len(r.Paths) > 0 && r.Dir == "":
|
||||
return fmt.Errorf("the request names paths and names no module directory (dir) they are relative to")
|
||||
case len(r.Paths) == 0 && r.Dir != "":
|
||||
return fmt.Errorf("the request names a module directory and no path")
|
||||
}
|
||||
if r.Dir != "" && (outside(r.Dir) || strings.HasPrefix(r.Dir, "/")) {
|
||||
return fmt.Errorf("the module directory %q leaves the repository", r.Dir)
|
||||
}
|
||||
for _, p := range r.Paths {
|
||||
if p == "" || strings.HasPrefix(p, "/") || outside(p) || path.Clean(p) == "." {
|
||||
return fmt.Errorf("the path %q leaves the module's directory or is not a path in it", p)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func outside(p string) bool {
|
||||
c := path.Clean(p)
|
||||
return c == ".." || strings.HasPrefix(c, "../")
|
||||
}
|
||||
@@ -0,0 +1,57 @@
|
||||
package buildstep
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// A build's fetch is handed the paths of a module that names them (novox/hq ADR 0499 clause 4): the builder fills them
|
||||
// from the indexer's kept inputs and the build module reads them from the call, never from the manifest (ADR 0495
|
||||
// clause 1). The request survives the trip through a tool call's arguments.
|
||||
func TestAFetchRequestCarriesTheNamedPathsThroughTheToolCall(t *testing.T) {
|
||||
req := FetchRequest{Repository: "novox/mesh-sdk", Commit: strings.Repeat("a", 40), Ref: "refs/pull/7/head",
|
||||
Dir: ".", Paths: []string{"package.json", "src"}}
|
||||
args := req.Args()
|
||||
// A tool call's arguments arrive as JSON: lists of any.
|
||||
args["paths"] = []any{"package.json", "src"}
|
||||
got, err := ParseFetchRequest(args)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !reflect.DeepEqual(got, req) {
|
||||
t.Fatalf("got %+v, want %+v", got, req)
|
||||
}
|
||||
plain, err := ParseFetchRequest(map[string]any{"repository": "novox/x", "commit": strings.Repeat("b", 40),
|
||||
"base": strings.Repeat("c", 40), "beside": []any{}})
|
||||
if err != nil || plain.Paths != nil || plain.Base != strings.Repeat("c", 40) {
|
||||
t.Fatalf("a request without paths: %+v, %v", plain, err)
|
||||
}
|
||||
|
||||
ok := map[string]any{"repository": "novox/x", "commit": strings.Repeat("b", 40)}
|
||||
for _, c := range []struct {
|
||||
set map[string]any
|
||||
says string
|
||||
}{
|
||||
{map[string]any{"repository": "novox"}, "is not a repository"},
|
||||
{map[string]any{"repository": "novox/../x"}, "is not a repository"},
|
||||
{map[string]any{"commit": "abc"}, "is not a full commit"},
|
||||
{map[string]any{"base": "abc"}, "is not a trunk commit"},
|
||||
{map[string]any{"paths": []any{"src"}}, "names no module directory"},
|
||||
{map[string]any{"dir": "modules/a", "paths": []any{"../b"}}, "leaves the module's directory"},
|
||||
{map[string]any{"dir": "../a", "paths": []any{"src"}}, "leaves the repository"},
|
||||
{map[string]any{"dir": "modules/a", "paths": []any{3}}, "paths is a list of strings"},
|
||||
{map[string]any{"dir": "modules/a"}, "names a module directory and no path"},
|
||||
} {
|
||||
args := map[string]any{}
|
||||
for k, v := range ok {
|
||||
args[k] = v
|
||||
}
|
||||
for k, v := range c.set {
|
||||
args[k] = v
|
||||
}
|
||||
if _, err := ParseFetchRequest(args); err == nil || !strings.Contains(err.Error(), c.says) {
|
||||
t.Errorf("%v: %v, wanted a refusal saying %q", c.set, err, c.says)
|
||||
}
|
||||
}
|
||||
}
|
||||
+49
@@ -0,0 +1,49 @@
|
||||
// The probe a run step's isolation is proved with (novox/hq ADR 0448; issue 1309, S19): run inside the step, it says
|
||||
// on stdout every way it reached what a step must not, and "shut off" when it reached none. Its arguments: the files
|
||||
// it must not read (the forge key, a bus login), a TCP address and a unix socket of the machine it must not reach, a
|
||||
// name it must not resolve, and a cache file it writes, which must be unchanged after the step.
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net"
|
||||
"os"
|
||||
"time"
|
||||
)
|
||||
|
||||
func main() {
|
||||
if len(os.Args) != 7 {
|
||||
fmt.Println("usage: probe <key> <login> <tcp address> <unix socket> <name> <cache file>")
|
||||
os.Exit(2)
|
||||
}
|
||||
key, login, tcp, socket, name, cache := os.Args[1], os.Args[2], os.Args[3], os.Args[4], os.Args[5], os.Args[6]
|
||||
reached := 0
|
||||
say := func(f string, a ...any) { reached++; fmt.Printf(f+"\n", a...) }
|
||||
for _, f := range []string{key, login} {
|
||||
if _, err := os.ReadFile(f); err == nil {
|
||||
say("read %s", f)
|
||||
}
|
||||
}
|
||||
if c, err := net.DialTimeout("tcp", tcp, 2*time.Second); err == nil {
|
||||
c.Close()
|
||||
say("reached %s", tcp)
|
||||
}
|
||||
if c, err := net.DialTimeout("unix", socket, 2*time.Second); err == nil {
|
||||
c.Close()
|
||||
say("reached %s", socket)
|
||||
}
|
||||
if addrs, err := net.LookupHost(name); err == nil {
|
||||
say("resolved %s to %v", name, addrs)
|
||||
}
|
||||
if home, err := os.UserHomeDir(); err == nil {
|
||||
if entries, err := os.ReadDir(home); err == nil && len(entries) > 0 {
|
||||
say("the home %s holds %d entries", home, len(entries))
|
||||
}
|
||||
}
|
||||
if err := os.WriteFile(cache, []byte("written by the step\n"), 0o644); err != nil {
|
||||
say("could not write the cache under its throwaway layer: %v", err)
|
||||
}
|
||||
if reached == 0 {
|
||||
fmt.Println("shut off")
|
||||
}
|
||||
}
|
||||
+57
-1
@@ -6,9 +6,12 @@ import (
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/forgekey"
|
||||
)
|
||||
|
||||
@@ -19,12 +22,16 @@ type ModuleTree struct {
|
||||
Tree string `json:"tree"`
|
||||
Manifest string `json:"manifest"`
|
||||
Locks map[string]string `json:"locks,omitempty"`
|
||||
// Paths are, of a module whose every artifact names its paths, each named path's git object id at the commit; Tree
|
||||
// is then their delivery.NamedTree with Manifest (novox/hq ADR 0499 clause 2). Absent otherwise.
|
||||
Paths map[string]string `json:"paths,omitempty"`
|
||||
}
|
||||
|
||||
var lockFiles = []string{"go.sum", "package-lock.json"}
|
||||
|
||||
// ModuleTrees are the trees, in commit, of every directory of the workspace dir that holds a module.json, by directory
|
||||
// (the repository's root is "."). The workspace is at commit, so the manifests and the lock files are read from it. env is
|
||||
// (the repository's root is "."); for a module whose every artifact names its paths, the named tree of those paths and
|
||||
// their objects (novox/hq ADR 0499 clause 2). The workspace is at commit, so the manifests and the lock files are read from it. env is
|
||||
// the environment git runs with (nil for none). Both build modules' fetch steps answer it for a merge.
|
||||
func ModuleTrees(ctx context.Context, run forgekey.Runner, env []string, dir, commit string) (map[string]ModuleTree, error) {
|
||||
r := run(ctx, env, "git", "-C", dir, "ls-files", "--", "module.json", "*/module.json")
|
||||
@@ -51,6 +58,9 @@ func ModuleTrees(ctx context.Context, run forgekey.Runner, env []string, dir, co
|
||||
return nil, err
|
||||
}
|
||||
mt.Manifest = sha256Hex(raw)
|
||||
if err := narrow(ctx, run, env, dir, commit, mdir, raw, &mt); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, lock := range lockFiles {
|
||||
if raw, err := os.ReadFile(filepath.Join(dir, filepath.FromSlash(mdir), lock)); err == nil {
|
||||
if mt.Locks == nil {
|
||||
@@ -64,4 +74,50 @@ func ModuleTrees(ctx context.Context, run forgekey.Runner, env []string, dir, co
|
||||
return trees, nil
|
||||
}
|
||||
|
||||
// narrow sets mt's Paths and its named Tree when the module's manifest names paths on every artifact (novox/hq ADR 0499
|
||||
// clause 2): one `git ls-tree -z` gives each named path's object at commit. The merge preview reads the manifest here, as
|
||||
// the indexer does, with catalogue.ParseManifestForInputs (issue 1751), and no other build step reads paths from one (ADR 0495 clause 1). A manifest that does not parse
|
||||
// is not narrowed: the indexer gives it none. A named path absent at commit fails, naming the module and the path.
|
||||
func narrow(ctx context.Context, run forgekey.Runner, env []string, dir, commit, mdir string, raw []byte, mt *ModuleTree) error {
|
||||
m, err := catalogue.ParseManifestForInputs(raw)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
paths, every := m.NamedPaths()
|
||||
if !every {
|
||||
return nil
|
||||
}
|
||||
full := make([]string, len(paths))
|
||||
for i, p := range paths {
|
||||
full[i] = path.Join(mdir, p)
|
||||
}
|
||||
r := run(ctx, env, "git", append([]string{"-C", dir, "ls-tree", "-z", commit, "--"}, full...)...)
|
||||
if r.Status != 0 || r.Err != "" {
|
||||
return fmt.Errorf("git ls-tree %s of the module %s: %s", commit, mdir, firstLine(r.Err+" "+r.Stderr))
|
||||
}
|
||||
objects := map[string]string{}
|
||||
for _, l := range strings.Split(r.Stdout, "\x00") {
|
||||
meta, name, ok := strings.Cut(l, "\t")
|
||||
fields := strings.Fields(meta)
|
||||
if !ok || len(fields) != 3 {
|
||||
continue
|
||||
}
|
||||
objects[name] = fields[2]
|
||||
}
|
||||
mt.Paths = map[string]string{}
|
||||
for i, p := range paths {
|
||||
object := objects[full[i]]
|
||||
if object == "" {
|
||||
return fmt.Errorf("the module %s names the path %q, which is not in %s", mdir, p, commit)
|
||||
}
|
||||
mt.Paths[p] = object
|
||||
}
|
||||
named, err := delivery.NamedTree(mt.Manifest, mt.Paths)
|
||||
if err != nil {
|
||||
return fmt.Errorf("the module %s: %w", mdir, err)
|
||||
}
|
||||
mt.Tree = named
|
||||
return nil
|
||||
}
|
||||
|
||||
func sha256Hex(b []byte) string { h := sha256.Sum256(b); return hex.EncodeToString(h[:]) }
|
||||
|
||||
@@ -0,0 +1,63 @@
|
||||
//go:build integration
|
||||
|
||||
package buildstep
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/forgekey"
|
||||
)
|
||||
|
||||
// On a real repository shaped like mesh-sdk (novox/hq ADR 0499 clauses 2 and 6), a commit touching only the Go package
|
||||
// leaves the TypeScript package's tree value equal, and a commit touching its source changes it; a module that names no
|
||||
// path keeps its directory's tree id.
|
||||
func TestAGoOnlyCommitLeavesTheNamedTreeEqualOnRealGit(t *testing.T) {
|
||||
repo := t.TempDir()
|
||||
realGit(t, repo, "init", "-q", "-b", "main")
|
||||
writeFile(t, repo, "module.json", `{"module":"mesh-sdk","version":"1","build":{"artifacts":[`+
|
||||
`{"name":"lib","kind":"package","language":"typescript","paths":["src","test","package.json","package-lock.json","tsconfig.json"]}]}}`)
|
||||
writeFile(t, repo, "src/index.ts", "export const a = 1\n")
|
||||
writeFile(t, repo, "test/a.test.ts", "// a\n")
|
||||
writeFile(t, repo, "package.json", "{}\n")
|
||||
writeFile(t, repo, "package-lock.json", "{}\n")
|
||||
writeFile(t, repo, "tsconfig.json", "{}\n")
|
||||
writeFile(t, repo, "go/delivery/x.go", "package delivery\n")
|
||||
writeFile(t, repo, "other/module.json", `{"module":"other","version":"1"}`)
|
||||
writeFile(t, repo, "other/a.txt", "a\n")
|
||||
commit := func(msg string) string {
|
||||
realGit(t, repo, "add", "-A")
|
||||
realGit(t, repo, "commit", "-q", "-m", msg)
|
||||
return realGit(t, repo, "rev-parse", "HEAD")
|
||||
}
|
||||
trees := func(c string) map[string]ModuleTree {
|
||||
got, err := ModuleTrees(context.Background(), forgekey.ExecRunner, nil, repo, c)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return got
|
||||
}
|
||||
first := trees(commit("first"))
|
||||
if !strings.HasPrefix(first["."].Tree, "named:") || len(first["."].Paths) != 5 {
|
||||
t.Fatalf("the root module was not narrowed: %+v", first["."])
|
||||
}
|
||||
if want := realGit(t, repo, "rev-parse", "HEAD:other"); first["other"].Tree != want || first["other"].Paths != nil {
|
||||
t.Fatalf("a module naming no path: %+v, want its directory's tree %s", first["other"], want)
|
||||
}
|
||||
|
||||
writeFile(t, repo, "go/delivery/x.go", "package delivery\n\nconst B = 2\n")
|
||||
goOnly := trees(commit("go only"))
|
||||
if goOnly["."].Tree != first["."].Tree {
|
||||
t.Fatalf("a Go-only commit changed the TypeScript package's tree: %s, was %s", goOnly["."].Tree, first["."].Tree)
|
||||
}
|
||||
|
||||
writeFile(t, repo, "src/index.ts", "export const a = 2\n")
|
||||
src := trees(commit("src"))
|
||||
if src["."].Tree == first["."].Tree {
|
||||
t.Fatal("a commit to src/ left the TypeScript package's tree equal")
|
||||
}
|
||||
if src["."].Paths["src"] != realGit(t, repo, "rev-parse", "HEAD:src") {
|
||||
t.Fatalf("the object of src is %s, git says %s", src["."].Paths["src"], realGit(t, repo, "rev-parse", "HEAD:src"))
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,113 @@
|
||||
package buildstep
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/forgekey"
|
||||
)
|
||||
|
||||
// A module whose every artifact names its paths takes its tree from them (novox/hq ADR 0499 clause 2): ModuleTrees asks
|
||||
// git for each named path's object at the commit, keeps them in Paths and answers their delivery.NamedTree as Tree; a
|
||||
// module naming none keeps its directory's tree, as before.
|
||||
func TestModuleTreesNarrowsAModuleWhoseArtifactsNameTheirPaths(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
sdk := `{"module":"mesh-sdk","version":"1","build":{"artifacts":[{"name":"lib","kind":"package","language":"typescript","paths":["src","package.json"]}]}}`
|
||||
writeTreeFile(t, root, "module.json", sdk)
|
||||
writeTreeFile(t, root, "modules/b/module.json", `{"module":"b","version":"1"}`)
|
||||
merge := strings.Repeat("c", 40)
|
||||
var lsTree []string
|
||||
run := func(ctx context.Context, env []string, name string, args ...string) forgekey.Ran {
|
||||
switch args[2] {
|
||||
case "ls-files":
|
||||
return forgekey.Ran{Stdout: "module.json\nmodules/b/module.json\n"}
|
||||
case "rev-parse":
|
||||
return forgekey.Ran{Stdout: "tree-of-" + args[3] + "\n"}
|
||||
case "ls-tree":
|
||||
lsTree = append(lsTree, strings.Join(args[4:], " "))
|
||||
return forgekey.Ran{Stdout: "100644 blob " + strings.Repeat("b", 40) + "\tpackage.json\x00" +
|
||||
"040000 tree " + strings.Repeat("a", 40) + "\tsrc\x00"}
|
||||
}
|
||||
return forgekey.Ran{Status: 1, Stderr: "unexpected " + args[2]}
|
||||
}
|
||||
got, err := ModuleTrees(context.Background(), run, nil, root, merge)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
paths := map[string]string{"src": strings.Repeat("a", 40), "package.json": strings.Repeat("b", 40)}
|
||||
want, _ := delivery.NamedTree(hexOf(sdk), paths)
|
||||
if g := got["."]; g.Tree != want || len(g.Paths) != 2 || g.Paths["src"] != paths["src"] || g.Paths["package.json"] != paths["package.json"] {
|
||||
t.Fatalf("the root module = %+v, want tree %s and paths %v", g, want, paths)
|
||||
}
|
||||
if len(lsTree) != 1 || lsTree[0] != merge+" -- package.json src" {
|
||||
t.Fatalf("git ls-tree ran as %v", lsTree)
|
||||
}
|
||||
if g := got["modules/b"]; g.Tree != "tree-of-"+merge+":modules/b" || g.Paths != nil {
|
||||
t.Fatalf("a module naming no path was narrowed: %+v", g)
|
||||
}
|
||||
}
|
||||
|
||||
// A named path absent at the commit fails the answer, naming the module and the path: a tree computed without it would
|
||||
// pin a build source that does not exist.
|
||||
func TestModuleTreesFailsNamingANamedPathAbsentAtTheCommit(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
writeTreeFile(t, root, "modules/a/module.json", `{"module":"a","version":"1","build":{"artifacts":[{"name":"lib","kind":"package","language":"typescript","paths":["src","gone"]}]}}`)
|
||||
run := func(ctx context.Context, env []string, name string, args ...string) forgekey.Ran {
|
||||
switch args[2] {
|
||||
case "ls-files":
|
||||
return forgekey.Ran{Stdout: "modules/a/module.json\n"}
|
||||
case "rev-parse":
|
||||
return forgekey.Ran{Stdout: "t\n"}
|
||||
case "ls-tree":
|
||||
if strings.Join(args[4:], " ") != strings.Repeat("c", 40)+" -- modules/a/gone modules/a/src" {
|
||||
return forgekey.Ran{Status: 1, Stderr: "wrong arguments " + strings.Join(args[3:], " ")}
|
||||
}
|
||||
return forgekey.Ran{Stdout: "040000 tree " + strings.Repeat("a", 40) + "\tmodules/a/src\x00"}
|
||||
}
|
||||
return forgekey.Ran{Status: 1}
|
||||
}
|
||||
_, err := ModuleTrees(context.Background(), run, nil, root, strings.Repeat("c", 40))
|
||||
if err == nil || !strings.Contains(err.Error(), `"gone"`) || !strings.Contains(err.Error(), "modules/a") {
|
||||
t.Fatalf("an absent named path gave %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// A manifest whose config value falls back to a predefined key is narrowed here as the indexer narrows it: both read it
|
||||
// with catalogue.ParseManifestForInputs (issue 1751), so a build module holding no published list computes the same
|
||||
// tree (ADR 0499 clause 2).
|
||||
func TestModuleTreesNarrowsAManifestFallingBackToAPredefinedKey(t *testing.T) {
|
||||
was := catalogue.ReplacePublished(nil) // the build modules' case: no published list
|
||||
defer catalogue.ReplacePublished(was)
|
||||
root := t.TempDir()
|
||||
raw := `{"module":"xterm","version":"1","config":{"size":{"type":"number","description":"The text size.","predefined":"font-size","default":12,"runtime":true}},` +
|
||||
`"build":{"artifacts":[{"name":"lib","kind":"package","language":"typescript","paths":["src"]}]}}`
|
||||
if _, err := catalogue.ParseManifest([]byte(raw)); err == nil {
|
||||
t.Fatal("ParseManifest reads the manifest with no list; the test proves nothing")
|
||||
}
|
||||
if _, err := catalogue.ParseManifestForInputs([]byte(raw)); err != nil {
|
||||
t.Fatalf("the indexer's read refuses the manifest: %v", err)
|
||||
}
|
||||
writeTreeFile(t, root, "module.json", raw)
|
||||
run := func(ctx context.Context, env []string, name string, args ...string) forgekey.Ran {
|
||||
switch args[2] {
|
||||
case "ls-files":
|
||||
return forgekey.Ran{Stdout: "module.json\n"}
|
||||
case "rev-parse":
|
||||
return forgekey.Ran{Stdout: "whole\n"}
|
||||
case "ls-tree":
|
||||
return forgekey.Ran{Stdout: "040000 tree " + strings.Repeat("a", 40) + "\tsrc\x00"}
|
||||
}
|
||||
return forgekey.Ran{Status: 1}
|
||||
}
|
||||
got, err := ModuleTrees(context.Background(), run, nil, root, strings.Repeat("c", 40))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want, _ := delivery.NamedTree(hexOf(raw), map[string]string{"src": strings.Repeat("a", 40)})
|
||||
if g := got["."]; g.Tree != want || g.Paths["src"] == "" {
|
||||
t.Fatalf("the module = %+v, want the named tree %s", g, want)
|
||||
}
|
||||
}
|
||||
@@ -394,6 +394,72 @@ func (b *Build) problems(module string) []string {
|
||||
"%s: %q mirrors %q, which names no tag or digest", module, a.Name, a.From))
|
||||
}
|
||||
problems = append(problems, upstreamFileProblems(module, a)...)
|
||||
problems = append(problems, pathsProblems(module, a)...)
|
||||
}
|
||||
return problems
|
||||
}
|
||||
|
||||
// NamedPaths are the paths a module's fingerprint takes its tree from (novox/hq ADR 0499 clause 2): the union of the
|
||||
// paths its artifacts name, cleaned and sorted, and every true only when the module builds something and every artifact
|
||||
// names paths; otherwise none, and the module keeps its directory's tree.
|
||||
//
|
||||
// **Read by the core alone** (ADR 0495 clause 1, the lead's decision L8 of R4e): the indexer and its merge preview
|
||||
// (buildstep.ModuleTrees) call it. A build module's fetch step is handed the paths by the builder, from the indexer's
|
||||
// kept inputs (delivery.KeptInputs2.Paths), and reads no manifest for them.
|
||||
func (m Manifest) NamedPaths() (paths []string, every bool) {
|
||||
if m.Build == nil || len(m.Build.Artifacts) == 0 {
|
||||
return nil, false
|
||||
}
|
||||
seen := map[string]bool{}
|
||||
for _, a := range m.Build.Artifacts {
|
||||
if len(a.Paths) == 0 {
|
||||
return nil, false
|
||||
}
|
||||
for _, p := range a.Paths {
|
||||
seen[path.Clean(p)] = true
|
||||
}
|
||||
}
|
||||
for p := range seen {
|
||||
paths = append(paths, p)
|
||||
}
|
||||
sort.Strings(paths)
|
||||
return paths, true
|
||||
}
|
||||
|
||||
// pathsProblems is what is wrong with an artifact's `paths` (novox/hq ADR 0499 clause 1): named on a kind whose build
|
||||
// source is derived or named otherwise, or a path that leaves the module's directory, is its whole directory, or is a
|
||||
// pattern.
|
||||
func pathsProblems(module string, a Artifact) []string {
|
||||
if a.Paths == nil {
|
||||
return nil
|
||||
}
|
||||
switch {
|
||||
case a.Kind == ArtifactBundle && a.Language == "go":
|
||||
return []string{fmt.Sprintf("%s: %q is a Go program and names paths: its build source is derived from the "+
|
||||
"package it compiles (novox/hq ADR 0267 clause 1, ADR 0499 clause 1)", module, a.Name)}
|
||||
case a.Kind != ArtifactBundle && a.Kind != ArtifactPackage:
|
||||
return []string{fmt.Sprintf("%s: %q is a %q and names paths; only a package or a bundle in a language other "+
|
||||
"than Go names the paths it is built from (novox/hq ADR 0499 clause 1)", module, a.Name, a.Kind)}
|
||||
case len(a.Paths) == 0:
|
||||
return []string{fmt.Sprintf("%s: %q names no path in paths; leave the word out to build from the module's "+
|
||||
"directory", module, a.Name)}
|
||||
}
|
||||
var problems []string
|
||||
for _, p := range a.Paths {
|
||||
clean := path.Clean(p)
|
||||
switch {
|
||||
case strings.TrimSpace(p) == "":
|
||||
problems = append(problems, fmt.Sprintf("%s: %q names an empty path", module, a.Name))
|
||||
case strings.HasPrefix(p, "/") || clean == ".." || strings.HasPrefix(clean, "../"):
|
||||
problems = append(problems, fmt.Sprintf("%s: %q names the path %q, which leaves the module's directory",
|
||||
module, a.Name, p))
|
||||
case clean == ".":
|
||||
problems = append(problems, fmt.Sprintf("%s: %q names the module's whole directory (%q) as a path; leave "+
|
||||
"the word out for that", module, a.Name, p))
|
||||
case strings.ContainsAny(p, "*?[]{}\\"):
|
||||
problems = append(problems, fmt.Sprintf("%s: %q names %q, which is a pattern; paths names files and "+
|
||||
"directories", module, a.Name, p))
|
||||
}
|
||||
}
|
||||
return problems
|
||||
}
|
||||
|
||||
@@ -42,7 +42,8 @@ func deliveryVerbs() []Verb {
|
||||
"paths": "the files it changes, comma-separated, from the repository's root",
|
||||
"base": "the branch it merges into (default main)"}, []string{"repository", "paths"})},
|
||||
{Name: "checks", Description: "What the mesh's checks said of a pull request's head or of one commit: each " +
|
||||
"of the commit's mesh statuses (mesh/merge-gate, mesh/repo-check, mesh/delivery, …) with its state, " +
|
||||
"of the commit's mesh statuses (mesh/check, the one a main requires from R4e, hq's mesh/repo-check, " +
|
||||
"mesh/delivery, …) with its state, " +
|
||||
"description and when it was set; the merge check's full verdict as the controller said it — each " +
|
||||
"layer's summary, the machine that ran it, when, its build id and its report; and whether the branch's " +
|
||||
"protection would let it merge, every required status being success.",
|
||||
|
||||
@@ -1191,6 +1191,17 @@ type Artifact struct {
|
||||
// (novox/hq ADR 0193). Absent for nearly every bundle, which is then one file per entrypoint.
|
||||
External []string `json:"external,omitempty"`
|
||||
|
||||
// Paths are the files and directories, relative to the module's directory, a package or a bundle in a language
|
||||
// other than Go is built from (novox/hq ADR 0499 clause 1): with the module's manifest they are its build source,
|
||||
// and a module whose every artifact names them takes its fingerprint's tree from them alone (clause 2). Absent means
|
||||
// the module's directory, as before.
|
||||
//
|
||||
// **Named, because the build source of a package sharing a repository cannot be derived.** A Go program's is
|
||||
// (ADR 0267 clause 1), an archive's `from` names it and an image's is the context its manifest names, so each of those
|
||||
// refuses the word. A path stays inside the module's directory and is no pattern: a build source that could reach
|
||||
// out of it, or grow with a directory listing, would be one no fingerprint pins.
|
||||
Paths []string `json:"paths,omitempty"`
|
||||
|
||||
// Env is what a tools bundle is given on a machine (novox/hq ADR 0192): words and their values,
|
||||
// paths and constants composed with ${dir:…} and ${port:…} exactly as a container's environment
|
||||
// is, never a secret's content. The node's runtime hands it to this bundle and to no other.
|
||||
|
||||
@@ -0,0 +1,29 @@
|
||||
package catalogue
|
||||
|
||||
import (
|
||||
"os"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// mesh-sdk's own manifest is judged by the controller that runs when it is checked, and that controller reads it
|
||||
// strictly: a manifest word ships one release ahead of its first user. R4e's group is judged by today's controller
|
||||
// (novox/hq ADR 0503 §4), which does not know `paths` (ADR 0499), so mesh-sdk's lib names no paths until a release
|
||||
// after R4e's delivery adds them back (ADR 0499 §6, issue 1758).
|
||||
func TestOwnManifestUsesNoWordTheRunningControllerRefuses(t *testing.T) {
|
||||
raw, err := os.ReadFile("../../module.json")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m, err := ParseManifest(raw)
|
||||
if err != nil {
|
||||
t.Fatalf("mesh-sdk's module.json does not parse: %v", err)
|
||||
}
|
||||
if m.Build == nil || len(m.Build.Artifacts) == 0 {
|
||||
t.Fatal("mesh-sdk's module.json declares no artifact")
|
||||
}
|
||||
for _, a := range m.Build.Artifacts {
|
||||
if a.Paths != nil {
|
||||
t.Errorf("artifact %q names paths %v: the running controller refuses the word until R4e is delivered", a.Name, a.Paths)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,67 @@
|
||||
package catalogue
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// An artifact names the paths it is built from with `paths` (novox/hq ADR 0499 clause 1): a package, or a bundle in a
|
||||
// language other than Go, may; a Go program's artifact, an archive, an image and an upstream artifact may not, and the
|
||||
// refusal names the artifact and its kind. Each path stays inside the module's directory: no `..`, no absolute path,
|
||||
// no pattern.
|
||||
func TestAnArtifactNamesThePathsItIsBuiltFrom(t *testing.T) {
|
||||
m, err := ParseManifest([]byte(`{"module":"mesh-sdk","version":"1","slug":"sdk","build":{"artifacts":[
|
||||
{"name":"lib","kind":"package","language":"typescript",
|
||||
"paths":["src","test","package.json","package-lock.json","tsconfig.json"]}]},"resources":[]}`))
|
||||
if err != nil {
|
||||
t.Fatalf("a package naming its paths was refused: %v", err)
|
||||
}
|
||||
if got := strings.Join(m.Build.Artifacts[0].Paths, ","); got != "src,test,package.json,package-lock.json,tsconfig.json" {
|
||||
t.Fatalf("the paths were not kept as written: %q", got)
|
||||
}
|
||||
if _, err := ParseManifest([]byte(`{"module":"a","version":"1","build":{"artifacts":[
|
||||
{"name":"tools","kind":"bundle","language":"typescript","entrypoints":["index.js"],"loads":["index.js"],"paths":["src","package.json"]}]}}`)); err != nil {
|
||||
t.Fatalf("a TypeScript bundle naming its paths was refused: %v", err)
|
||||
}
|
||||
|
||||
for _, c := range []struct{ artifact, says string }{
|
||||
{`{"name":"x","kind":"bundle","language":"go","system":"linux-amd64","paths":["cmd"]}`, `"x" is a Go program`},
|
||||
{`{"name":"x","kind":"archive","from":"files","paths":["files"]}`, `"x" is a "archive"`},
|
||||
{`{"name":"x","kind":"image","from":"Dockerfile","paths":["src"]}`, `"x" is a "image"`},
|
||||
{`{"name":"x","kind":"upstream","from":"nats:2.10","paths":["src"]}`, `"x" is a "upstream"`},
|
||||
{`{"name":"x","kind":"package","language":"typescript","paths":["../other"]}`, "leaves the module's directory"},
|
||||
{`{"name":"x","kind":"package","language":"typescript","paths":["src/../../x"]}`, "leaves the module's directory"},
|
||||
{`{"name":"x","kind":"package","language":"typescript","paths":["/etc"]}`, "leaves the module's directory"},
|
||||
{`{"name":"x","kind":"package","language":"typescript","paths":["src/*.ts"]}`, "is a pattern"},
|
||||
{`{"name":"x","kind":"package","language":"typescript","paths":["src/[ab]"]}`, "is a pattern"},
|
||||
{`{"name":"x","kind":"package","language":"typescript","paths":[""]}`, "names an empty path"},
|
||||
{`{"name":"x","kind":"package","language":"typescript","paths":["."]}`, "names the module's whole directory"},
|
||||
{`{"name":"x","kind":"package","language":"typescript","paths":[]}`, "names no path"},
|
||||
} {
|
||||
_, err := ParseManifest([]byte(`{"module":"a","version":"1","build":{"artifacts":[` + c.artifact + `]}}`))
|
||||
if err == nil || !strings.Contains(err.Error(), c.says) {
|
||||
t.Errorf("%s: %v, wanted a refusal saying %q", c.artifact, err, c.says)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The paths a module's tree is taken from are the union its artifacts name, cleaned and sorted, and only when every
|
||||
// artifact names some (novox/hq ADR 0499 clause 2); a module with an artifact that names none, or with none at all,
|
||||
// keeps its directory's tree.
|
||||
func TestAModulesNamedPathsAreTheUnionWhenEveryArtifactNamesThem(t *testing.T) {
|
||||
m := Manifest{Module: "a", Build: &Build{Artifacts: []Artifact{
|
||||
{Name: "lib", Kind: ArtifactPackage, Paths: []string{"src/", "package.json"}},
|
||||
{Name: "tools", Kind: ArtifactBundle, Paths: []string{"./src", "tools"}},
|
||||
}}}
|
||||
paths, every := m.NamedPaths()
|
||||
if !every || strings.Join(paths, ",") != "package.json,src,tools" {
|
||||
t.Fatalf("NamedPaths = %v, %v; want [package.json src tools], true", paths, every)
|
||||
}
|
||||
m.Build.Artifacts = append(m.Build.Artifacts, Artifact{Name: "img", Kind: ArtifactImage, From: "Dockerfile"})
|
||||
if paths, every := m.NamedPaths(); every || paths != nil {
|
||||
t.Fatalf("a module with an artifact naming no path gave %v, %v", paths, every)
|
||||
}
|
||||
if paths, every := (Manifest{Module: "b"}).NamedPaths(); every || paths != nil {
|
||||
t.Fatalf("a module that builds nothing gave %v, %v", paths, every)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,41 @@
|
||||
package catalogue
|
||||
|
||||
import "encoding/json"
|
||||
|
||||
// ParseManifestForInputs reads a manifest for the inputs of its build and its fingerprint: the indexer's and the
|
||||
// builder's read (novox/hq issue 1751). It is never a judge of config values. A value falling back to a predefined key
|
||||
// (ADR 0348 §7) is judged against the list the mesh published when this process holds one (PublishedPredefined);
|
||||
// where it holds none, as in the indexer and the builder, each key a value names is taken as the value declares it (its
|
||||
// type, trusted when it is), since the list is read where a manifest is judged: the merge gate from the tree's root,
|
||||
// the controller from what it published. No input of a fingerprint or a build comes from the list. Everything else
|
||||
// ParseManifest refuses is refused here too.
|
||||
func ParseManifestForInputs(raw []byte) (Manifest, error) {
|
||||
list := PublishedPredefined()
|
||||
if list == nil {
|
||||
list = fallbacksOf(raw)
|
||||
}
|
||||
return parseManifest(raw, list)
|
||||
}
|
||||
|
||||
// fallbacksOf is the list of the predefined keys a manifest's config values fall back to, each as the value declares
|
||||
// it. A manifest that is not JSON gives an empty list, and the parser says what is wrong with it.
|
||||
func fallbacksOf(raw []byte) PredefinedList {
|
||||
var m struct {
|
||||
Config map[string]struct {
|
||||
Type string `json:"type"`
|
||||
Description string `json:"description"`
|
||||
Predefined string `json:"predefined"`
|
||||
Trusted bool `json:"trusted"`
|
||||
} `json:"config"`
|
||||
}
|
||||
list := PredefinedList{}
|
||||
if json.Unmarshal(raw, &m) != nil {
|
||||
return list
|
||||
}
|
||||
for _, d := range m.Config {
|
||||
if d.Predefined != "" {
|
||||
list[d.Predefined] = PredefinedKey{Type: d.Type, Description: d.Description, Trusted: d.Trusted}
|
||||
}
|
||||
}
|
||||
return list
|
||||
}
|
||||
@@ -0,0 +1,40 @@
|
||||
package catalogue
|
||||
|
||||
import (
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// The indexer and the builder hold no published list of predefined keys; a manifest whose config values fall back to
|
||||
// one (claude-code, dunst, xterm at mesh-catalog 835bf22b) is still read for its build's and fingerprint's inputs
|
||||
// (issue 1751).
|
||||
func TestParseManifestForInputsReadsAValueFallingBackToAPredefinedKey(t *testing.T) { // issue 1751
|
||||
was := ReplacePublished(nil) // the indexer's and the builder's case: no published list
|
||||
defer ReplacePublished(was)
|
||||
for _, module := range []string{"claude-code", "dunst", "xterm"} {
|
||||
raw, err := os.ReadFile("testdata/catalogue-835bf22b-" + module + ".module.json")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := ParseManifest(raw); err == nil {
|
||||
t.Fatalf("%s: ParseManifest reads it with no list; the test proves nothing", module)
|
||||
}
|
||||
if m, err := ParseManifestForInputs(raw); err != nil || m.Module != module {
|
||||
t.Errorf("%s: %v", module, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseManifestForInputsStillRefusesWhatTheParserRefuses(t *testing.T) { // issue 1751
|
||||
was := ReplacePublished(nil)
|
||||
defer ReplacePublished(was)
|
||||
raw, _ := os.ReadFile("testdata/catalogue-835bf22b-xterm.module.json")
|
||||
bad := strings.Replace(string(raw), `"predefined": "font-size"`, `"predefined": "font-size", "trusted": "yes"`, 1)
|
||||
if bad == string(raw) {
|
||||
t.Fatal("the fixture no longer names font-size; the test proves nothing")
|
||||
}
|
||||
if _, err := ParseManifestForInputs([]byte(bad)); err == nil {
|
||||
t.Error("a manifest the parser refuses was read")
|
||||
}
|
||||
}
|
||||
@@ -423,13 +423,6 @@ func CatalogueProblems(shelf Shelf) []string {
|
||||
"operator's account, which every agent runs as (novox/hq ADR 0259)", module, why))
|
||||
}
|
||||
}
|
||||
// A read of a module's state that module does not keep (novox/hq ADR 0201) — said only where the
|
||||
// owner is on the shelf, as a consumer may be installed before its emitter.
|
||||
var manifests []Manifest
|
||||
for _, module := range shelfOrder(shelf) {
|
||||
manifests = append(manifests, shelf[module])
|
||||
}
|
||||
problems = append(problems, StateReadsNothingDeclares(manifests)...)
|
||||
sort.Strings(problems)
|
||||
return problems
|
||||
}
|
||||
|
||||
@@ -126,46 +126,3 @@ func StateProblems(m Manifest) []string {
|
||||
}
|
||||
return problems
|
||||
}
|
||||
|
||||
// ControllerMirrors are the controller's record mirrors a module may read, each `mesh-controller.<name>`
|
||||
// (novox/hq ADR 0289, design 54 §5, design 57 F3). **The controller keeps them, not its manifest's `state`**:
|
||||
// they are asserted with the controller's own buckets, with options of their own, and written by the
|
||||
// controller's account alone (the writers table); declared as a module's state, they would be asserted a
|
||||
// second way and granted to the module's own principal as well. Named here so a read of one is a read of
|
||||
// kept state. The bus's list is broker.ControllerMirrors; a test keeps the two agreeing.
|
||||
// plans-building is the plans in their build phase, read by the checker to hold checks while a delivery builds (ADR 0483
|
||||
// clause 2, issue 1687). declared-packages is the packages a node's declaration names (ADR 0507, issue 1840/S1c).
|
||||
var ControllerMirrors = []string{"assignments", "declared-config", "predefined", "builds", "rollouts", "checks-running", "plans-building", "declared-packages"}
|
||||
|
||||
// StateReadsNothingDeclares is every read across a catalogue whose owner is present and declares no
|
||||
// such state. An absent owner says nothing — a module may be installed long before the one whose
|
||||
// state it reads, as a consumer may before its emitter (design 32 §1).
|
||||
func StateReadsNothingDeclares(manifests []Manifest) []string {
|
||||
declared := map[string]map[string]bool{}
|
||||
for _, m := range manifests {
|
||||
own := map[string]bool{}
|
||||
for _, s := range m.State {
|
||||
own[s.Name] = true
|
||||
}
|
||||
declared[m.Module] = own
|
||||
}
|
||||
if own, present := declared[ControllerSeatName]; present {
|
||||
for _, name := range ControllerMirrors {
|
||||
own[name] = true
|
||||
}
|
||||
}
|
||||
var problems []string
|
||||
for _, m := range manifests {
|
||||
for _, r := range m.Reads {
|
||||
module, local, err := ReadState(r)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
if own, present := declared[module]; present && !own[local] {
|
||||
problems = append(problems, fmt.Sprintf(
|
||||
"%s reads %q, and %s keeps no state called %q", m.Module, r, module, local))
|
||||
}
|
||||
}
|
||||
}
|
||||
return problems
|
||||
}
|
||||
|
||||
@@ -52,78 +52,29 @@ func TestAStateNameIsLocalAndOneToken(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// A read whose owner is present must name a state that owner keeps; an absent owner says nothing,
|
||||
// because a module may be installed before the one whose state it reads.
|
||||
func TestAReadNamesStateItsOwnerKeeps(t *testing.T) {
|
||||
owner := Manifest{Module: "licence-manager", State: []StateDeclaration{{Name: "bindings"}}}
|
||||
good := Manifest{Module: "claude-code", Reads: []string{"licence-manager.bindings", "absent.anything"}}
|
||||
bad := Manifest{Module: "other", Reads: []string{"licence-manager.tokens"}}
|
||||
if p := StateReadsNothingDeclares([]Manifest{owner, good}); len(p) != 0 {
|
||||
t.Fatalf("a read of declared state was refused: %v", p)
|
||||
}
|
||||
p := StateReadsNothingDeclares([]Manifest{owner, bad})
|
||||
if len(p) != 1 || !strings.Contains(p[0], `licence-manager keeps no state called "tokens"`) {
|
||||
t.Fatalf("a read of state nobody keeps was not named: %v", p)
|
||||
}
|
||||
}
|
||||
|
||||
// **The controller's record mirrors are kept state** (design 54 §5, design 57 F3): a module may read
|
||||
// `mesh-controller.assignments` and `mesh-controller.declared-config` although the controller's manifest
|
||||
// declares no `state`; a read of anything else of the controller's is still named.
|
||||
func TestAReadOfTheControllersMirrorsIsAReadOfKeptState(t *testing.T) {
|
||||
controller := Manifest{Module: ControllerSeatName}
|
||||
reader := Manifest{Module: "mesh-config", Reads: []string{"mesh-controller.assignments", "mesh-controller.declared-config"}}
|
||||
if p := StateReadsNothingDeclares([]Manifest{controller, reader}); len(p) != 0 {
|
||||
t.Fatalf("a read of the controller's mirrors was refused: %v", p)
|
||||
}
|
||||
other := Manifest{Module: "other", Reads: []string{"mesh-controller.calls"}}
|
||||
if p := StateReadsNothingDeclares([]Manifest{controller, other}); len(p) != 1 {
|
||||
t.Fatalf("a read of the controller's own calls was not named: %v", p)
|
||||
}
|
||||
}
|
||||
|
||||
// A module may read `mesh-controller.plans-building` (novox/hq ADR 0483 clause 2, issue 1687): the checker holds a check
|
||||
// while a delivery builds, and the composed check refused its read as state the controller does not keep.
|
||||
func TestAReadOfThePlansBuildingIsAReadOfKeptState(t *testing.T) {
|
||||
controller := Manifest{Module: ControllerSeatName}
|
||||
checker := Manifest{Module: "mesh-delivery-checker", Reads: []string{"mesh-controller.plans-building"}}
|
||||
if p := StateReadsNothingDeclares([]Manifest{controller, checker}); len(p) != 0 {
|
||||
t.Fatalf("a read of the controller's plans-building was refused: %v", p)
|
||||
}
|
||||
}
|
||||
|
||||
// A module may read `mesh-controller.declared-packages` (issue 1840/S1c, ADR 0507): the packages a node's declaration names.
|
||||
func TestAReadOfTheDeclaredPackagesIsAReadOfKeptState(t *testing.T) {
|
||||
controller := Manifest{Module: ControllerSeatName}
|
||||
reader := Manifest{Module: "some-reader", Reads: []string{"mesh-controller.declared-packages"}}
|
||||
if p := StateReadsNothingDeclares([]Manifest{controller, reader}); len(p) != 0 {
|
||||
t.Fatalf("a read of the controller's declared-packages was refused: %v", p)
|
||||
}
|
||||
}
|
||||
|
||||
// **Across the whole catalogue**: every state name is local, and every read whose owner is present
|
||||
// names state that owner keeps.
|
||||
// **Across the whole catalogue**: every state name is local and every read names a module's state.
|
||||
func TestEveryManifestsStateIsLocalAndEveryReadIsKept(t *testing.T) {
|
||||
manifests := theCatalogue(t)
|
||||
var problems []string
|
||||
for _, m := range manifests {
|
||||
problems = append(problems, StateProblems(m)...)
|
||||
}
|
||||
problems = append(problems, StateReadsNothingDeclares(manifests)...)
|
||||
if len(problems) > 0 {
|
||||
t.Fatalf("the catalogue's state is not what ADR 0201 says:\n %s", strings.Join(problems, "\n "))
|
||||
}
|
||||
}
|
||||
|
||||
// `module check` says it too: the cross-catalogue pass names a read nothing on the shelf keeps.
|
||||
func TestTheCataloguePassNamesAReadItsOwnerDoesNotKeep(t *testing.T) {
|
||||
// The catalogue pass no longer judges one module's reads against another's manifest (novox/hq ADR 0495 clauses 1 and 5,
|
||||
// issue 1790): what an owner keeps is what it announces on the bus, and each module's own announcement test holds that.
|
||||
func TestTheCataloguePassJudgesNoReadAgainstTheOwnersManifest(t *testing.T) {
|
||||
shelf := Shelf{
|
||||
"licence-manager": {Module: "licence-manager", State: []StateDeclaration{{Name: "bindings"}}},
|
||||
"claude-code": {Module: "claude-code", Reads: []string{"licence-manager.tokens"}},
|
||||
}
|
||||
problems := CatalogueProblems(shelf)
|
||||
if len(problems) != 1 || !strings.Contains(problems[0], `keeps no state called "tokens"`) {
|
||||
t.Fatalf("the catalogue pass said %v", problems)
|
||||
for _, p := range CatalogueProblems(shelf) {
|
||||
if strings.Contains(p, "keeps no state") {
|
||||
t.Fatalf("the catalogue pass judged a read against the owner's manifest: %s", p)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,249 @@
|
||||
{
|
||||
"module": "claude-code",
|
||||
"version": "1",
|
||||
"moved": true,
|
||||
"slug": "agent",
|
||||
"capabilities": [
|
||||
"package-manager"
|
||||
],
|
||||
"requires": [
|
||||
"mcp-endpoint",
|
||||
{
|
||||
"name": "gitea-read",
|
||||
"optional": true
|
||||
}
|
||||
],
|
||||
"binds": {
|
||||
"mcp-endpoint": "${dir:state}/mcp-endpoint.json"
|
||||
},
|
||||
"secrets": {
|
||||
"gitea-read": "${dir:state}/gitea-read.secret"
|
||||
},
|
||||
"uses": [
|
||||
"operator-channel"
|
||||
],
|
||||
"state": [
|
||||
"servers",
|
||||
"holdings",
|
||||
"config",
|
||||
"proposals",
|
||||
"questions",
|
||||
"memory",
|
||||
"waits"
|
||||
],
|
||||
"reads": [
|
||||
"claude-licence-manager.bindings"
|
||||
],
|
||||
"tools": [
|
||||
"claude_code_status",
|
||||
"claude_code_render",
|
||||
"claude_code_guard",
|
||||
"claude_code_pull",
|
||||
"claude_code_grant",
|
||||
"claude_code_add_api_key",
|
||||
"claude_code_registrations",
|
||||
"claude_code_memory",
|
||||
"claude_code_mcp_list",
|
||||
"claude_code_mcp_register",
|
||||
"claude_code_mcp_unregister",
|
||||
"claude_code_skill_list",
|
||||
"claude_code_skill_register",
|
||||
"claude_code_skill_unregister",
|
||||
"claude_code_agent_list",
|
||||
"claude_code_agent_register",
|
||||
"claude_code_agent_unregister",
|
||||
"claude_code_command_list",
|
||||
"claude_code_command_register",
|
||||
"claude_code_command_unregister",
|
||||
"claude_code_hook_list",
|
||||
"claude_code_hook_register",
|
||||
"claude_code_hook_unregister",
|
||||
"claude_code_output_style_list",
|
||||
"claude_code_output_style_register",
|
||||
"claude_code_output_style_unregister",
|
||||
"claude_code_instructions_list",
|
||||
"claude_code_instructions_register",
|
||||
"claude_code_instructions_unregister",
|
||||
"claude_code_instructions_propose",
|
||||
"claude_code_proposals",
|
||||
"claude_code_ask",
|
||||
"claude_code_answer",
|
||||
"claude_code_wait",
|
||||
"claude_code_settings_get",
|
||||
"claude_code_settings_set",
|
||||
"claude_code_settings_clear",
|
||||
"claude_code_permission_add",
|
||||
"claude_code_permission_remove",
|
||||
"claude_code_config_list",
|
||||
"claude_code_config_show",
|
||||
"claude_code_config_status",
|
||||
"claude_code_config_import",
|
||||
"claude_code_home_show",
|
||||
"claude_code_home_remove",
|
||||
"claude_code_home_removed",
|
||||
"claude_code_home_restore",
|
||||
"claude_code_judge",
|
||||
"claude_code_judged"
|
||||
],
|
||||
"data": {
|
||||
"own": [
|
||||
{
|
||||
"id": "agent-home",
|
||||
"path": "${dir:agent-home}",
|
||||
"class": "valuable",
|
||||
"why": "the operator's agent's own files: its memory, history and projects"
|
||||
}
|
||||
]
|
||||
},
|
||||
"config": {
|
||||
"role": {
|
||||
"type": "string",
|
||||
"description": "What this machine is for, in a sentence, written into the instructions every Claude Code session on it reads; empty says nothing.",
|
||||
"trusted": true,
|
||||
"default": ""
|
||||
},
|
||||
"mcp_servers": {
|
||||
"type": "object",
|
||||
"description": "The tool servers every session on this machine is given beside the mesh's own, by name, each as Claude Code's managed MCP configuration takes it; empty adds none.",
|
||||
"trusted": true,
|
||||
"default": {}
|
||||
},
|
||||
"managed_settings": {
|
||||
"type": "object",
|
||||
"description": "Keys of Claude Code's managed settings every session on this machine obeys, such as spinnerTipsEnabled, laid over the mesh's own; empty adds none. Falls back to agent-managed-settings, the mesh's or this node's.",
|
||||
"trusted": true,
|
||||
"predefined": "agent-managed-settings",
|
||||
"default": {}
|
||||
},
|
||||
"instructions": {
|
||||
"type": "object",
|
||||
"description": "The instructions every session on this machine reads, by name, each a text as one line or a list of its lines; empty adds none.",
|
||||
"trusted": true,
|
||||
"default": {}
|
||||
},
|
||||
"output_styles": {
|
||||
"type": "object",
|
||||
"description": "The output styles offered to every session on this machine, by name, each a text as one line or a list of its lines; empty adds none.",
|
||||
"trusted": true,
|
||||
"default": {}
|
||||
},
|
||||
"skills": {
|
||||
"type": "object",
|
||||
"description": "The skills offered to every session on this machine, by name, each its files by path, a text as one line or a list of its lines; a skill without its SKILL.md is left out; empty adds none.",
|
||||
"trusted": true,
|
||||
"default": {}
|
||||
},
|
||||
"commands": {
|
||||
"type": "object",
|
||||
"description": "The slash commands offered to every session on this machine, by name, each a text as one line or a list of its lines; empty adds none.",
|
||||
"trusted": true,
|
||||
"default": {}
|
||||
},
|
||||
"agents": {
|
||||
"type": "object",
|
||||
"description": "The agents offered to every session on this machine, by name, each a text as one line or a list of its lines; empty adds none.",
|
||||
"trusted": true,
|
||||
"default": {}
|
||||
}
|
||||
},
|
||||
"resources": [
|
||||
{
|
||||
"id": "package",
|
||||
"type": "package",
|
||||
"package": "claude-code"
|
||||
},
|
||||
{
|
||||
"id": "managed",
|
||||
"type": "directory",
|
||||
"path": "/etc/claude-code",
|
||||
"mode": "0755",
|
||||
"unknown": "program"
|
||||
},
|
||||
{
|
||||
"id": "start",
|
||||
"type": "file",
|
||||
"path": "/usr/local/bin/claude-agent",
|
||||
"mode": "0755",
|
||||
"template": "files/claude-agent"
|
||||
},
|
||||
{
|
||||
"id": "agent-account-name",
|
||||
"type": "file",
|
||||
"path": "/etc/claude-code/agent-account",
|
||||
"mode": "0644",
|
||||
"template": "files/agent-account"
|
||||
},
|
||||
{
|
||||
"id": "agent-account",
|
||||
"type": "user",
|
||||
"name": "${machine:agent-account}",
|
||||
"home": "${machine:agent-home}",
|
||||
"root": "${machine:agent-root}"
|
||||
},
|
||||
{
|
||||
"id": "agent-home",
|
||||
"type": "directory",
|
||||
"path": "${machine:agent-home}/.claude",
|
||||
"mode": "0700",
|
||||
"owner": "${machine:agent-account}",
|
||||
"unknown": "module"
|
||||
},
|
||||
{
|
||||
"id": "state",
|
||||
"type": "directory",
|
||||
"mode": "0700",
|
||||
"owner": "${machine:account}",
|
||||
"place": ".",
|
||||
"unknown": "program"
|
||||
},
|
||||
{
|
||||
"id": "facts",
|
||||
"type": "file",
|
||||
"path": "${dir:state}/facts.json",
|
||||
"mode": "0600",
|
||||
"owner": "${machine:account}",
|
||||
"template": "files/facts.json"
|
||||
},
|
||||
{
|
||||
"id": "settings",
|
||||
"type": "file",
|
||||
"path": "${dir:state}/settings.json",
|
||||
"mode": "0600",
|
||||
"owner": "${machine:account}",
|
||||
"takes-config": ["role", "mcp_servers", "managed_settings", "instructions", "output_styles", "skills", "commands", "agents"],
|
||||
"merge": "json",
|
||||
"template": "files/settings.json"
|
||||
}
|
||||
],
|
||||
"items": [
|
||||
{
|
||||
"collection": "shell:zsh:normal",
|
||||
"template": "files/zsh-normal"
|
||||
}
|
||||
],
|
||||
"build": {
|
||||
"artifacts": [
|
||||
{
|
||||
"name": "tools",
|
||||
"kind": "bundle",
|
||||
"language": "go",
|
||||
"system": "arch",
|
||||
"from": "cmd/claude-code",
|
||||
"binary": "claude-code",
|
||||
"loads": [
|
||||
"claude-code"
|
||||
],
|
||||
"env": {
|
||||
"MESH_CLAUDE_CODE_STATE": "${dir:state}",
|
||||
"MESH_CLAUDE_CODE_FACTS": "${dir:state}/facts.json",
|
||||
"MESH_CLAUDE_CODE_SETTINGS": "${dir:state}/settings.json"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "files",
|
||||
"kind": "archive",
|
||||
"from": "files"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,123 @@
|
||||
{
|
||||
"module": "dunst",
|
||||
"version": "1",
|
||||
"moved": true,
|
||||
"capabilities": [
|
||||
"package-manager"
|
||||
],
|
||||
"claims": [
|
||||
{
|
||||
"name": "node-notifier",
|
||||
"scope": "node",
|
||||
"serves": [
|
||||
"send",
|
||||
"history"
|
||||
]
|
||||
}
|
||||
],
|
||||
"tools": [
|
||||
"dunst_pause",
|
||||
"dunst_resume",
|
||||
"dunst_close_all",
|
||||
"dunst_rules",
|
||||
"dunst_count",
|
||||
"dunst_reload"
|
||||
],
|
||||
"config": {
|
||||
"font-size": {
|
||||
"type": "number",
|
||||
"description": "The size of a notification's text, in points of Inter: 10 reads well on a screen of about 100 DPI, and a denser screen needs a larger size.",
|
||||
"minimum": 6,
|
||||
"maximum": 72,
|
||||
"predefined": "font-size",
|
||||
"default": 10
|
||||
},
|
||||
"width": {
|
||||
"type": "number",
|
||||
"description": "The width of a notification, in pixels: 250 fits a title of about forty characters at 10 points, and a larger font needs a wider notification.",
|
||||
"minimum": 50,
|
||||
"maximum": 4000,
|
||||
"default": 250
|
||||
}
|
||||
},
|
||||
"resources": [
|
||||
{
|
||||
"id": "package",
|
||||
"type": "package",
|
||||
"package": "dunst"
|
||||
},
|
||||
{
|
||||
"id": "client",
|
||||
"type": "package",
|
||||
"package": "libnotify"
|
||||
},
|
||||
{
|
||||
"id": "configuration-dir",
|
||||
"type": "directory",
|
||||
"path": "${machine:account-home}/.config/dunst",
|
||||
"owner": "${machine:account}",
|
||||
"mode": "0755",
|
||||
"unknown": "move-out"
|
||||
},
|
||||
{
|
||||
"id": "dropins",
|
||||
"type": "directory",
|
||||
"path": "${machine:account-home}/.config/dunst/dunstrc.d",
|
||||
"owner": "${machine:account}",
|
||||
"mode": "0755",
|
||||
"unknown": "move-out",
|
||||
"knows": ["50-slack.conf"]
|
||||
},
|
||||
{
|
||||
"id": "configuration",
|
||||
"trusted": false,
|
||||
"type": "file",
|
||||
"path": "${machine:account-home}/.config/dunst/dunstrc",
|
||||
"owner": "${machine:account}",
|
||||
"mode": "0644",
|
||||
"template": "files/dunstrc"
|
||||
},
|
||||
{
|
||||
"id": "notifier",
|
||||
"type": "service",
|
||||
"unit": "dunst.service",
|
||||
"scope": "user",
|
||||
"user": "${machine:account}",
|
||||
"reload-on": [
|
||||
"configuration"
|
||||
]
|
||||
}
|
||||
],
|
||||
"build": {
|
||||
"artifacts": [
|
||||
{
|
||||
"name": "tools",
|
||||
"kind": "bundle",
|
||||
"language": "go",
|
||||
"system": "arch",
|
||||
"from": "cmd/dunst-tools",
|
||||
"binary": "dunst-tools",
|
||||
"loads": [
|
||||
"dunst-tools"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "files",
|
||||
"kind": "archive",
|
||||
"from": "files"
|
||||
}
|
||||
]
|
||||
},
|
||||
"emits": [
|
||||
"action-chosen",
|
||||
"started",
|
||||
"paused"
|
||||
],
|
||||
"items": [
|
||||
{
|
||||
"collection": "contribution:node-display-session:config",
|
||||
"name": "display-session",
|
||||
"template": "files/i3/50-dunst.conf"
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,236 @@
|
||||
{
|
||||
"module": "xterm",
|
||||
"version": "1",
|
||||
"moved": true,
|
||||
"capabilities": [
|
||||
"package-manager"
|
||||
],
|
||||
"requires": [
|
||||
"x11-display"
|
||||
],
|
||||
"claims": [
|
||||
{
|
||||
"name": "node-terminal-emulator",
|
||||
"scope": "node",
|
||||
"serves": [
|
||||
"open"
|
||||
]
|
||||
}
|
||||
],
|
||||
"tools": [
|
||||
"xterm_font",
|
||||
"xterm_colours"
|
||||
],
|
||||
"config": {
|
||||
"face": {
|
||||
"type": "string",
|
||||
"description": "The font family xterm draws with, as fc-list names it: JetBrainsMono Nerd Font, because research 026/04 chose it for its glyphs the prompt uses. A family the machine does not have falls back to the nearest installed face (xterm_font says which).",
|
||||
"pattern": "[A-Za-z0-9 ._-]{1,64}",
|
||||
"default": "JetBrainsMono Nerd Font"
|
||||
},
|
||||
"size": {
|
||||
"type": "number",
|
||||
"description": "The size of xterm's text, in points: 12 reads well on a screen of about 100 DPI, and a denser screen needs a larger size.",
|
||||
"minimum": 6,
|
||||
"maximum": 72,
|
||||
"predefined": "font-size",
|
||||
"default": 12
|
||||
},
|
||||
"save-lines": {
|
||||
"type": "number",
|
||||
"description": "How many lines of scrollback xterm keeps per terminal: 10000 holds a long build log, and each line costs memory in every open terminal.",
|
||||
"minimum": 0,
|
||||
"maximum": 1000000,
|
||||
"default": 10000
|
||||
},
|
||||
"term-name": {
|
||||
"type": "string",
|
||||
"description": "The TERM value xterm sets for the programs it runs: xterm-256color, because the palette has 256 colours and every program knows the name.",
|
||||
"pattern": "[A-Za-z0-9._-]{1,64}",
|
||||
"default": "xterm-256color"
|
||||
},
|
||||
"render-font": {
|
||||
"type": "boolean",
|
||||
"description": "Whether xterm draws with fontconfig faces (true) or the old X core fonts (false): true, because the face is a fontconfig family and core fonts cannot show it.",
|
||||
"default": true
|
||||
},
|
||||
"bold-mode": {
|
||||
"type": "boolean",
|
||||
"description": "Whether xterm draws bold text by overstriking the normal face (true): false, because the face has a real bold style and overstriking blurs it.",
|
||||
"default": false
|
||||
},
|
||||
"always-bold-mode": {
|
||||
"type": "boolean",
|
||||
"description": "Whether xterm overstrikes bold even where the face has a bold style (true): false, for the same reason as bold-mode.",
|
||||
"default": false
|
||||
},
|
||||
"allow-bold-fonts": {
|
||||
"type": "boolean",
|
||||
"description": "Whether xterm may use the face's bold style for bold text (true): true, so bold text is drawn with the real bold weight.",
|
||||
"default": true
|
||||
},
|
||||
"background": {
|
||||
"type": "string",
|
||||
"description": "The terminal's background colour, as #rrggbb: black, as both workstations ran it.",
|
||||
"pattern": "#[0-9a-fA-F]{6}",
|
||||
"default": "#000000"
|
||||
},
|
||||
"foreground": {
|
||||
"type": "string",
|
||||
"description": "The terminal's text colour, as #rrggbb: a light grey, easier on the eyes than white on black.",
|
||||
"pattern": "#[0-9a-fA-F]{6}",
|
||||
"default": "#a8a8a8"
|
||||
},
|
||||
"color0": {
|
||||
"type": "string",
|
||||
"description": "Palette entry 0, the normal black, as #rrggbb: #000000, as both workstations ran it.",
|
||||
"pattern": "#[0-9a-fA-F]{6}",
|
||||
"default": "#000000"
|
||||
},
|
||||
"color1": {
|
||||
"type": "string",
|
||||
"description": "Palette entry 1, the normal red, as #rrggbb: #9e1828, as both workstations ran it.",
|
||||
"pattern": "#[0-9a-fA-F]{6}",
|
||||
"default": "#9e1828"
|
||||
},
|
||||
"color2": {
|
||||
"type": "string",
|
||||
"description": "Palette entry 2, the normal green, as #rrggbb: #008800, as both workstations ran it.",
|
||||
"pattern": "#[0-9a-fA-F]{6}",
|
||||
"default": "#008800"
|
||||
},
|
||||
"color3": {
|
||||
"type": "string",
|
||||
"description": "Palette entry 3, the normal yellow, as #rrggbb: #d2bb4b, as both workstations ran it.",
|
||||
"pattern": "#[0-9a-fA-F]{6}",
|
||||
"default": "#d2bb4b"
|
||||
},
|
||||
"color4": {
|
||||
"type": "string",
|
||||
"description": "Palette entry 4, the normal blue, as #rrggbb: #414171, as both workstations ran it.",
|
||||
"pattern": "#[0-9a-fA-F]{6}",
|
||||
"default": "#414171"
|
||||
},
|
||||
"color5": {
|
||||
"type": "string",
|
||||
"description": "Palette entry 5, the normal magenta, as #rrggbb: #963c59, as both workstations ran it.",
|
||||
"pattern": "#[0-9a-fA-F]{6}",
|
||||
"default": "#963c59"
|
||||
},
|
||||
"color6": {
|
||||
"type": "string",
|
||||
"description": "Palette entry 6, the normal cyan, as #rrggbb: #418179, as both workstations ran it.",
|
||||
"pattern": "#[0-9a-fA-F]{6}",
|
||||
"default": "#418179"
|
||||
},
|
||||
"color7": {
|
||||
"type": "string",
|
||||
"description": "Palette entry 7, the normal white, as #rrggbb: #bebebe, as both workstations ran it.",
|
||||
"pattern": "#[0-9a-fA-F]{6}",
|
||||
"default": "#bebebe"
|
||||
},
|
||||
"color8": {
|
||||
"type": "string",
|
||||
"description": "Palette entry 8, the bright black, as #rrggbb: #666666, as both workstations ran it.",
|
||||
"pattern": "#[0-9a-fA-F]{6}",
|
||||
"default": "#666666"
|
||||
},
|
||||
"color9": {
|
||||
"type": "string",
|
||||
"description": "Palette entry 9, the bright red, as #rrggbb: #bc5766, as both workstations ran it.",
|
||||
"pattern": "#[0-9a-fA-F]{6}",
|
||||
"default": "#bc5766"
|
||||
},
|
||||
"color10": {
|
||||
"type": "string",
|
||||
"description": "Palette entry 10, the bright green, as #rrggbb: #61a171, as both workstations ran it.",
|
||||
"pattern": "#[0-9a-fA-F]{6}",
|
||||
"default": "#61a171"
|
||||
},
|
||||
"color11": {
|
||||
"type": "string",
|
||||
"description": "Palette entry 11, the bright yellow, as #rrggbb: #e7db52, as both workstations ran it.",
|
||||
"pattern": "#[0-9a-fA-F]{6}",
|
||||
"default": "#e7db52"
|
||||
},
|
||||
"color12": {
|
||||
"type": "string",
|
||||
"description": "Palette entry 12, the bright blue, as #rrggbb: #5085af, as both workstations ran it.",
|
||||
"pattern": "#[0-9a-fA-F]{6}",
|
||||
"default": "#5085af"
|
||||
},
|
||||
"color13": {
|
||||
"type": "string",
|
||||
"description": "Palette entry 13, the bright magenta, as #rrggbb: #a97a99, as both workstations ran it.",
|
||||
"pattern": "#[0-9a-fA-F]{6}",
|
||||
"default": "#a97a99"
|
||||
},
|
||||
"color14": {
|
||||
"type": "string",
|
||||
"description": "Palette entry 14, the bright cyan, as #rrggbb: #6ba4a4, as both workstations ran it.",
|
||||
"pattern": "#[0-9a-fA-F]{6}",
|
||||
"default": "#6ba4a4"
|
||||
},
|
||||
"color15": {
|
||||
"type": "string",
|
||||
"description": "Palette entry 15, the bright white, as #rrggbb: #ffffff, as both workstations ran it.",
|
||||
"pattern": "#[0-9a-fA-F]{6}",
|
||||
"default": "#ffffff"
|
||||
},
|
||||
"copy-binding": {
|
||||
"type": "string",
|
||||
"description": "The key combination that copies the selection to the clipboard, in X translation syntax: Ctrl Shift ~Alt <Key> C, so a plain Ctrl+C still interrupts the program. It holds no colon, backslash or line break, which would end the resource.",
|
||||
"pattern": "[^:\\\\\\n]+",
|
||||
"default": "Ctrl Shift ~Alt <Key> C"
|
||||
},
|
||||
"paste-binding": {
|
||||
"type": "string",
|
||||
"description": "The key combination that pastes the clipboard, in X translation syntax: Ctrl Shift ~Alt <Key> V, the pair of copy-binding. It holds no colon, backslash or line break, which would end the resource.",
|
||||
"pattern": "[^:\\\\\\n]+",
|
||||
"default": "Ctrl Shift ~Alt <Key> V"
|
||||
}
|
||||
},
|
||||
"items": [
|
||||
{
|
||||
"collection": "environment",
|
||||
"name": "terminal",
|
||||
"attributes": {
|
||||
"kind": "variable",
|
||||
"name": "TERMINAL"
|
||||
},
|
||||
"template": "files/env-terminal"
|
||||
},
|
||||
{
|
||||
"collection": "shell:xresources:normal",
|
||||
"name": "xresources-normal",
|
||||
"template": "files/xresources-normal"
|
||||
}
|
||||
],
|
||||
"resources": [
|
||||
{
|
||||
"id": "package",
|
||||
"type": "package",
|
||||
"package": "xterm"
|
||||
}
|
||||
],
|
||||
"build": {
|
||||
"artifacts": [
|
||||
{
|
||||
"name": "tools",
|
||||
"kind": "bundle",
|
||||
"language": "go",
|
||||
"system": "arch",
|
||||
"from": "cmd/xterm-tools",
|
||||
"binary": "xterm-tools",
|
||||
"loads": [
|
||||
"xterm-tools"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "files",
|
||||
"kind": "archive",
|
||||
"from": "files"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
@@ -10,7 +10,8 @@ import (
|
||||
// the provision, and does not find out at its first build. The manifests are the catalogue's, as they are.
|
||||
|
||||
// builderNode is a machine that can run a build module: the package manager and user namespaces it asks of
|
||||
// the machine, and the forge account (mesh-scope, provided by the forge) stood in for.
|
||||
// the machine, and the forge account (mesh-scope, provided by the forge) and the artifact store the build
|
||||
// modules publish to (ADR 0513, provided by distribution) stood in for.
|
||||
func builderNode() Node {
|
||||
return Node{Name: "builder", Site: "house",
|
||||
Capabilities: map[string]bool{"package-manager": true, "user-namespaces": true}}
|
||||
@@ -19,7 +20,7 @@ func builderNode() Node {
|
||||
func toolchainShelf(t *testing.T) map[string]Manifest {
|
||||
t.Helper()
|
||||
m := map[string]Manifest{
|
||||
"forge": {Module: "forge", Version: "1", Provides: []Offer{{Name: "forge-account"}, {Name: "npm-package-registry"}}},
|
||||
"forge": {Module: "forge", Version: "1", Provides: []Offer{{Name: "forge-account"}, {Name: "npm-package-registry"}, {Name: "artifact-store"}}},
|
||||
}
|
||||
for _, name := range []string{"go", "nodejs", "mesh-build-go", "mesh-build-node", "dev-tools"} {
|
||||
m[name] = catalogueManifest(t, name)
|
||||
|
||||
+22
-56
@@ -147,24 +147,16 @@ var ControllerVerbs = []Verb{
|
||||
}, nil, "data")},
|
||||
// The verbs the delivery's owner asks with (novox/hq ADR 0239): the planner's answer, a group's order and
|
||||
// composed check, and the start and end of a walk. Callable by a person too; mesh-delivery is who needs them.
|
||||
{Name: "delivery-plan", Description: "The delivery plan of a diffset, as the planner computes it and keeping " +
|
||||
"nothing (novox/hq ADR 0239): what it moves and builds after them in tiers, what each machine receives and " +
|
||||
"what waits there, the steps that are not an ordinary send, and whether the gate would run.",
|
||||
Input: schema(map[string]string{"repository": "owner/repository", "head": "the commit at hand",
|
||||
"base": "the branch it merges into (default main)",
|
||||
"paths": "the files it changes, comma-separated, from the repository's root",
|
||||
"module-dirs": "the directories holding a module.json at the head, comma-separated, as the forge says them",
|
||||
"removed": "the files among paths it deletes, comma-separated"}, []string{"repository", "paths"})},
|
||||
{Name: "delivery-plan", Description: "Removed in R4e (ADR 0501 §5 / ADR 0503 §3): refused by name, naming its replacement",
|
||||
Input: schema(nil, nil)},
|
||||
{Name: "delivery-order", Description: "A delivery group's order (novox/hq ADR 0419): its members in the order " +
|
||||
"they are delivered, every pair and why — built by, verb before caller, version skew, engine before " +
|
||||
"controller — read from the module graph and the manifests' change alone; no `after:` line is read. The " +
|
||||
"cycle, with every module and edge of it, is the one refusal.",
|
||||
Input: schema(map[string]string{"members": "the members, as JSON: [{id, repository, base, head, number, paths, " +
|
||||
"module_dirs, module_dirs_said, removed}]"}, []string{"members"})},
|
||||
{Name: "delivery-check", Description: "Ask the build seat for a delivery group's composed check (novox/hq ADR " +
|
||||
"0239): every member's head laid over the mesh in turn and judged as one future state. Answers the ask's id; " +
|
||||
"the verdict is said as `checked` with the group's id. With one member and no group, that head is checked " +
|
||||
"again as its pull request is.",
|
||||
{Name: "delivery-check", Description: "Answers that the checker (mesh-delivery-checker) judges a group (novox/hq " +
|
||||
"ADR 0503 §3); asks nothing. Its inputs stay, since hq's repository check still names it (ADR 0513 §4).",
|
||||
Input: schema(map[string]string{"group": "the group's id (none for one head checked again)",
|
||||
"members": "the members, as JSON, as delivery-order takes them",
|
||||
"ready": "\"true\" when the pull request is marked ready: its check is queued before every other check (novox/hq ADR 0406 §3)"},
|
||||
@@ -378,45 +370,33 @@ var ControllerVerbs = []Verb{
|
||||
{Name: "clear", Description: "Cancel every waiting ask in the build queue — and with dead, every dead one too — each " +
|
||||
"recorded failed, cancelled by hand. Never touches one in flight.",
|
||||
Input: schema(map[string]string{"dead": "\"true\" to cancel the dead asks as well"}, nil, "dead")},
|
||||
{Name: "rebuild", Description: "Ask a module's current source again under a new id — the branch it follows — or, given a " +
|
||||
"build's id, that build's repository, path and ref. A module a plan holds unbuilt or failed joins that plan. Answers the new id.",
|
||||
Input: schema(map[string]string{"what": "a module's name, or a build's id"}, []string{"what"})},
|
||||
{Name: "replay", Description: "Ask a recorded build's repository and path again at the commit it built, under a new id. " +
|
||||
"A dry run unless register: nothing recorded or registered. Registering is refused when a newer build of the module " +
|
||||
"is registered — it would roll the older commit out (novox/hq issue 207) — unless older says so.",
|
||||
Input: schema(map[string]string{
|
||||
"id": "the build's id",
|
||||
"register": "\"true\" to register what it builds",
|
||||
"older": "\"true\", with register: even though a newer build of the module is registered",
|
||||
}, []string{"id"}, "register", "older")},
|
||||
{Name: "rebuild", Description: "Removed in R4e (ADR 0501 §5 / ADR 0503 §3): refused by name, naming its replacement",
|
||||
Input: schema(nil, nil)},
|
||||
{Name: "replay", Description: "Ask a recorded build's repository and path again at the commit it built, under a new id: " +
|
||||
"a dry run only, nothing recorded or registered (novox/hq ADR 0429 §18: a fingerprint's key is built again by the " +
|
||||
"builder, never registered by hand).",
|
||||
Input: schema(map[string]string{"id": "the build's id"}, []string{"id"})},
|
||||
// novox/hq issue 686: a builder ran a branch's store tests for the first time in its pull request's merge check.
|
||||
{Name: "try", Description: "Try a pushed branch or commit before its pull request (novox/hq issue 686): the build " +
|
||||
"seat merges it into the repository's trunk and runs the repository's own merge-check.sh on the merge, in the " +
|
||||
"toolchain containers and against the throwaway store and bus a pull request's merge check gets. Answers at " +
|
||||
{Name: "try", Description: "Try a pushed branch or commit before its pull request (novox/hq issue 686): the " +
|
||||
"checker (mesh-delivery-checker) merges the head onto the repository's trunk and runs its levels on the merge, " +
|
||||
"as it judges a pull request (ADR 0503 §3); no merge-check.sh is run and build-agent is not asked. Answers at " +
|
||||
"once with an id; asked again with that id it waits up to 20 s and answers pass, fail or error, how long it " +
|
||||
"took and the lines naming what failed — or that it is still waiting or running, to ask again. Sets no " +
|
||||
"status on the forge, registers and builds nothing, runs no merge gate, counts as a check in the build queue " +
|
||||
"(after deliveries and after every pull request's check, however long it waits) and is bounded as one. One " +
|
||||
"try of a commit at a time — a second ask answers the first's id — and at most four waiting or running in " +
|
||||
"all: a fifth is refused. Only for a repository of the mesh; the ref must be on the forge.",
|
||||
"status on the forge, registers and builds nothing for delivery. One try of a commit at a time — a second " +
|
||||
"ask answers the first's id. Only for a repository of the mesh; the ref must be on the forge.",
|
||||
Input: schema(map[string]string{
|
||||
"repository": "the repository on the forge, owner/name; with ref",
|
||||
"ref": "the branch or commit to try, as pushed to the forge",
|
||||
"base": "the branch it is merged into (the repository's default branch when none is named). The " +
|
||||
"merge-check.sh that runs is that branch's, never the ref's own: the same trust as a pull request's base",
|
||||
"base": "the branch it is merged into (the repository's default branch when none is named)",
|
||||
"tests": "only the tests this `go test -run` pattern names, where the repository's merge-check.sh reads " +
|
||||
"MESH_CHECK_TESTS; a script that does not runs whole",
|
||||
"store-only": "\"true\": only the repository's store tests, where its merge-check.sh reads " +
|
||||
"MESH_CHECK_STORE_ONLY",
|
||||
"id": "instead of a repository: the id a try answered, to read what came of it",
|
||||
"store-only": "\"true\": only the integration level",
|
||||
"id": "instead of a repository: the id a try answered, to read what came of it",
|
||||
}, nil, "store-only"),
|
||||
Replaces: []string{"sh merge-check.sh", "docker run postgres"}},
|
||||
// novox/hq ADR 0350 §5: asked by the delivery's owner when the forge says a pull request closed unmerged.
|
||||
{Name: "forget-checks", Description: "Let go of every build a pull request's merge checks made for its merge in " +
|
||||
"the artifact store's scratch namespace (novox/hq ADR 0350 §5): asked when it closed unmerged; a merged one's go " +
|
||||
"when the walk answering its merge ends, and any check build goes after a week.",
|
||||
Input: schema(map[string]string{"owner": "the repository's owner", "repo": "the repository",
|
||||
"number": "the pull request's number"}, []string{"owner", "repo", "number"})},
|
||||
{Name: "forget-checks", Description: "Removed in R4e (ADR 0501 §5 / ADR 0503 §3): refused by name, naming its replacement",
|
||||
Input: schema(nil, nil)},
|
||||
{Name: "kill", Description: "End a build where it runs: the machine that took it stops its commands and containers and " +
|
||||
"announces it failed, killed by hand — settled, never handed to another machine.",
|
||||
Input: schema(map[string]string{"id": "the build's id"}, []string{"id"})},
|
||||
@@ -595,22 +575,8 @@ var ControllerVerbs = []Verb{
|
||||
"machine": "one machine (optional)",
|
||||
"retired": "\"true\": only what is retired",
|
||||
}, nil, "retired")},
|
||||
// Three shapes, as the command has them: one repository, every module standing on a base (on), or
|
||||
// every module behind its source (behind). One per call; a second is refused as passed over.
|
||||
{Name: "build", Description: "Have the build machine build a repository — or, instead, every module that " +
|
||||
"stands on a base (on), or every module behind its source (behind). Answers at once with each build's id: " +
|
||||
"`builds` with an id follows it line by line, and a module is registered when its outcome comes.",
|
||||
Input: schema(map[string]string{
|
||||
"repository": "the repository's URL, or its path on the forge holding the git seat (owner/name); " +
|
||||
"needed unless on or behind is given",
|
||||
"path": "with repository: the module's directory inside it (optional)",
|
||||
"ref": "with repository: the branch, tag or commit to build (optional)",
|
||||
"on": "instead of a repository: a module's name. Every module the mesh holds that was built against " +
|
||||
"its artifacts is built again from its own source, bases among them asked first — the rebuild a " +
|
||||
"changed base needs, which behind does not see because their sources did not move",
|
||||
"behind": "\"true\", instead of a repository: build every module the mesh holds whose source has moved " +
|
||||
"past the commit it was built from, bases asked first",
|
||||
}, nil, "behind")},
|
||||
{Name: "build", Description: "Removed in R4e (ADR 0501 §5 / ADR 0503 §3): refused by name, naming its replacement",
|
||||
Input: schema(nil, nil)},
|
||||
// What the records say the registries may keep, asked on demand (novox/hq ADR 0251, to-be 51).
|
||||
{Name: "artifacts", Description: "Every artifact the mesh recorded making that lives in its artifact store, " +
|
||||
"each with its repository, digest, kind (image or archive) and state: kept — and why: a definition names " +
|
||||
|
||||
+109
-8
@@ -63,8 +63,18 @@ func ParseBuilderBuildKey(key string) (module, fingerprint string, err error) {
|
||||
return module, fingerprint, nil
|
||||
}
|
||||
|
||||
// BuiltArtifact is one artifact of a done build: its name in the manifest, its kind, its content digest and, for an
|
||||
// image, the reference it was pushed to.
|
||||
// BuiltArtifact is one artifact of a done build: its name in the manifest, its kind, its content digest (`sha256:` and
|
||||
// 64 hex digits) and the reference that names where it is (novox/hq ADR 0498 clauses 1, 5 and 6; the lead's decision
|
||||
// L1 of R4e), by kind:
|
||||
//
|
||||
// bundle, archive artifact-store://<module>/<artifact>/blobs/<digest> the blob pushed to the artifact store
|
||||
// image, upstream <registry>/<module>/<artifact>@<digest> the image registry's reference, which
|
||||
// artifact-store://<module>/<artifact>@<digest> catalogue.Recorded keeps without its address
|
||||
// package <name>@<version> the version published from the trunk
|
||||
//
|
||||
// <artifact> is Name and <digest> is Digest. A reference is empty for a candidate's image or package, which is never
|
||||
// pushed or published (ADR 0498 clause 6), and in a key done before R4e; the controller delivers only from a key whose
|
||||
// every artifact names one (Referenced, clause 8). A kind left empty (a key written before kinds) is not checked.
|
||||
type BuiltArtifact struct {
|
||||
Name string `json:"name"`
|
||||
Kind string `json:"kind,omitempty"`
|
||||
@@ -72,6 +82,83 @@ type BuiltArtifact struct {
|
||||
Reference string `json:"reference,omitempty"`
|
||||
}
|
||||
|
||||
const artifactStore = "artifact-store://"
|
||||
|
||||
// referenceProblem says what is wrong with a's reference for its kind, or "".
|
||||
func (a BuiltArtifact) referenceProblem() string {
|
||||
r := a.Reference
|
||||
if r == "" || a.Kind == "" {
|
||||
return ""
|
||||
}
|
||||
switch a.Kind {
|
||||
case "bundle", "archive":
|
||||
path, ok := strings.CutPrefix(r, artifactStore)
|
||||
module, rest, _ := strings.Cut(path, "/")
|
||||
if !ok || module == "" || rest != a.Name+"/blobs/"+a.Digest {
|
||||
return "is not artifact-store://<module>/" + a.Name + "/blobs/" + a.Digest
|
||||
}
|
||||
case "image", "upstream":
|
||||
path, digest, ok := strings.Cut(r, "@")
|
||||
if !ok || digest != a.Digest {
|
||||
return "does not name its digest " + a.Digest
|
||||
}
|
||||
var host, module, name string
|
||||
if rest, kept := strings.CutPrefix(path, artifactStore); kept {
|
||||
module, name, _ = strings.Cut(rest, "/")
|
||||
host = "kept"
|
||||
} else {
|
||||
parts := strings.Split(path, "/")
|
||||
if len(parts) == 3 {
|
||||
host, module, name = parts[0], parts[1], parts[2]
|
||||
}
|
||||
}
|
||||
if host == "" || !(host == "kept" || host == "localhost" || strings.ContainsAny(host, ".:")) || module == "" || name != a.Name {
|
||||
return "is not <registry>/<module>/" + a.Name + "@" + a.Digest
|
||||
}
|
||||
case "package":
|
||||
at := strings.LastIndex(r, "@")
|
||||
if at <= 0 || at == len(r)-1 || r[:at] == "@" {
|
||||
return "is not <name>@<version>"
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// Referenced says whether b is done and every artifact names where it is, which the controller delivers from (novox/hq
|
||||
// ADR 0498 clause 8), and names the artifacts that name no reference. A done key with no artifact (a module that builds
|
||||
// none, issue 1613) is referenced.
|
||||
func (b BuilderBuild) Referenced() (ok bool, missing []string) {
|
||||
for _, a := range b.Artifacts {
|
||||
if a.Reference == "" {
|
||||
missing = append(missing, a.Name)
|
||||
}
|
||||
}
|
||||
return b.State == BuildDone && len(missing) == 0, missing
|
||||
}
|
||||
|
||||
// The prefixes of a failed key's error that a build module answers (novox/hq ADR 0498 clauses 2, 4 and 5; ADR 0478
|
||||
// clause 2), each followed by the artifact and the words of what refused it.
|
||||
const (
|
||||
// NotStored: the push of a bundle, an archive or an upstream copy to the artifact store failed; built again.
|
||||
NotStored = "not stored: "
|
||||
// NotPinned: an upstream artifact names no digest.
|
||||
NotPinned = "not pinned: "
|
||||
// VersionTaken: the package registry holds the package's version with another packed file.
|
||||
VersionTaken = "version taken: "
|
||||
// RefusedKind: no build module builds this kind of artifact in its language.
|
||||
RefusedKind = "refused kind: "
|
||||
// WaitsFor: a fetch found the module's lock naming a package version the registry does not hold yet; the builder
|
||||
// writes the key queued with Waits (novox/hq ADR 0500 clause 6), and it is built once the package is published.
|
||||
WaitsFor = "waits for: "
|
||||
)
|
||||
|
||||
// FailedUntilChanged says whether a failed key's error is one that is not built again until the module's fingerprint
|
||||
// changes (not pinned, version taken, refused kind: the same source fails the same way); every other failed key, a
|
||||
// push that was not stored included, is built again as a new revision (ADR 0429 clause 18).
|
||||
func FailedUntilChanged(err string) bool {
|
||||
return strings.HasPrefix(err, NotPinned) || strings.HasPrefix(err, VersionTaken) || strings.HasPrefix(err, RefusedKind)
|
||||
}
|
||||
|
||||
// ToolchainUsed is what a build used for one language (ADR 0405 clauses 2 and 7): the installed toolchain's version and
|
||||
// the build module's recipe version.
|
||||
type ToolchainUsed struct {
|
||||
@@ -82,7 +169,8 @@ type ToolchainUsed struct {
|
||||
// BuilderBuild is the value of a builder build key: its State (queued, building, done or failed), By the builder
|
||||
// instance that holds it, Step what it is doing or did last, Manifest the digest of the module's manifest (the key of
|
||||
// the indexer's manifests bucket), Artifacts when done, Toolchains by language when it built or is building (a done
|
||||
// build names them, so two builders that differ are seen: ADR 0405 clause 2), and Error when failed.
|
||||
// build names them, so two builders that differ are seen: ADR 0405 clause 2), Error when failed, and Waits when queued for a
|
||||
// package version.
|
||||
type BuilderBuild struct {
|
||||
V int `json:"v"`
|
||||
State string `json:"state"`
|
||||
@@ -92,6 +180,10 @@ type BuilderBuild struct {
|
||||
Artifacts []BuiltArtifact `json:"artifacts,omitempty"`
|
||||
Toolchains map[string]ToolchainUsed `json:"toolchains,omitempty"`
|
||||
Error string `json:"error,omitempty"`
|
||||
// Waits is, on a queued key only, the package version `<name>@<version>` the module's lock names and the registry
|
||||
// does not yet hold: the key is not claimed until that package's trunk key is done and published (novox/hq ADR 0500
|
||||
// clause 6). A reader that does not know the field reads the key as queued.
|
||||
Waits string `json:"waits,omitempty"`
|
||||
}
|
||||
|
||||
func (b BuilderBuild) check() error {
|
||||
@@ -120,6 +212,17 @@ func (b BuilderBuild) check() error {
|
||||
if a.Digest == "" {
|
||||
return fmt.Errorf("delivery: the artifact %q of the build has no digest", a.Name)
|
||||
}
|
||||
if p := a.referenceProblem(); p != "" {
|
||||
return fmt.Errorf("delivery: the reference %q of the %s %q %s", a.Reference, a.Kind, a.Name, p)
|
||||
}
|
||||
}
|
||||
if b.Waits != "" {
|
||||
if b.State != BuildQueued {
|
||||
return fmt.Errorf("delivery: a %s build waits for %q; only a queued key waits for a package", b.State, b.Waits)
|
||||
}
|
||||
if p := (BuiltArtifact{Kind: "package", Reference: b.Waits}).referenceProblem(); p != "" {
|
||||
return fmt.Errorf("delivery: the package a key waits for, %q, %s", b.Waits, p)
|
||||
}
|
||||
}
|
||||
switch b.State {
|
||||
case BuildQueued, BuildBuilding:
|
||||
@@ -127,9 +230,7 @@ func (b BuilderBuild) check() error {
|
||||
return fmt.Errorf("delivery: a %s build carries no artifact and no error", b.State)
|
||||
}
|
||||
case BuildDone:
|
||||
if len(b.Artifacts) == 0 {
|
||||
return fmt.Errorf("delivery: a done build names no artifact")
|
||||
}
|
||||
// A module that builds no artifact gets a done key too, and is delivered from it (issue 1613).
|
||||
if b.Manifest == "" {
|
||||
return fmt.Errorf("delivery: a done build names no manifest digest")
|
||||
}
|
||||
@@ -144,8 +245,8 @@ func (b BuilderBuild) check() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// EncodeBuilderBuild is b as JSON at Version; it refuses a build its state does not allow (done without artifacts or a
|
||||
// manifest digest, failed without an error, an artifact without a name or digest, a toolchain without a version or a
|
||||
// EncodeBuilderBuild is b as JSON at Version; it refuses a build its state does not allow (done without a manifest
|
||||
// digest, failed without an error, an artifact without a name or digest, a toolchain without a version or a
|
||||
// recipe version).
|
||||
func EncodeBuilderBuild(b BuilderBuild) ([]byte, error) {
|
||||
if err := b.check(); err != nil {
|
||||
|
||||
@@ -0,0 +1,143 @@
|
||||
package delivery
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func artifactOf(name, kind, digest, reference string) BuiltArtifact {
|
||||
return BuiltArtifact{Name: name, Kind: kind, Digest: digest, Reference: reference}
|
||||
}
|
||||
|
||||
// A done key names where each artifact is (novox/hq ADR 0498 clauses 1, 5 and 6; the lead's decision L1 of R4e): a
|
||||
// bundle or an archive by its artifact-store blob, an image or an upstream copy by the registry's reference to its
|
||||
// digest (or the kept artifact-store form), a package by name and version. A reference of the wrong form for its kind,
|
||||
// or naming another digest or artifact, is refused, naming the artifact.
|
||||
func TestADoneKeyNamesEachArtifactsReferenceInTheFormOfItsKind(t *testing.T) {
|
||||
d := "sha256:" + digestOf('6')
|
||||
good := []BuiltArtifact{
|
||||
artifactOf("code", "bundle", d, "artifact-store://mail/code/blobs/"+d),
|
||||
artifactOf("files", "archive", d, "artifact-store://mail/files/blobs/"+d),
|
||||
artifactOf("server", "image", d, "10.0.0.1:5000/mail/server@"+d),
|
||||
artifactOf("server", "image", d, "artifact-store://mail/server@"+d),
|
||||
artifactOf("nats", "upstream", d, "registry.example/nats/nats@"+d),
|
||||
artifactOf("lib", "package", d, "@novox/mesh-sdk@0.1.15"),
|
||||
artifactOf("lib", "package", d, "left-pad@1.3.0"),
|
||||
artifactOf("old", "", d, "anything at all"),
|
||||
}
|
||||
for _, a := range good {
|
||||
b := doneBuild()
|
||||
b.Artifacts = []BuiltArtifact{a}
|
||||
if _, err := EncodeBuilderBuild(b); err != nil {
|
||||
t.Errorf("%+v was refused: %v", a, err)
|
||||
}
|
||||
}
|
||||
other := "sha256:" + digestOf('7')
|
||||
bad := []BuiltArtifact{
|
||||
artifactOf("code", "bundle", d, "artifact-store://mail/code/blobs/"+other),
|
||||
artifactOf("code", "bundle", d, "artifact-store://mail/other/blobs/"+d),
|
||||
artifactOf("code", "bundle", d, "10.0.0.1:5000/mail/code@"+d),
|
||||
artifactOf("server", "image", d, "10.0.0.1:5000/mail@"+d),
|
||||
artifactOf("server", "image", d, "10.0.0.1:5000/mail/server@"+other),
|
||||
artifactOf("server", "image", d, "mail/server@"+d),
|
||||
artifactOf("server", "image", d, "artifact-store://mail/server/blobs/"+d),
|
||||
artifactOf("lib", "package", d, "@novox/mesh-sdk"),
|
||||
artifactOf("lib", "package", d, "0.1.15"),
|
||||
}
|
||||
for _, a := range bad {
|
||||
b := doneBuild()
|
||||
b.Artifacts = []BuiltArtifact{a}
|
||||
if _, err := EncodeBuilderBuild(b); err == nil || !strings.Contains(err.Error(), `"`+a.Name+`"`) {
|
||||
t.Errorf("%+v: %v, wanted a refusal naming the artifact", a, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The controller delivers only from a done key whose every artifact names a reference (ADR 0498 clause 8): Referenced
|
||||
// says so and names the artifacts without one.
|
||||
func TestReferencedNamesTheArtifactsWithoutAReference(t *testing.T) {
|
||||
d := "sha256:" + digestOf('6')
|
||||
b := doneBuild()
|
||||
b.Artifacts = []BuiltArtifact{
|
||||
artifactOf("code", "bundle", d, "artifact-store://mail/code/blobs/"+d),
|
||||
artifactOf("server", "image", d, ""),
|
||||
}
|
||||
if ok, missing := b.Referenced(); ok || len(missing) != 1 || missing[0] != "server" {
|
||||
t.Fatalf("Referenced = %v, %v; want false, [server]", ok, missing)
|
||||
}
|
||||
b.Artifacts[1].Reference = "10.0.0.1:5000/mail/server@" + d
|
||||
if ok, missing := b.Referenced(); !ok || len(missing) != 0 {
|
||||
t.Fatalf("Referenced = %v, %v; want true", ok, missing)
|
||||
}
|
||||
if ok, _ := (BuilderBuild{State: BuildQueued}).Referenced(); ok {
|
||||
t.Fatal("a queued key is referenced")
|
||||
}
|
||||
}
|
||||
|
||||
// The errors a build module answers open with a fixed prefix (ADR 0498 clauses 2, 4 and 5; ADR 0478 clause 2), and three
|
||||
// of them are not built again until the module's fingerprint changes; a failed push is.
|
||||
func TestOnlyAPushFailureIsBuiltAgainBeforeTheFingerprintChanges(t *testing.T) {
|
||||
for _, c := range []struct {
|
||||
err string
|
||||
again bool
|
||||
}{
|
||||
{NotStored + "code: the store said 500", true},
|
||||
{NotPinned + "nats: upstream names no digest", false},
|
||||
{VersionTaken + "@novox/mesh-sdk@0.1.15 holds another file", false},
|
||||
{RefusedKind + "a package in go", false},
|
||||
{"go test: 2 failed", true},
|
||||
} {
|
||||
if got := FailedUntilChanged(c.err); got == c.again {
|
||||
t.Errorf("FailedUntilChanged(%q) = %v", c.err, got)
|
||||
}
|
||||
}
|
||||
if NotStored != "not stored: " || NotPinned != "not pinned: " || VersionTaken != "version taken: " || RefusedKind != "refused kind: " {
|
||||
t.Fatal("a prefix changed its words")
|
||||
}
|
||||
}
|
||||
|
||||
// A key whose module's lock names a version of the group's package not yet published waits, queued, saying which
|
||||
// (ADR 0500 clause 6): an older reader reads it as queued, and only a queued key waits.
|
||||
func TestAKeyWaitsQueuedForAPackageVersion(t *testing.T) {
|
||||
b := BuilderBuild{State: BuildQueued, Manifest: digestOf('4'), Waits: "@novox/mesh-sdk@0.1.15"}
|
||||
data, err := EncodeBuilderBuild(b)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
back, err := DecodeBuilderBuild(data)
|
||||
if err != nil || back.Waits != b.Waits || back.State != BuildQueued {
|
||||
t.Fatalf("decoded %+v, %v", back, err)
|
||||
}
|
||||
building := BuilderBuild{State: BuildBuilding, By: "x", Waits: "@novox/mesh-sdk@0.1.15"}
|
||||
if _, err := EncodeBuilderBuild(building); err == nil {
|
||||
t.Fatal("a building key that waits was accepted")
|
||||
}
|
||||
if _, err := EncodeBuilderBuild(BuilderBuild{State: BuildQueued, Waits: "mesh-sdk"}); err == nil {
|
||||
t.Fatal("a wait naming no version was accepted")
|
||||
}
|
||||
}
|
||||
|
||||
// A module that builds no artifact gets a done key too (issue 1613), and it is delivered: Referenced is ok for a done
|
||||
// key with no artifact, since no artifact lacks a reference; a queued one with none still is not.
|
||||
func TestADoneKeyWithNoArtifactIsReferenced(t *testing.T) {
|
||||
b := BuilderBuild{State: BuildDone, Manifest: digestOf('4')}
|
||||
data, err := EncodeBuilderBuild(b)
|
||||
if err != nil {
|
||||
t.Fatalf("a done key with no artifact was refused: %v", err)
|
||||
}
|
||||
back, err := DecodeBuilderBuild(data)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if ok, missing := back.Referenced(); !ok || len(missing) != 0 {
|
||||
t.Fatalf("Referenced = %v, %v; want true", ok, missing)
|
||||
}
|
||||
}
|
||||
|
||||
// A Node fetch that finds the lock naming a package version the registry does not hold fails with WaitsFor; the
|
||||
// builder then writes the key queued with Waits, and the error is not one that waits for the fingerprint to change.
|
||||
func TestWaitsForIsAPrefixThatIsBuiltAgain(t *testing.T) {
|
||||
if WaitsFor != "waits for: " || FailedUntilChanged(WaitsFor+"@novox/mesh-sdk@0.1.15") {
|
||||
t.Fatalf("WaitsFor = %q", WaitsFor)
|
||||
}
|
||||
}
|
||||
@@ -15,7 +15,7 @@ func doneBuild() BuilderBuild {
|
||||
Manifest: digestOf('4'),
|
||||
Artifacts: []BuiltArtifact{
|
||||
{Name: "code", Kind: "bundle", Digest: "sha256:" + digestOf('5')},
|
||||
{Name: "server", Kind: "image", Digest: "sha256:" + digestOf('6'), Reference: "registry.example/mail@sha256:" + digestOf('6')},
|
||||
{Name: "server", Kind: "image", Digest: "sha256:" + digestOf('6'), Reference: "registry.example/mail/server@sha256:" + digestOf('6')},
|
||||
},
|
||||
Toolchains: map[string]ToolchainUsed{
|
||||
"go": {Version: "1.26.0", Recipe: "3"},
|
||||
@@ -83,14 +83,13 @@ func TestBuilderBuildRoundTrip(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// 1358/AC3: done without artifacts or without a manifest digest, failed without an error, an artifact without a digest
|
||||
// 1358/AC3: done without a manifest digest, failed without an error, an artifact without a digest
|
||||
// and an unknown state are refused.
|
||||
func TestBuilderBuildChecked(t *testing.T) {
|
||||
cases := map[string]struct {
|
||||
edit func(*BuilderBuild)
|
||||
words []string
|
||||
}{
|
||||
"done without artifacts": {func(b *BuilderBuild) { b.Artifacts = nil }, []string{"artifact"}},
|
||||
"done without a manifest": {func(b *BuilderBuild) { b.Manifest = "" }, []string{"manifest"}},
|
||||
"a malformed manifest": {func(b *BuilderBuild) { b.Manifest = "abc" }, []string{"abc"}},
|
||||
"an artifact without a digest": {func(b *BuilderBuild) { b.Artifacts[1].Digest = "" }, []string{"digest", "server"}},
|
||||
@@ -125,8 +124,8 @@ func TestBuilderBuildChecked(t *testing.T) {
|
||||
t.Error("a queued build with artifacts accepted")
|
||||
}
|
||||
// decode refuses what encode refuses
|
||||
if _, err := DecodeBuilderBuild([]byte(`{"v":1,"state":"done","manifest":"` + digestOf('4') + `"}`)); err == nil {
|
||||
t.Error("decoded a done build without artifacts")
|
||||
if _, err := DecodeBuilderBuild([]byte(`{"v":1,"state":"done"}`)); err == nil {
|
||||
t.Error("decoded a done build without a manifest digest")
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,40 @@
|
||||
package delivery
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// A candidate's module carries the git objects of its named paths at the preview (novox/hq ADR 0499 clause 2), so the
|
||||
// builder's fetch request for a candidate names them; an older candidate decodes without them, and a path that is not
|
||||
// clean and relative, or names no object, is refused.
|
||||
func TestACandidatesModuleCarriesItsNamedPaths(t *testing.T) {
|
||||
c := sampleCandidate()
|
||||
paths := map[string]string{"src": strings.Repeat("a", 40), "package.json": strings.Repeat("b", 40)}
|
||||
c.Pulls[0].Fps = map[string]CandidateModule{"mesh-mail": {FP2: SourcePrefix2 + strings.Repeat("1", 64),
|
||||
Manifest: strings.Repeat("2", 64), Paths: paths}}
|
||||
c.FPSet = FingerprintSet(c.Pulls)
|
||||
data, err := EncodeCandidate(c)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
back, err := DecodeCandidate(data)
|
||||
if err != nil || back.Pulls[0].Fps["mesh-mail"].Paths["src"] != paths["src"] {
|
||||
t.Fatalf("decoded %+v, %v", back.Pulls[0].Fps, err)
|
||||
}
|
||||
old := sampleCandidate()
|
||||
old.Pulls[0].Fps = map[string]CandidateModule{"mesh-mail": {FP2: SourcePrefix2 + strings.Repeat("1", 64)}}
|
||||
old.FPSet = FingerprintSet(old.Pulls)
|
||||
if data, err := EncodeCandidate(old); err != nil {
|
||||
t.Fatal(err)
|
||||
} else if b, err := DecodeCandidate(data); err != nil || b.Pulls[0].Fps["mesh-mail"].Paths != nil {
|
||||
t.Fatalf("an older candidate: %+v, %v", b, err)
|
||||
}
|
||||
for _, bad := range []map[string]string{{"../x": "a"}, {"/src": "a"}, {"": "a"}, {"src": ""}} {
|
||||
c.Pulls[0].Fps = map[string]CandidateModule{"mesh-mail": {FP2: SourcePrefix2 + strings.Repeat("1", 64), Paths: bad}}
|
||||
c.FPSet = FingerprintSet(c.Pulls)
|
||||
if _, err := EncodeCandidate(c); err == nil {
|
||||
t.Errorf("%v was accepted", bad)
|
||||
}
|
||||
}
|
||||
}
|
||||
+3
-1
@@ -29,7 +29,9 @@ const (
|
||||
)
|
||||
|
||||
// The contexts of the statuses the mesh owes the forge: the gate's, the delivery's view of a head and of a group
|
||||
// (design 47), and the checker's, which is not required until R4f (ADR 0484 clause 2, ADR 0453 clause 4).
|
||||
// (design 47), and the checker's. From R4e the checker's mesh/check is the one status a repository's main requires,
|
||||
// in place of mesh/merge-gate and mesh/repo-check (ADR 0503 clauses 3 and 5); a main that requires only
|
||||
// mesh/repo-check (hq) keeps it, and the controller writes it for a repository the checker writes no status for.
|
||||
const (
|
||||
StatusContextMergeGate = "mesh/merge-gate"
|
||||
StatusContextDelivery = "mesh/delivery"
|
||||
|
||||
@@ -0,0 +1,32 @@
|
||||
package delivery
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// From R4e a modules value needs no old form (novox/hq ADR 0453 clause 6; issue 1851): a value with only the new
|
||||
// fingerprint, or only the reason it has none, is written and read; an old value with fp or none still reads; a value
|
||||
// with no fingerprint of either form and no reason is still refused.
|
||||
func TestAModulesValueNeedsNoOldForm(t *testing.T) {
|
||||
fp2 := SourcePrefix2 + strings.Repeat("a", 64)
|
||||
for _, m := range []Module{
|
||||
{FP2: fp2, Repository: "a/b", Commit: "c"},
|
||||
{None2: "no lock", Repository: "a/b", Commit: "c"},
|
||||
} {
|
||||
data, err := EncodeModule(m)
|
||||
if err != nil {
|
||||
t.Fatalf("%+v was refused: %v", m, err)
|
||||
}
|
||||
back, err := DecodeModule(data)
|
||||
if err != nil || back.FP2 != m.FP2 || back.None2 != m.None2 || back.FP != "" || back.None != "" {
|
||||
t.Fatalf("decoded %+v, %v", back, err)
|
||||
}
|
||||
}
|
||||
if _, err := DecodeModule([]byte(`{"v":1,"fp":"src1:x","repository":"a/b","commit":"c"}`)); err != nil {
|
||||
t.Fatalf("an old value no longer reads: %v", err)
|
||||
}
|
||||
if _, err := EncodeModule(Module{Repository: "a/b", Commit: "c"}); err == nil {
|
||||
t.Fatal("a value with no fingerprint and no reason was accepted")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,33 @@
|
||||
package delivery
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// The indexer's modules value carries each artifact's named paths (novox/hq ADR 0499; the lead's decision L8 of R4e):
|
||||
// the checker tells from it whether a change touches lib's paths without reading a manifest. An older value has none.
|
||||
func TestAModulesValueCarriesEachArtifactsNamedPaths(t *testing.T) {
|
||||
m := Module{FP2: SourcePrefix2 + strings.Repeat("a", 64), Repository: "novox/mesh-sdk", Commit: "c",
|
||||
ArtifactPaths: map[string][]string{"lib": {"package-lock.json", "package.json", "src", "test", "tsconfig.json"}}}
|
||||
data, err := EncodeModule(m)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !strings.Contains(string(data), `"paths":{"lib":[`) {
|
||||
t.Fatalf("the value carries no paths: %s", data)
|
||||
}
|
||||
back, err := DecodeModule(data)
|
||||
if err != nil || len(back.ArtifactPaths["lib"]) != 5 || back.ArtifactPaths["lib"][2] != "src" {
|
||||
t.Fatalf("decoded %+v, %v", back.ArtifactPaths, err)
|
||||
}
|
||||
if old, err := DecodeModule([]byte(`{"v":1,"fp":"src1:x","repository":"a/b","commit":"c"}`)); err != nil || old.ArtifactPaths != nil {
|
||||
t.Fatalf("an older value: %+v, %v", old, err)
|
||||
}
|
||||
for _, bad := range []map[string][]string{{"": {"src"}}, {"lib": {}}, {"lib": {"../x"}}, {"lib": {""}}} {
|
||||
m.ArtifactPaths = bad
|
||||
if _, err := EncodeModule(m); err == nil {
|
||||
t.Errorf("%v was accepted", bad)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,47 @@
|
||||
package delivery
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"path"
|
||||
"sort"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// NamedTreePrefix opens the tree input of a module whose every artifact names its paths, so such a value is never read
|
||||
// as a git tree id and a reader of a kept Tree can tell the two apart.
|
||||
const NamedTreePrefix = "named:"
|
||||
|
||||
// NamedTree is the tree input of the new-form fingerprint of a module whose every artifact names the paths it is built
|
||||
// from (novox/hq ADR 0499 clause 2): `named:` and the hex sha256 of the lines "manifest <digest>" and, sorted,
|
||||
// "path <path> <object>", joined by a newline. manifest is the module's ManifestDigest (the sha256 of its module.json's
|
||||
// exact bytes, as Module.Manifest and the merge preview carry it); paths are the named paths, each clean and relative to
|
||||
// the module's directory as the manifest names them (catalogue.Manifest.NamedPaths), with the git object id each has at
|
||||
// the commit (a blob for a file, a tree for a directory).
|
||||
//
|
||||
// **One function computes it** (clause 2): the indexer, its merge preview (buildstep.ModuleTrees) and the build modules
|
||||
// call this one, and no second computation exists. It refuses no path, a named path with no object (absent at the
|
||||
// commit), a path that is not clean, and a manifest that is not a digest, naming them: the indexer then gives the
|
||||
// module none, with that reason (clause 3).
|
||||
func NamedTree(manifest string, paths map[string]string) (string, error) {
|
||||
if _, err := ManifestsKey(manifest); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if len(paths) == 0 {
|
||||
return "", fmt.Errorf("delivery: the module names no path, so it has no named tree")
|
||||
}
|
||||
lines := make([]string, 0, len(paths))
|
||||
for p, object := range paths {
|
||||
if p == "" || p != path.Clean(p) || strings.HasPrefix(p, "/") || p == ".." || strings.HasPrefix(p, "../") {
|
||||
return "", fmt.Errorf("delivery: the named path %q is not a clean path inside the module's directory", p)
|
||||
}
|
||||
if object == "" {
|
||||
return "", fmt.Errorf("delivery: the named path %q has no git object at the commit", p)
|
||||
}
|
||||
lines = append(lines, "path "+p+" "+object)
|
||||
}
|
||||
sort.Strings(lines)
|
||||
sum := sha256.Sum256([]byte("manifest " + manifest + "\n" + strings.Join(lines, "\n")))
|
||||
return NamedTreePrefix + hex.EncodeToString(sum[:]), nil
|
||||
}
|
||||
@@ -0,0 +1,79 @@
|
||||
package delivery
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// The named paths' git objects travel beside a module's other new-form inputs (novox/hq ADR 0499 clauses 2 and 3): in
|
||||
// the forge bridge's messages (Forge.Paths), in the merge preview (PreviewModule.Paths) and in what the indexer keeps
|
||||
// (KeptInputs2.Paths), so a computation from kept inputs gives the merge's value. Each is read as absent by an older
|
||||
// reader, and an older value, without them, still decodes.
|
||||
func TestTheNamedPathsObjectsRoundTripBesideTheOtherInputs(t *testing.T) {
|
||||
manifest := sha256Of(`{"module":"mesh-sdk"}`)
|
||||
paths := map[string]string{"src": strings.Repeat("a", 40), "package.json": strings.Repeat("b", 40)}
|
||||
tree, err := NamedTree(manifest, paths)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
m := Module{FP2: SourcePrefix2 + strings.Repeat("1", 64), Manifest: manifest, Repository: "novox/mesh-sdk",
|
||||
Commit: strings.Repeat("c", 40), Inputs2: &KeptInputs2{Tree: tree, Paths: paths}}
|
||||
m.FP = SourcePrefix + strings.Repeat("0", 64)
|
||||
data, err := EncodeModule(m)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !strings.Contains(string(data), `"paths":{`) {
|
||||
t.Fatalf("the kept inputs carry no paths: %s", data)
|
||||
}
|
||||
back, err := DecodeModule(data)
|
||||
if err != nil || back.Inputs2.Paths["src"] != paths["src"] || back.Inputs2.Tree != tree {
|
||||
t.Fatalf("decoded %+v, %v", back.Inputs2, err)
|
||||
}
|
||||
// What the indexer keeps must compute to the tree it kept: a mismatch is a value no recompute could give.
|
||||
bad := m
|
||||
bad.Inputs2 = &KeptInputs2{Tree: tree, Paths: map[string]string{"src": strings.Repeat("d", 40)}}
|
||||
if _, err := EncodeModule(bad); err == nil || !strings.Contains(err.Error(), "named tree") {
|
||||
t.Fatalf("kept paths that do not compute to the kept tree were accepted: %v", err)
|
||||
}
|
||||
// An older value has no paths, and its directory tree stays its tree.
|
||||
old := `{"v":1,"fp":"` + m.FP + `","manifest":"` + manifest + `","inputs2":{"tree":"` + strings.Repeat("e", 40) +
|
||||
`"},"repository":"novox/x","commit":"` + strings.Repeat("c", 40) + `"}`
|
||||
if o, err := DecodeModule([]byte(old)); err != nil || o.Inputs2.Paths != nil {
|
||||
t.Fatalf("an older value: %+v, %v", o, err)
|
||||
}
|
||||
|
||||
f := Forge{Kind: ForgeMerged, Repository: "novox/mesh-sdk", Number: 1, Head: strings.Repeat("h", 40),
|
||||
Merge: strings.Repeat("m", 40), Root: strings.Repeat("r", 40), Trees: map[string]string{".": strings.Repeat("r", 40)},
|
||||
Paths: map[string]map[string]string{".": paths}}
|
||||
data, err = EncodeForge(f)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if fb, err := DecodeForge(data); err != nil || fb.Paths["."]["package.json"] != paths["package.json"] {
|
||||
t.Fatalf("decoded forge %+v, %v", fb.Paths, err)
|
||||
}
|
||||
f.Paths = map[string]map[string]string{"other": paths}
|
||||
if _, err := EncodeForge(f); err == nil {
|
||||
t.Fatal("named paths of a directory the message has no tree of were accepted")
|
||||
}
|
||||
f.Paths = map[string]map[string]string{".": {"src": ""}}
|
||||
if _, err := EncodeForge(f); err == nil {
|
||||
t.Fatal("a named path with no object was accepted")
|
||||
}
|
||||
|
||||
p := Preview{Repository: "novox/mesh-sdk", Head: strings.Repeat("a", 40), Trunk: strings.Repeat("b", 40),
|
||||
Merge: strings.Repeat("c", 40), Modules: map[string]PreviewModule{".": {Tree: tree, Manifest: manifest, Paths: paths}}}
|
||||
data, err = EncodePreview(p)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if pb, err := DecodePreview(data); err != nil || pb.Modules["."].Paths["src"] != paths["src"] {
|
||||
t.Fatalf("decoded preview %+v, %v", pb.Modules, err)
|
||||
}
|
||||
p.Modules["."] = PreviewModule{Tree: strings.Repeat("e", 40), Manifest: manifest, Paths: paths}
|
||||
if _, err := EncodePreview(p); err == nil || !strings.Contains(err.Error(), "named tree") {
|
||||
t.Fatalf("a preview whose tree is not its paths' named tree was accepted: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,68 @@
|
||||
package delivery
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func sha256Of(s string) string { h := sha256.Sum256([]byte(s)); return hex.EncodeToString(h[:]) }
|
||||
|
||||
// A module whose every artifact names its paths takes its tree from its manifest and those paths alone (novox/hq ADR
|
||||
// 0499 clause 2): a change outside them leaves the value equal, a change to a named path or to the manifest changes it,
|
||||
// and the order the paths are given in changes nothing.
|
||||
func TestTheNamedTreeIsOfTheManifestAndTheNamedPathsAlone(t *testing.T) {
|
||||
manifest := sha256Of(`{"module":"mesh-sdk"}`)
|
||||
paths := map[string]string{"src": strings.Repeat("a", 40), "package.json": strings.Repeat("b", 40)}
|
||||
v, err := NamedTree(manifest, paths)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !strings.HasPrefix(v, NamedTreePrefix) || !isHex64(strings.TrimPrefix(v, NamedTreePrefix)) {
|
||||
t.Fatalf("the value %q is not %s and 64 hex digits", v, NamedTreePrefix)
|
||||
}
|
||||
// A change outside the named paths is no input at all: the same inputs give the same value, in any map order.
|
||||
again, _ := NamedTree(manifest, map[string]string{"package.json": strings.Repeat("b", 40), "src": strings.Repeat("a", 40)})
|
||||
if again != v {
|
||||
t.Fatalf("the same inputs gave %q and %q", v, again)
|
||||
}
|
||||
changed, _ := NamedTree(manifest, map[string]string{"src": strings.Repeat("c", 40), "package.json": strings.Repeat("b", 40)})
|
||||
if changed == v {
|
||||
t.Fatal("a changed named path left the value equal")
|
||||
}
|
||||
otherManifest, _ := NamedTree(sha256Of(`{"module":"mesh-sdk","version":"2"}`), paths)
|
||||
if otherManifest == v {
|
||||
t.Fatal("a changed manifest left the value equal")
|
||||
}
|
||||
renamed, _ := NamedTree(manifest, map[string]string{"lib": strings.Repeat("a", 40), "package.json": strings.Repeat("b", 40)})
|
||||
if renamed == v {
|
||||
t.Fatal("a path named differently with the same object left the value equal")
|
||||
}
|
||||
|
||||
// The lines are fixed, so every computation of it agrees: pinned here.
|
||||
want := "named:" + sha256Of("manifest "+manifest+"\npath package.json "+strings.Repeat("b", 40)+"\npath src "+strings.Repeat("a", 40))
|
||||
if v != want {
|
||||
t.Fatalf("NamedTree = %q, want %q", v, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheNamedTreeRefusesWhatItCannotPin(t *testing.T) {
|
||||
manifest := sha256Of("{}")
|
||||
for _, c := range []struct {
|
||||
manifest string
|
||||
paths map[string]string
|
||||
says string
|
||||
}{
|
||||
{manifest, nil, "names no path"},
|
||||
{manifest, map[string]string{"src": ""}, `the named path "src" has no git object`},
|
||||
{"abc", map[string]string{"src": "x"}, "not a manifest digest"},
|
||||
{manifest, map[string]string{"src/../x": "x"}, "is not a clean path"},
|
||||
{manifest, map[string]string{"": "x"}, "is not a clean path"},
|
||||
{manifest, map[string]string{"/src": "x"}, "is not a clean path"},
|
||||
} {
|
||||
if _, err := NamedTree(c.manifest, c.paths); err == nil || !strings.Contains(err.Error(), c.says) {
|
||||
t.Errorf("NamedTree(%q, %v) = %v, wanted a refusal saying %q", c.manifest, c.paths, err, c.says)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -98,12 +98,15 @@ func ParsePreviewKey(key string) (repository, head, trunk string, err error) {
|
||||
return repository, parts[1], parts[2], nil
|
||||
}
|
||||
|
||||
// PreviewModule is what the preview holds of one module directory: the tree sha of the directory, the digest of its
|
||||
// PreviewModule is what the preview holds of one module directory: the tree sha of the directory (or its named tree), the digest of its
|
||||
// manifest (the key in the manifests bucket) and the hex sha256 of its lock files by path relative to it.
|
||||
type PreviewModule struct {
|
||||
Tree string `json:"tree"`
|
||||
Manifest string `json:"manifest"`
|
||||
Locks map[string]string `json:"locks,omitempty"`
|
||||
// Paths are, of a module whose every artifact names its paths, each named path's git object id at the merge; Tree
|
||||
// is then their NamedTree (novox/hq ADR 0499 clause 2). Absent otherwise.
|
||||
Paths map[string]string `json:"paths,omitempty"`
|
||||
}
|
||||
|
||||
// Preview is the value of a preview key: the repository, the head and the trunk commit it was merged onto, the merge
|
||||
@@ -152,6 +155,9 @@ func (p Preview) check() error {
|
||||
return fmt.Errorf("delivery: the lock %q of the preview's module %q has an empty path or hash", path, dir)
|
||||
}
|
||||
}
|
||||
if err := checkNamed(m.Tree, m.Manifest, m.Paths, fmt.Sprintf("the preview's module %q", dir)); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
+86
-11
@@ -5,6 +5,7 @@ import (
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"path"
|
||||
"reflect"
|
||||
"sort"
|
||||
"strings"
|
||||
@@ -276,6 +277,9 @@ type CandidateModule struct {
|
||||
FP2 string `json:"fp2,omitempty"`
|
||||
None2 string `json:"none2,omitempty"`
|
||||
Manifest string `json:"manifest,omitempty"`
|
||||
// Paths are, of a module whose every artifact names its paths, each named path's git object id at the preview
|
||||
// (novox/hq ADR 0499 clause 2), so the builder's fetch request for the candidate names them. Absent otherwise.
|
||||
Paths map[string]string `json:"paths,omitempty"`
|
||||
}
|
||||
|
||||
func (m CandidateModule) check() error {
|
||||
@@ -285,6 +289,9 @@ func (m CandidateModule) check() error {
|
||||
case m.FP2 == "" && m.None2 == "":
|
||||
return fmt.Errorf("delivery: a module has neither fp2 nor none2")
|
||||
}
|
||||
if err := checkPathObjects(m.Paths); err != nil {
|
||||
return err
|
||||
}
|
||||
return checkNewForm(m.FP2, m.Manifest)
|
||||
}
|
||||
|
||||
@@ -356,7 +363,7 @@ func (c Candidate) check() error {
|
||||
if err := m.check(); err != nil {
|
||||
return fmt.Errorf("delivery: the candidate's module %q: %w", module, err)
|
||||
}
|
||||
if before, ok := seen[module]; ok && before != m {
|
||||
if before, ok := seen[module]; ok && !reflect.DeepEqual(before, m) {
|
||||
return fmt.Errorf("delivery: two pull requests of the candidate give the module %q different fingerprints", module)
|
||||
}
|
||||
seen[module] = m
|
||||
@@ -370,12 +377,13 @@ func (c Candidate) check() error {
|
||||
|
||||
// Module is the value of a modules key: the module's source fingerprint, or none and why (a build that resolves
|
||||
// packages when built gets none, said here), and where it was computed: Repository and Commit are the trunk merge,
|
||||
// Src the source revision then (0 before the gate writes source.main). Exactly one of FP and None is set.
|
||||
// Src the source revision then (0 before the gate writes source.main). At most one of FP and None is set.
|
||||
//
|
||||
// From R4c the indexer writes the new form beside it (novox/hq ADR 0453 clause 6): FP2, the fingerprint of the new
|
||||
// form (`src2:` and 64 hex digits), or None2, why it has none; one of them whenever either is, never both; and
|
||||
// Manifest, the digest of the module's manifest, the key of the manifests bucket. A value written before has none of
|
||||
// the three, and a reader of today's shape ignores them.
|
||||
// the three, and a reader of today's shape ignores them. From R4e a value may carry the new form alone (ADR 0453 clause
|
||||
// 6; issue 1851): one of FP, None, FP2 and None2 is set, and the old form stays readable.
|
||||
type Module struct {
|
||||
V int `json:"v"`
|
||||
FP string `json:"fp,omitempty"`
|
||||
@@ -398,6 +406,11 @@ type Module struct {
|
||||
// builds the fingerprint only with these. Empty when FP2 is empty or the module has no toolchain; a value written
|
||||
// before has none (issue 1615).
|
||||
Toolchains map[string]ToolchainUsed `json:"toolchains,omitempty"`
|
||||
// ArtifactPaths are, of each artifact that names the paths it is built from, those paths as its manifest names
|
||||
// them, by artifact (novox/hq ADR 0499 clause 1): the indexer fills them from each catalogue.Artifact.Paths it reads,
|
||||
// so the checker tells whether a change touches an artifact's build source without reading a manifest (the lead's
|
||||
// decision L8 of R4e). Absent for an artifact that names none, and in a value written before.
|
||||
ArtifactPaths map[string][]string `json:"paths,omitempty"`
|
||||
}
|
||||
|
||||
// RootPath is Module.Path of a module whose manifest is its repository's root.
|
||||
@@ -411,6 +424,39 @@ type KeptInputs2 struct {
|
||||
Builder string `json:"builder,omitempty"`
|
||||
Contexts map[string]string `json:"contexts,omitempty"`
|
||||
Locks map[string]string `json:"locks,omitempty"`
|
||||
// Paths are, of a module whose every artifact names its paths, each named path's git object id at the merge, by
|
||||
// path relative to the module's directory; Tree is then their NamedTree with the module's manifest digest, so a
|
||||
// computation from kept inputs gives the merge's value (novox/hq ADR 0499 clause 2). Absent for every other module,
|
||||
// whose Tree is its directory's tree, and in a value written before.
|
||||
Paths map[string]string `json:"paths,omitempty"`
|
||||
}
|
||||
|
||||
// checkPathObjects refuses a named path that is not clean and relative to the module's directory, or names no object.
|
||||
func checkPathObjects(paths map[string]string) error {
|
||||
for p, object := range paths {
|
||||
if p == "" || p != path.Clean(p) || strings.HasPrefix(p, "/") || p == ".." || strings.HasPrefix(p, "../") {
|
||||
return fmt.Errorf("delivery: the named path %q is not a clean path inside the module's directory", p)
|
||||
}
|
||||
if object == "" {
|
||||
return fmt.Errorf("delivery: the named path %q names no git object", p)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// checkNamed refuses kept named paths that do not compute to the kept tree with the module's manifest digest.
|
||||
func checkNamed(tree, manifest string, paths map[string]string, of string) error {
|
||||
if len(paths) == 0 {
|
||||
return nil
|
||||
}
|
||||
named, err := NamedTree(manifest, paths)
|
||||
if err != nil {
|
||||
return fmt.Errorf("delivery: %s: %w", of, err)
|
||||
}
|
||||
if named != tree {
|
||||
return fmt.Errorf("delivery: %s keeps the tree %q, and its named paths give the named tree %q", of, tree, named)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// With is the full inputs of module m: the kept ones and the given toolchains.
|
||||
@@ -423,8 +469,8 @@ func (m Module) check() error {
|
||||
switch {
|
||||
case m.FP != "" && m.None != "":
|
||||
return fmt.Errorf("delivery: a module has both a fingerprint and a reason for none")
|
||||
case m.FP == "" && m.None == "":
|
||||
return fmt.Errorf("delivery: a module has neither a fingerprint nor a reason for none")
|
||||
case m.FP == "" && m.None == "" && m.FP2 == "" && m.None2 == "":
|
||||
return fmt.Errorf("delivery: a module has neither a fingerprint nor a reason for none, of either form")
|
||||
case m.FP != "" && !strings.HasPrefix(m.FP, SourcePrefix):
|
||||
return fmt.Errorf("delivery: the module's fingerprint %q does not start with %s", m.FP, SourcePrefix)
|
||||
case m.FP2 != "" && m.None2 != "":
|
||||
@@ -438,6 +484,21 @@ func (m Module) check() error {
|
||||
if m.Inputs2 != nil && m.Manifest == "" {
|
||||
return fmt.Errorf("delivery: a module keeps new-form inputs but has no manifest digest")
|
||||
}
|
||||
for artifact, paths := range m.ArtifactPaths {
|
||||
if artifact == "" || len(paths) == 0 {
|
||||
return fmt.Errorf("delivery: the module's named paths of the artifact %q are empty or name no artifact", artifact)
|
||||
}
|
||||
for _, p := range paths {
|
||||
if p == "" || p != path.Clean(p) || strings.HasPrefix(p, "/") || p == ".." || strings.HasPrefix(p, "../") {
|
||||
return fmt.Errorf("delivery: the artifact %q names the path %q, not a clean path inside the module's directory", artifact, p)
|
||||
}
|
||||
}
|
||||
}
|
||||
if m.Inputs2 != nil {
|
||||
if err := checkNamed(m.Inputs2.Tree, m.Manifest, m.Inputs2.Paths, "the module's kept inputs"); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return checkNewForm(m.FP2, m.Manifest)
|
||||
}
|
||||
|
||||
@@ -465,9 +526,13 @@ type Forge struct {
|
||||
// relative to it (novox/hq ADR 0465 clause 1). Older writers send neither.
|
||||
Manifests map[string]string `json:"manifests,omitempty"`
|
||||
Locks map[string]map[string]string `json:"locks,omitempty"`
|
||||
Act string `json:"act,omitempty"`
|
||||
Words string `json:"words,omitempty"`
|
||||
Open []OpenPull `json:"open,omitempty"`
|
||||
// Paths are, of each module directory whose manifest names paths on every artifact, each named path's git object
|
||||
// id at the same commit as its manifest, by path relative to the directory (novox/hq ADR 0499 clause 3). The
|
||||
// indexer names the paths; a message without them gives such a module none. Older writers send none.
|
||||
Paths map[string]map[string]string `json:"paths,omitempty"`
|
||||
Act string `json:"act,omitempty"`
|
||||
Words string `json:"words,omitempty"`
|
||||
Open []OpenPull `json:"open,omitempty"`
|
||||
}
|
||||
|
||||
func (f Forge) check() error {
|
||||
@@ -537,8 +602,8 @@ func (f Forge) check() error {
|
||||
if len(f.Dirs) > 0 || f.DirsKnown {
|
||||
return fmt.Errorf("delivery: a refused message carries no module directories")
|
||||
}
|
||||
if len(f.Manifests) > 0 || len(f.Locks) > 0 {
|
||||
return fmt.Errorf("delivery: a refused message carries no manifests or lock hashes")
|
||||
if len(f.Manifests) > 0 || len(f.Locks) > 0 || len(f.Paths) > 0 {
|
||||
return fmt.Errorf("delivery: a refused message carries no manifests, lock hashes or named paths")
|
||||
}
|
||||
if f.Act == "" {
|
||||
return fmt.Errorf("delivery: a refused message needs the act it answers")
|
||||
@@ -571,6 +636,16 @@ func (f Forge) checkInputs(known func(dir string) bool, of string) error {
|
||||
}
|
||||
}
|
||||
}
|
||||
for dir, paths := range f.Paths {
|
||||
if !known(dir) {
|
||||
return fmt.Errorf("delivery: the named paths of the directory %q are not of %s", dir, of)
|
||||
}
|
||||
for path, object := range paths {
|
||||
if path == "" || object == "" {
|
||||
return fmt.Errorf("delivery: the named path %q of the directory %q has an empty path or object", path, dir)
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -578,7 +653,7 @@ func (f Forge) checkInputs(known func(dir string) bool, of string) error {
|
||||
// an open pull request that is not `<owner>/<name>` and a number.
|
||||
func (f Forge) checkOpen() error {
|
||||
if f.Repository != "" || f.Branch != "" || f.Number != 0 || f.Head != "" || f.Action != "" || len(f.Dirs) > 0 || f.DirsKnown ||
|
||||
f.Merge != "" || len(f.Trees) > 0 || len(f.Manifests) > 0 || len(f.Locks) > 0 || f.Root != "" || f.Act != "" || f.Words != "" {
|
||||
f.Merge != "" || len(f.Trees) > 0 || len(f.Manifests) > 0 || len(f.Locks) > 0 || len(f.Paths) > 0 || f.Root != "" || f.Act != "" || f.Words != "" {
|
||||
return fmt.Errorf("delivery: an open message carries only the open pull requests it names")
|
||||
}
|
||||
for _, p := range f.Open {
|
||||
|
||||
+80
@@ -0,0 +1,80 @@
|
||||
package stdio
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/announce"
|
||||
)
|
||||
|
||||
// What this bundle said it keeps (novox/hq ADR 0495 clause 2; issue 1790).
|
||||
var (
|
||||
keepsMu sync.Mutex
|
||||
kept map[string]bool // nil until Keeps: no state is refused then
|
||||
announced []byte // the announcement, published once the bundle serves
|
||||
)
|
||||
|
||||
// Keeps says on the bus, as this bundle's module, every state it keeps and which of them others may read (novox/hq
|
||||
// ADR 0495 clause 2): what a reader is granted is what the owner announced (clause 3). Call it before Serve; the
|
||||
// announcement is published through the runtime once the bundle serves, or at once when it already does. From then
|
||||
// on a state of the module's own that it did not announce is refused at the call, naming it, so code and announcement
|
||||
// cannot drift (clause 5). It refuses a state no bucket could be named by and a state given twice.
|
||||
func Keeps(states ...announce.State) error {
|
||||
module := os.Getenv("MESH_MODULE")
|
||||
if module == "" {
|
||||
module = os.Getenv(ServedAs)
|
||||
}
|
||||
data, err := announce.Encode(announce.States{Module: module, Node: os.Getenv("MESH_NODE"), States: states})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
names := map[string]bool{}
|
||||
for _, s := range states {
|
||||
names[s.Name] = true
|
||||
}
|
||||
keepsMu.Lock()
|
||||
kept, announced = names, data
|
||||
keepsMu.Unlock()
|
||||
currentMu.Lock()
|
||||
serving := current != nil
|
||||
currentMu.Unlock()
|
||||
if serving {
|
||||
return announce1()
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// announce1 publishes the announcement Keeps made, if any.
|
||||
func announce1() error {
|
||||
keepsMu.Lock()
|
||||
data := announced
|
||||
keepsMu.Unlock()
|
||||
if data == nil {
|
||||
return nil
|
||||
}
|
||||
return Emit(announce.Event, json.RawMessage(data))
|
||||
}
|
||||
|
||||
// refusedUnannounced is the error of a call on a state of the module's own that Keeps did not name, or nil. A read of
|
||||
// another module's state (`<module>.<name>`) is the grant's business.
|
||||
func refusedUnannounced(name string) error {
|
||||
if strings.Contains(name, ".") {
|
||||
return nil
|
||||
}
|
||||
keepsMu.Lock()
|
||||
defer keepsMu.Unlock()
|
||||
if kept != nil && !kept[name] {
|
||||
return fmt.Errorf("the state %q is not among the states this module announced it keeps (stdio.Keeps)", name)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// forgetKeeps is for a test: nothing announced.
|
||||
func forgetKeeps() {
|
||||
keepsMu.Lock()
|
||||
kept, announced = nil, nil
|
||||
keepsMu.Unlock()
|
||||
}
|
||||
@@ -0,0 +1,63 @@
|
||||
package stdio
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/announce"
|
||||
)
|
||||
|
||||
// A Go bundle says what it keeps from its code (novox/hq ADR 0495 clauses 1 and 2): Keeps, called before Serve,
|
||||
// announces through the runtime once the bundle serves, as the module the runtime names; and a state of its own that it
|
||||
// did not announce is refused at the call, naming it, so code and announcement cannot drift.
|
||||
func TestABundleAnnouncesWhatItKeepsAndBindsNothingElse(t *testing.T) {
|
||||
t.Setenv(ServedAs, "chime")
|
||||
t.Setenv("MESH_MODULE", "chime")
|
||||
t.Setenv("MESH_NODE", "anchor")
|
||||
defer forgetKeeps()
|
||||
// Not yet serving: an earlier test's channel is gone with its test.
|
||||
currentMu.Lock()
|
||||
current = nil
|
||||
currentMu.Unlock()
|
||||
if err := Keeps(announce.State{Name: "rings", Readable: true}, announce.State{Name: "own"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := State("other").Get("k"); err == nil || !strings.Contains(err.Error(), `"other"`) {
|
||||
t.Fatalf("a state not announced was bound: %v", err)
|
||||
}
|
||||
inR, inW := io.Pipe()
|
||||
outR, outW := io.Pipe()
|
||||
go func() { _ = ServeOn("", nil, inR, outW) }()
|
||||
defer inW.Close()
|
||||
lines := bufio.NewScanner(outR)
|
||||
got := make(chan map[string]any, 1)
|
||||
go func() {
|
||||
if lines.Scan() {
|
||||
var m map[string]any
|
||||
_ = json.Unmarshal(lines.Bytes(), &m)
|
||||
got <- m
|
||||
}
|
||||
}()
|
||||
var asked map[string]any
|
||||
select {
|
||||
case asked = <-got:
|
||||
case <-time.After(3 * time.Second):
|
||||
t.Fatal("the bundle announced nothing")
|
||||
}
|
||||
p, _ := asked["params"].(map[string]any)
|
||||
if asked["method"] != "mesh/publish" || p["key"] != announce.Event {
|
||||
t.Fatalf("the announcement was not published through the runtime: %v", asked)
|
||||
}
|
||||
raw, _ := json.Marshal(p["body"])
|
||||
s, err := announce.Decode(raw)
|
||||
if err != nil || s.Module != "chime" || s.Node != "anchor" || !s.Readable("rings") || !s.Keeps("own") || s.Readable("own") {
|
||||
t.Fatalf("announced %s (%v)", raw, err)
|
||||
}
|
||||
if err := Keeps(announce.State{Name: "bad_name"}); err == nil {
|
||||
t.Fatal("an announcement no bucket could be named by was accepted")
|
||||
}
|
||||
}
|
||||
+7
-2
@@ -10,13 +10,17 @@
|
||||
# 1. the unit level: `go test ./...` untagged, no bus or other process, timed and held to a bound
|
||||
# (unit_bound below); over it the check fails and names the slowest tests (issue 895);
|
||||
# 2. the integration level: the whole suite with `-tags integration`, each test against a bus of its
|
||||
# own (internal/testbus, which issue 1171 brings with the first test that needs a bus) so nothing is
|
||||
# own (testbus, which issue 1171 brings with the first test that needs a bus) so nothing is
|
||||
# serialised. With no tagged test yet, the level runs the unit tests again and says their count.
|
||||
#
|
||||
# **A test that starts a bus or another process carries `//go:build integration`** as the first line of
|
||||
# its file; a file that held both kinds is two files (`<name>_integration_test.go` beside `<name>_test.go`).
|
||||
# Each level says its test count and its time.
|
||||
#
|
||||
# **A test that needs bubblewrap carries `//go:build integration && isolation`** (issue 1309's proof that a run step
|
||||
# is shut off): this script vets it but does not run it, since the toolchain it runs in has no bubblewrap; the checker
|
||||
# runs the `isolation` tag on the build nodes, which have it (R4e).
|
||||
#
|
||||
# No dependency is committed (no vendor directory: .gitignore; the operator, 2026-10-19; issue 1304). Every dependency
|
||||
# is resolved by the version and checksum go.sum names (novox/hq ADR 0405 §5), from its public source: this binding
|
||||
# requires nothing of the mesh's own forge. `go mod verify` fails by module name when a downloaded dependency does not
|
||||
@@ -98,6 +102,7 @@ fi
|
||||
go vet ./...
|
||||
# The files tagged `integration` are vetted too, or nothing would read them.
|
||||
go vet -tags integration ./...
|
||||
go vet -tags integration,isolation ./...
|
||||
|
||||
# NOT RACE-CHECKED here: the SDK's suite is small and carries no store or other shared state between
|
||||
# packages, so the race detector is not required of this check (novox/hq issue 1175); a module that needs
|
||||
@@ -117,7 +122,7 @@ if [ "$status" -ne 0 ]; then
|
||||
fi
|
||||
judge_unit_run "$work/unit.out" "$unit_bound"
|
||||
|
||||
# The integration level: the whole suite with the tag, each test against its own bus (internal/testbus).
|
||||
# The integration level: the whole suite with the tag, each test against its own bus (testbus).
|
||||
integration_start=$(date +%s)
|
||||
status=0
|
||||
go test -tags integration -count=1 -v ./... >"$work/integration.out" 2>&1 || status=$?
|
||||
|
||||
@@ -24,6 +24,12 @@ type Conn struct {
|
||||
mu sync.Mutex
|
||||
next uint64
|
||||
pending map[string]map[uint64]context.CancelCauseFunc
|
||||
|
||||
// What Keeps announced (keeps.go): the states' names (nil before Keeps), the announcement, and whether a
|
||||
// reconnect announces it again.
|
||||
kept map[string]bool
|
||||
announced []byte
|
||||
reannounces bool
|
||||
}
|
||||
|
||||
// errPermissionRefused marks a context cancelled because the bus refused this account's publish of the
|
||||
|
||||
@@ -22,7 +22,7 @@ import (
|
||||
)
|
||||
|
||||
// selfSignedServer is a nats-server with JetStream and TLS, self-signed, for this test alone (mirroring
|
||||
// go/internal/testbus.Start, with a certificate added since testbus has none): a mesh's bus presents its own
|
||||
// go/testbus.Start, with a certificate added since testbus has none): a mesh's bus presents its own
|
||||
// certificate in no public trust store, which is exactly what a process pins to (ADR 0436 clause 4).
|
||||
func selfSignedServer(t *testing.T) (url, fingerprint string) {
|
||||
t.Helper()
|
||||
|
||||
@@ -38,6 +38,7 @@ var sendGrants = []string{
|
||||
"$KV.mesh-mail_sends.>",
|
||||
"mesh.mod.mesh-mail.event.mail.send.failed",
|
||||
"mesh.mod.mesh-mail.event.mail.send.sent",
|
||||
"mesh.mod.mesh-mail.event.states",
|
||||
"mesh.seat.operator-channel.call.tell.*.novox~mesh-mail~send",
|
||||
"mesh.seat.operator-channel.call.tell.novox~mesh-mail~send",
|
||||
"mesh.seat.operator-channel.tool.tell",
|
||||
|
||||
@@ -0,0 +1,71 @@
|
||||
package process
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/announce"
|
||||
)
|
||||
|
||||
// Keeps says on the bus, as this process's module, every state it keeps and which of them others may read (novox/hq
|
||||
// ADR 0495 clause 2; issue 1790), on announce.Subject; what a reader is granted is what the owner announced (clause 3).
|
||||
// It publishes at once and again after every reconnect, so the controller hears it whenever it missed it. From then on
|
||||
// a state of the module that it did not announce is refused at the call, naming it, so code and announcement cannot
|
||||
// drift (clause 5). It refuses a state no bucket could be named by and a state given twice.
|
||||
func (c *Conn) Keeps(states ...announce.State) error {
|
||||
if err := c.remember(states); err != nil {
|
||||
return err
|
||||
}
|
||||
c.mu.Lock()
|
||||
first := !c.reannounces
|
||||
c.reannounces = true
|
||||
c.mu.Unlock()
|
||||
if first {
|
||||
before := c.nc.Opts.ReconnectedCB
|
||||
c.nc.SetReconnectHandler(func(nc *nats.Conn) {
|
||||
if before != nil {
|
||||
before(nc)
|
||||
}
|
||||
_ = c.publishKeeps()
|
||||
})
|
||||
}
|
||||
return c.publishKeeps()
|
||||
}
|
||||
|
||||
// remember makes the announcement of states and binds only them from now on.
|
||||
func (c *Conn) remember(states []announce.State) error {
|
||||
data, err := announce.Encode(announce.States{Module: c.account.Module, Node: c.account.Node, States: states})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
names := map[string]bool{}
|
||||
for _, s := range states {
|
||||
names[s.Name] = true
|
||||
}
|
||||
c.mu.Lock()
|
||||
c.kept, c.announced = names, data
|
||||
c.mu.Unlock()
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *Conn) publishKeeps() error {
|
||||
c.mu.Lock()
|
||||
data := c.announced
|
||||
c.mu.Unlock()
|
||||
if err := c.Emit(announce.Event, json.RawMessage(data)); err != nil {
|
||||
return fmt.Errorf("announcing what %s keeps: %w", c.account.Module, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// refusedUnannounced is the error of a bind of a state Keeps did not name, or nil (also before Keeps).
|
||||
func (c *Conn) refusedUnannounced(name string) error {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
if c.kept != nil && !c.kept[name] {
|
||||
return fmt.Errorf("the state %q is not among the states %s announced it keeps (Conn.Keeps)", name, c.account.Module)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,45 @@
|
||||
//go:build integration
|
||||
|
||||
package process
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/announce"
|
||||
)
|
||||
|
||||
// 1790/S10: on a real bus, a process's Keeps lands on its module's own announcement subject, in the events stream, as
|
||||
// the module's announcement; a state it did not announce is then refused at the call.
|
||||
func TestAProcessAnnouncesWhatItKeepsOnTheBus(t *testing.T) {
|
||||
url, account := grantedSendProcess(t)
|
||||
_, admin := adminJS(t, url)
|
||||
stream, err := admin.CreateStream(ctxFor(t), jetstream.StreamConfig{Name: "MAIL_STATES", Subjects: []string{announce.Subject("mesh-mail")}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
conn, err := Connect(account)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer conn.Close()
|
||||
if err := conn.Keeps(announce.State{Name: "accounts", Readable: true}, announce.State{Name: "drafts"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
msg, err := stream.GetMsg(ctxFor(t), 1)
|
||||
if err != nil {
|
||||
t.Fatalf("no announcement in the stream: %v", err)
|
||||
}
|
||||
if msg.Subject != "mesh.mod.mesh-mail.event.states" || msg.Header.Get("x-source") != "mesh-mail" {
|
||||
t.Fatalf("announced on %s with %v", msg.Subject, msg.Header)
|
||||
}
|
||||
got, err := announce.Decode(msg.Data)
|
||||
if err != nil || !got.Readable("accounts") || !got.Keeps("drafts") || got.Readable("drafts") || got.Node != "novox" {
|
||||
t.Fatalf("announced %s (%v)", msg.Data, err)
|
||||
}
|
||||
if _, err := conn.State("outbox").Get("k"); err == nil || !strings.Contains(err.Error(), `"outbox"`) {
|
||||
t.Fatalf("an unannounced state was bound: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,28 @@
|
||||
package process
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/announce"
|
||||
)
|
||||
|
||||
// After Keeps a process binds only the states it announced (novox/hq ADR 0495 clause 5): any other of its module's
|
||||
// states is refused at the call, naming it, before the bus is asked; before Keeps nothing is refused.
|
||||
func TestAProcessBindsOnlyTheStatesItAnnounced(t *testing.T) {
|
||||
c := &Conn{account: Account{Module: "mesh-mail", Node: "novox"}}
|
||||
if err := c.remember([]announce.State{{Name: "accounts", Readable: true}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, err := c.State("drafts").Get("k")
|
||||
if err == nil || !strings.Contains(err.Error(), `"drafts"`) || !strings.Contains(err.Error(), "Keeps") {
|
||||
t.Fatalf("an unannounced state was bound: %v", err)
|
||||
}
|
||||
if err := c.remember([]announce.State{{Name: "a_b"}}); err == nil {
|
||||
t.Fatal("an announcement no bucket could be named by was accepted")
|
||||
}
|
||||
got, err := announce.Decode(c.announced)
|
||||
if err != nil || got.Module != "mesh-mail" || got.Node != "novox" || !got.Readable("accounts") {
|
||||
t.Fatalf("the announcement is %s (%v)", c.announced, err)
|
||||
}
|
||||
}
|
||||
@@ -43,6 +43,9 @@ func (s State) bucket() string { return s.Module + "_" + s.Name }
|
||||
func (c *Conn) State(name string) State {
|
||||
bucket := c.account.Module + "_" + name
|
||||
return State{Module: c.account.Module, Name: name, closed: func() bool { return c.nc.IsClosed() }, kv: func(ctx context.Context) (jetstream.KeyValue, error) {
|
||||
if err := c.refusedUnannounced(name); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
js, err := jetstream.New(c.nc)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
|
||||
@@ -50,6 +50,9 @@ func State(name string) KeptState { return KeptState{name: name} }
|
||||
const stateTimeout = 30 * time.Second
|
||||
|
||||
func (s KeptState) ask(verb string, params map[string]any) (json.RawMessage, error) {
|
||||
if err := refusedUnannounced(s.name); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
c, err := live()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
|
||||
@@ -180,6 +180,9 @@ func ServeOn(name string, tools []Tool, in io.Reader, out io.Writer) error {
|
||||
currentMu.Lock()
|
||||
current = c
|
||||
currentMu.Unlock()
|
||||
// What the bundle said it keeps is announced once it serves: the runtime answers the publish on this channel, so
|
||||
// it is asked from beside the loop that reads the answer.
|
||||
go func() { _ = announce1() }()
|
||||
scanner := bufio.NewScanner(in)
|
||||
scanner.Buffer(make([]byte, 1<<20), 1<<24)
|
||||
for scanner.Scan() {
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
// Package testbus is the bus an integration test of the SDK's Go binding runs against: a server of
|
||||
// its own, of the release go.mod pins, started for the one test and gone after it.
|
||||
// its own, of the release go.mod pins, started for the one test and gone after it. It is exported so a
|
||||
// module's own integration tests start against the same (announcetest, novox/hq ADR 0495 clause 5;
|
||||
// issue 1182 names the controller's copy).
|
||||
//
|
||||
// **Why one per test.** A test that reaches a shared bus asserts, reads and removes objects by fixed
|
||||
// name — a stream, a consumer — and two tests at once would step on each other's: one deleting what
|
||||
-21
@@ -1,21 +0,0 @@
|
||||
MIT License
|
||||
|
||||
Copyright (c) 2024 Antithesis
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all
|
||||
copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
SOFTWARE.
|
||||
-214
@@ -1,214 +0,0 @@
|
||||
//go:build enable_antithesis_sdk
|
||||
|
||||
// Package assert enables defining [test properties] about your program or [workload]. It is part of the [Antithesis Go SDK], which enables Go applications to integrate with the [Antithesis platform].
|
||||
//
|
||||
// Code that uses this package should be instrumented with the [antithesis-go-generator] utility. This step is required for the Always, Sometime, and Reachable methods. It is not required for the Unreachable and AlwaysOrUnreachable methods, but it will improve the experience of using them.
|
||||
//
|
||||
// These functions are no-ops with minimal performance overhead when called outside of the Antithesis environment. However, if the environment variable ANTITHESIS_SDK_LOCAL_OUTPUT is set, these functions will log to the file pointed to by that variable using a structured JSON format defined [here]. This allows you to make use of the Antithesis assertions package in your regular testing, or even in production. In particular, very few assertions frameworks offer a convenient way to define [Sometimes assertions], but they can be quite useful even outside Antithesis.
|
||||
//
|
||||
// Each function in this package takes a parameter called message, which is a human readable identifier used to aggregate assertions. Antithesis generates one test property per unique message and this test property will be named "<message>" in the [triage report].
|
||||
//
|
||||
// This test property either passes or fails, which depends upon the evaluation of every assertion that shares its message. Different assertions in different parts of the code should have different message, but the same assertion should always have the same message even if it is moved to a different file.
|
||||
//
|
||||
// Each function also takes a parameter called details, which is a key-value map of optional additional information provided by the user to add context for assertion failures. The information that is logged will appear in the [triage report], under the details section of the corresponding property. Normally the values passed to details are evaluated at runtime.
|
||||
//
|
||||
// [Antithesis Go SDK]: https://antithesis.com/docs/using_antithesis/sdk/go/
|
||||
// [Antithesis platform]: https://antithesis.com
|
||||
// [test properties]: https://antithesis.com/docs/properties_assertions/properties/
|
||||
// [workload]: https://antithesis.com/docs/test_templates/first_test/
|
||||
// [antithesis-go-generator]: https://antithesis.com/docs/using_antithesis/sdk/go/instrumentor/
|
||||
// [triage report]: https://antithesis.com/docs/reports/
|
||||
// [here]: https://antithesis.com/docs/using_antithesis/sdk/fallback/
|
||||
// [Sometimes assertions]: https://antithesis.com/docs/best_practices/sometimes_assertions/
|
||||
//
|
||||
// [details]: https://antithesis.com/docs/reports/
|
||||
package assert
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
)
|
||||
|
||||
type assertInfo struct {
|
||||
Location *locationInfo `json:"location"`
|
||||
Details map[string]any `json:"details"`
|
||||
AssertType string `json:"assert_type"`
|
||||
DisplayType string `json:"display_type"`
|
||||
Message string `json:"message"`
|
||||
Id string `json:"id"`
|
||||
Hit bool `json:"hit"`
|
||||
MustHit bool `json:"must_hit"`
|
||||
Condition bool `json:"condition"`
|
||||
}
|
||||
|
||||
// Create a custom json marshaler for assertInfo so that we can force Errors to be marshaled with their error details.
|
||||
// Without this, custom errors are marshaled as an empty object because the default json marshaler doesn't include the error
|
||||
// (because it's a method - not an exported struct field).
|
||||
func (f assertInfo) MarshalJSON() ([]byte, error) {
|
||||
type alias assertInfo // prevent infinite recursion
|
||||
a := alias(f)
|
||||
if a.Details != nil {
|
||||
a.Details = normalizeMap(a.Details)
|
||||
}
|
||||
return json.Marshal(a)
|
||||
}
|
||||
|
||||
type jsonError struct {
|
||||
innerError error
|
||||
}
|
||||
|
||||
func (e jsonError) MarshalJSON() ([]byte, error) {
|
||||
// Marshal this as the debug output string instead of e.Error(). These should be equivalent, but Sprintf correctly
|
||||
// handles nil values for us (which otherwise are annoying to defend against due to this - https://go.dev/doc/faq#nil_error)
|
||||
return json.Marshal(fmt.Sprintf("%+v", e.innerError))
|
||||
}
|
||||
|
||||
// Recursively replace any `error` with jsonError while doing a deep copy.
|
||||
// Most of the logic is in the normalize method below. This method exists to localize the type assertions
|
||||
// and provide a function that takes in/out a map instead of any.
|
||||
func normalizeMap(v map[string]any) map[string]any {
|
||||
return normalize(v).(map[string]any)
|
||||
}
|
||||
|
||||
func normalize(input any) any {
|
||||
// This switch will miss some cases (pointers, structs, non-any types), but should catch a very large proportion of real error interfaces
|
||||
// in real details objects. We can augment this if we find other cases common enough to support.
|
||||
switch inputTyped := input.(type) {
|
||||
case error:
|
||||
// Check if the underlying error implements json.Marshaler, so that if the error
|
||||
// already knows who to marshal itself, we don't override that.
|
||||
if _, ok := inputTyped.(json.Marshaler); ok {
|
||||
return inputTyped
|
||||
} else {
|
||||
return jsonError{inputTyped}
|
||||
}
|
||||
case map[string]any:
|
||||
out := make(map[string]any, len(inputTyped))
|
||||
for k, v := range inputTyped {
|
||||
out[k] = normalize(v)
|
||||
}
|
||||
return out
|
||||
case []any:
|
||||
out := make([]any, len(inputTyped))
|
||||
for i := range inputTyped {
|
||||
out[i] = normalize(inputTyped[i])
|
||||
}
|
||||
return out
|
||||
default:
|
||||
return input
|
||||
}
|
||||
}
|
||||
|
||||
type wrappedAssertInfo struct {
|
||||
A *assertInfo `json:"antithesis_assert"`
|
||||
}
|
||||
|
||||
// --------------------------------------------------------------------------------
|
||||
// Assertions
|
||||
// --------------------------------------------------------------------------------
|
||||
const (
|
||||
wasHit = true
|
||||
mustBeHit = true
|
||||
optionallyHit = false
|
||||
expectingTrue = true
|
||||
)
|
||||
|
||||
const (
|
||||
universalTest = "always"
|
||||
existentialTest = "sometimes"
|
||||
reachabilityTest = "reachability"
|
||||
)
|
||||
|
||||
const (
|
||||
alwaysDisplay = "Always"
|
||||
alwaysOrUnreachableDisplay = "AlwaysOrUnreachable"
|
||||
sometimesDisplay = "Sometimes"
|
||||
reachableDisplay = "Reachable"
|
||||
unreachableDisplay = "Unreachable"
|
||||
)
|
||||
|
||||
// Always asserts that condition is true every time this function is called, and that it is called at least once. The corresponding test property will be viewable in the Antithesis SDK: Always group of your triage report.
|
||||
func Always(condition bool, message string, details map[string]any) {
|
||||
locationInfo := newLocationInfo(offsetAPICaller)
|
||||
id := makeKey(message, locationInfo)
|
||||
assertImpl(condition, message, details, locationInfo, wasHit, mustBeHit, universalTest, alwaysDisplay, id)
|
||||
}
|
||||
|
||||
// AlwaysOrUnreachable asserts that condition is true every time this function is called. The corresponding test property will pass if the assertion is never encountered (unlike Always assertion types). This test property will be viewable in the “Antithesis SDK: Always” group of your triage report.
|
||||
func AlwaysOrUnreachable(condition bool, message string, details map[string]any) {
|
||||
locationInfo := newLocationInfo(offsetAPICaller)
|
||||
id := makeKey(message, locationInfo)
|
||||
assertImpl(condition, message, details, locationInfo, wasHit, optionallyHit, universalTest, alwaysOrUnreachableDisplay, id)
|
||||
}
|
||||
|
||||
// Sometimes asserts that condition is true at least one time that this function was called. (If the assertion is never encountered, the test property will therefore fail.) This test property will be viewable in the “Antithesis SDK: Sometimes” group.
|
||||
func Sometimes(condition bool, message string, details map[string]any) {
|
||||
locationInfo := newLocationInfo(offsetAPICaller)
|
||||
id := makeKey(message, locationInfo)
|
||||
assertImpl(condition, message, details, locationInfo, wasHit, mustBeHit, existentialTest, sometimesDisplay, id)
|
||||
}
|
||||
|
||||
// Unreachable asserts that a line of code is never reached. The corresponding test property will fail if this function is ever called. (If it is never called the test property will therefore pass.) This test property will be viewable in the “Antithesis SDK: Reachablity assertions” group.
|
||||
func Unreachable(message string, details map[string]any) {
|
||||
locationInfo := newLocationInfo(offsetAPICaller)
|
||||
id := makeKey(message, locationInfo)
|
||||
assertImpl(false, message, details, locationInfo, wasHit, optionallyHit, reachabilityTest, unreachableDisplay, id)
|
||||
}
|
||||
|
||||
// Reachable asserts that a line of code is reached at least once. The corresponding test property will pass if this function is ever called. (If it is never called the test property will therefore fail.) This test property will be viewable in the “Antithesis SDK: Reachablity assertions” group.
|
||||
func Reachable(message string, details map[string]any) {
|
||||
locationInfo := newLocationInfo(offsetAPICaller)
|
||||
id := makeKey(message, locationInfo)
|
||||
assertImpl(true, message, details, locationInfo, wasHit, mustBeHit, reachabilityTest, reachableDisplay, id)
|
||||
}
|
||||
|
||||
// AssertRaw is a low-level method designed to be used by third-party frameworks. Regular users of the assert package should not call it.
|
||||
func AssertRaw(cond bool, message string, details map[string]any,
|
||||
classname, funcname, filename string, line int,
|
||||
hit bool, mustHit bool,
|
||||
assertType string, displayType string,
|
||||
id string,
|
||||
) {
|
||||
assertImpl(cond, message, details,
|
||||
&locationInfo{classname, funcname, filename, line, columnUnknown},
|
||||
hit, mustHit,
|
||||
assertType, displayType,
|
||||
id)
|
||||
}
|
||||
|
||||
func assertImpl(cond bool, message string, details map[string]any,
|
||||
loc *locationInfo,
|
||||
hit bool, mustHit bool,
|
||||
assertType string, displayType string,
|
||||
id string,
|
||||
) {
|
||||
trackerEntry := assertTracker.getTrackerEntry(id, loc.Filename, loc.Classname)
|
||||
|
||||
// Always grab the Filename and Classname captured when the trackerEntry was established
|
||||
// This provides the consistency needed between instrumentation-time and runtime
|
||||
if loc.Filename != trackerEntry.Filename {
|
||||
loc.Filename = trackerEntry.Filename
|
||||
}
|
||||
|
||||
if loc.Classname != trackerEntry.Classname {
|
||||
loc.Classname = trackerEntry.Classname
|
||||
}
|
||||
|
||||
aI := &assertInfo{
|
||||
Hit: hit,
|
||||
MustHit: mustHit,
|
||||
AssertType: assertType,
|
||||
DisplayType: displayType,
|
||||
Message: message,
|
||||
Condition: cond,
|
||||
Id: id,
|
||||
Location: loc,
|
||||
Details: details,
|
||||
}
|
||||
|
||||
trackerEntry.emit(aI)
|
||||
}
|
||||
|
||||
func makeKey(message string, _ *locationInfo) string {
|
||||
return message
|
||||
}
|
||||
-16
@@ -1,16 +0,0 @@
|
||||
//go:build !enable_antithesis_sdk
|
||||
|
||||
package assert
|
||||
|
||||
func Always(condition bool, message string, details map[string]any) {}
|
||||
func AlwaysOrUnreachable(condition bool, message string, details map[string]any) {}
|
||||
func Sometimes(condition bool, message string, details map[string]any) {}
|
||||
func Unreachable(message string, details map[string]any) {}
|
||||
func Reachable(message string, details map[string]any) {}
|
||||
func AssertRaw(cond bool, message string, details map[string]any,
|
||||
classname, funcname, filename string, line int,
|
||||
hit bool, mustHit bool,
|
||||
assertType string, displayType string,
|
||||
id string,
|
||||
) {
|
||||
}
|
||||
-30
@@ -1,30 +0,0 @@
|
||||
package assert
|
||||
|
||||
// Allowable numeric types of comparison parameters
|
||||
type Number interface {
|
||||
~int | ~int8 | ~int16 | ~int32 | ~int64 | ~uint8 | ~uint16 | ~uint32 | ~float32 | ~float64 | ~uint64 | ~uint | ~uintptr
|
||||
}
|
||||
|
||||
// Internally, numeric guidanceFn Operands only use these
|
||||
type operandConstraint interface {
|
||||
int32 | int64 | uint64 | float64
|
||||
}
|
||||
|
||||
type numConstraint interface {
|
||||
uint64 | float64
|
||||
}
|
||||
|
||||
// Used for boolean assertions
|
||||
type NamedBool struct {
|
||||
First string `json:"first"`
|
||||
Second bool `json:"second"`
|
||||
}
|
||||
|
||||
// Convenience function to construct a NamedBool used for boolean assertions
|
||||
func NewNamedBool(first string, second bool) *NamedBool {
|
||||
p := NamedBool{
|
||||
First: first,
|
||||
Second: second,
|
||||
}
|
||||
return &p
|
||||
}
|
||||
-66
@@ -1,66 +0,0 @@
|
||||
//go:build enable_antithesis_sdk
|
||||
|
||||
package assert
|
||||
|
||||
import (
|
||||
"sync"
|
||||
|
||||
"github.com/antithesishq/antithesis-sdk-go/internal"
|
||||
)
|
||||
|
||||
// TODO: Tracker is intended to prevent sending the same guidance
|
||||
// more than once. In this case, we always send, so the tracker
|
||||
// is not presently used.
|
||||
type booleanGuidance struct {
|
||||
n int
|
||||
}
|
||||
|
||||
type booleanGuidanceTracker map[string]*booleanGuidance
|
||||
|
||||
var (
|
||||
boolean_guidance_tracker booleanGuidanceTracker = make(booleanGuidanceTracker)
|
||||
boolean_guidance_tracker_mutex sync.Mutex
|
||||
boolean_guidance_info_mutex sync.Mutex
|
||||
)
|
||||
|
||||
func (tracker booleanGuidanceTracker) getTrackerEntry(messageKey string) *booleanGuidance {
|
||||
var trackerEntry *booleanGuidance
|
||||
var ok bool
|
||||
|
||||
if tracker == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
boolean_guidance_tracker_mutex.Lock()
|
||||
defer boolean_guidance_tracker_mutex.Unlock()
|
||||
if trackerEntry, ok = boolean_guidance_tracker[messageKey]; !ok {
|
||||
trackerEntry = newBooleanGuidance()
|
||||
tracker[messageKey] = trackerEntry
|
||||
}
|
||||
|
||||
return trackerEntry
|
||||
}
|
||||
|
||||
// Create a boolean guidance tracker
|
||||
func newBooleanGuidance() *booleanGuidance {
|
||||
trackerInfo := booleanGuidance{}
|
||||
return &trackerInfo
|
||||
}
|
||||
|
||||
func (tI *booleanGuidance) send_value(bgI *booleanGuidanceInfo) {
|
||||
if tI == nil {
|
||||
return
|
||||
}
|
||||
|
||||
boolean_guidance_info_mutex.Lock()
|
||||
defer boolean_guidance_info_mutex.Unlock()
|
||||
|
||||
// The tracker entry should be consulted to determine
|
||||
// if this Guidance info has already been sent, or not.
|
||||
|
||||
emitBooleanGuidance(bgI)
|
||||
}
|
||||
|
||||
func emitBooleanGuidance(bgI *booleanGuidanceInfo) error {
|
||||
return internal.Json_data(map[string]any{"antithesis_guidance": bgI})
|
||||
}
|
||||
-58
@@ -1,58 +0,0 @@
|
||||
//go:build enable_antithesis_sdk
|
||||
|
||||
package assert
|
||||
|
||||
import (
|
||||
"path"
|
||||
"runtime"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// stackFrameOffset indicates how many frames to go up in the
|
||||
// call stack to find the filename/location/line info. As
|
||||
// this work is always done in NewLocationInfo(), the offset is
|
||||
// specified from the perspective of NewLocationInfo
|
||||
type stackFrameOffset int
|
||||
|
||||
// Order is important here since iota is being used
|
||||
const (
|
||||
offsetNewLocationInfo stackFrameOffset = iota
|
||||
offsetHere
|
||||
offsetAPICaller
|
||||
offsetAPICallersCaller
|
||||
)
|
||||
|
||||
// locationInfo represents the attributes known at instrumentation time
|
||||
// for each Antithesis assertion discovered
|
||||
type locationInfo struct {
|
||||
Classname string `json:"class"`
|
||||
Funcname string `json:"function"`
|
||||
Filename string `json:"file"`
|
||||
Line int `json:"begin_line"`
|
||||
Column int `json:"begin_column"`
|
||||
}
|
||||
|
||||
// columnUnknown is used when the column associated with
|
||||
// a locationInfo is not available
|
||||
const columnUnknown = 0
|
||||
|
||||
// NewLocationInfo creates a locationInfo directly from
|
||||
// the current execution context
|
||||
func newLocationInfo(nframes stackFrameOffset) *locationInfo {
|
||||
// Get location info and add to details
|
||||
funcname := "*function*"
|
||||
classname := "*class*"
|
||||
pc, filename, line, ok := runtime.Caller(int(nframes))
|
||||
if !ok {
|
||||
filename = "*file*"
|
||||
line = 0
|
||||
} else {
|
||||
if this_func := runtime.FuncForPC(pc); this_func != nil {
|
||||
fullname := this_func.Name()
|
||||
funcname = path.Ext(fullname)
|
||||
classname, _ = strings.CutSuffix(fullname, funcname)
|
||||
funcname = funcname[1:]
|
||||
}
|
||||
}
|
||||
return &locationInfo{classname, funcname, filename, line, columnUnknown}
|
||||
}
|
||||
-323
@@ -1,323 +0,0 @@
|
||||
//go:build enable_antithesis_sdk
|
||||
|
||||
package assert
|
||||
|
||||
import (
|
||||
"math"
|
||||
"sync"
|
||||
|
||||
"github.com/antithesishq/antithesis-sdk-go/internal"
|
||||
)
|
||||
|
||||
// --------------------------------------------------------------------------------
|
||||
// IntegerGap is used for:
|
||||
// - int, int8, int16, int32, int64:
|
||||
// - uint, uint8, uint16, uint32, uint64, uintptr:
|
||||
//
|
||||
// FloatGap is used for:
|
||||
// - float32, float64
|
||||
// --------------------------------------------------------------------------------
|
||||
type numericGapType int
|
||||
|
||||
const (
|
||||
integerGapType numericGapType = iota
|
||||
floatGapType
|
||||
)
|
||||
|
||||
func gapTypeForOperand[T Number](num T) numericGapType {
|
||||
gapType := integerGapType
|
||||
|
||||
switch any(num).(type) {
|
||||
case float32, float64:
|
||||
gapType = floatGapType
|
||||
}
|
||||
return gapType
|
||||
}
|
||||
|
||||
// --------------------------------------------------------------------------------
|
||||
// numericGuidanceTracker - Tracking Info for Numeric Guidance
|
||||
//
|
||||
// For GuidanceFnMaximize:
|
||||
// - gap is the largest value sent so far
|
||||
//
|
||||
// For GuidanceFnMinimize:
|
||||
// - gap is the most negative value sent so far
|
||||
//
|
||||
// --------------------------------------------------------------------------------
|
||||
type numericGuidanceInfo struct {
|
||||
gap any
|
||||
descriminator numericGapType
|
||||
maximize bool
|
||||
}
|
||||
|
||||
type numericGuidanceTracker map[string]*numericGuidanceInfo
|
||||
|
||||
var (
|
||||
numeric_guidance_tracker numericGuidanceTracker = make(numericGuidanceTracker)
|
||||
numeric_guidance_tracker_mutex sync.Mutex
|
||||
numeric_guidance_info_mutex sync.Mutex
|
||||
)
|
||||
|
||||
func (tracker numericGuidanceTracker) getTrackerEntry(messageKey string, trackerType numericGapType, maximize bool) *numericGuidanceInfo {
|
||||
var trackerEntry *numericGuidanceInfo
|
||||
var ok bool
|
||||
|
||||
if tracker == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
numeric_guidance_tracker_mutex.Lock()
|
||||
defer numeric_guidance_tracker_mutex.Unlock()
|
||||
if trackerEntry, ok = numeric_guidance_tracker[messageKey]; !ok {
|
||||
trackerEntry = newNumericGuidanceInfo(trackerType, maximize)
|
||||
tracker[messageKey] = trackerEntry
|
||||
}
|
||||
|
||||
return trackerEntry
|
||||
}
|
||||
|
||||
// Create an numeric guidance entry
|
||||
func newNumericGuidanceInfo(trackerType numericGapType, maximize bool) *numericGuidanceInfo {
|
||||
|
||||
var gap any
|
||||
if trackerType == integerGapType {
|
||||
gap = newGapValue(uint64(math.MaxUint64), maximize)
|
||||
} else {
|
||||
gap = newGapValue(float64(math.MaxFloat64), maximize)
|
||||
}
|
||||
trackerInfo := numericGuidanceInfo{
|
||||
maximize: maximize,
|
||||
descriminator: trackerType,
|
||||
gap: gap,
|
||||
}
|
||||
return &trackerInfo
|
||||
}
|
||||
|
||||
func (tI *numericGuidanceInfo) should_maximize() bool {
|
||||
return tI.maximize
|
||||
}
|
||||
|
||||
func (tI *numericGuidanceInfo) is_integer_gap() bool {
|
||||
return tI.descriminator == integerGapType
|
||||
}
|
||||
|
||||
// --------------------------------------------------------------------------------
|
||||
// Represents integral and floating point extremes
|
||||
// --------------------------------------------------------------------------------
|
||||
type gapValue[T numConstraint] struct {
|
||||
gap_size T
|
||||
gap_is_negative bool
|
||||
}
|
||||
|
||||
func newGapValue[T numConstraint](sz T, is_neg bool) any {
|
||||
switch any(sz).(type) {
|
||||
case uint64:
|
||||
return gapValue[uint64]{gap_size: uint64(sz), gap_is_negative: is_neg}
|
||||
|
||||
case float64:
|
||||
return gapValue[float64]{gap_size: float64(sz), gap_is_negative: is_neg}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func is_same_sign(left_val int64, right_val int64) bool {
|
||||
same_sign := false
|
||||
if left_val < 0 {
|
||||
// left is negative
|
||||
if right_val < 0 {
|
||||
same_sign = true
|
||||
}
|
||||
} else {
|
||||
// left is non-negative
|
||||
if right_val >= 0 {
|
||||
same_sign = true
|
||||
}
|
||||
}
|
||||
return same_sign
|
||||
}
|
||||
|
||||
func abs_int64(val int64) uint64 {
|
||||
if val >= 0 {
|
||||
return uint64(val)
|
||||
}
|
||||
return uint64(0 - val)
|
||||
}
|
||||
|
||||
func is_greater_than[T numConstraint](left gapValue[T], right gapValue[T]) bool {
|
||||
if !left.gap_is_negative && !right.gap_is_negative {
|
||||
return left.gap_size > right.gap_size
|
||||
}
|
||||
if !left.gap_is_negative && right.gap_is_negative {
|
||||
return true // any positive is greater than a negative
|
||||
}
|
||||
if left.gap_is_negative && right.gap_is_negative {
|
||||
return right.gap_size > left.gap_size
|
||||
}
|
||||
if left.gap_is_negative && !right.gap_is_negative {
|
||||
return false // any negative is less than a positive
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func is_less_than[T numConstraint](left gapValue[T], right gapValue[T]) bool {
|
||||
if !left.gap_is_negative && !right.gap_is_negative {
|
||||
return left.gap_size < right.gap_size
|
||||
}
|
||||
if !left.gap_is_negative && right.gap_is_negative {
|
||||
return false // any positive is greater than a negative
|
||||
}
|
||||
if left.gap_is_negative && right.gap_is_negative {
|
||||
return right.gap_size < left.gap_size
|
||||
}
|
||||
if left.gap_is_negative && !right.gap_is_negative {
|
||||
return true // any negative is less than a positive
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func send_value_if_needed(tI *numericGuidanceInfo, gI *guidanceInfo) {
|
||||
if tI == nil {
|
||||
return
|
||||
}
|
||||
|
||||
numeric_guidance_info_mutex.Lock()
|
||||
defer numeric_guidance_info_mutex.Unlock()
|
||||
|
||||
// if this is a catalog entry (gI.hit is false)
|
||||
// do not update the reference gap in the tracker (tI *numericGuidanceInfo)
|
||||
if !gI.Hit {
|
||||
emitGuidance(gI)
|
||||
return
|
||||
}
|
||||
|
||||
should_send := false
|
||||
maximize := tI.should_maximize()
|
||||
|
||||
var gap gapValue[uint64]
|
||||
var float_gap gapValue[float64]
|
||||
|
||||
// Needs to have individual case statements to assist
|
||||
// the compiler to infer the actual type of the var named 'operands'
|
||||
switch operands := (gI.Data).(type) {
|
||||
case numericOperands[int32]:
|
||||
gap = makeGap(operands)
|
||||
case numericOperands[int64]:
|
||||
gap = makeGap(operands)
|
||||
case numericOperands[uint64]:
|
||||
gap = makeGap(operands)
|
||||
case numericOperands[float64]:
|
||||
float_gap = makeFloatGap(operands)
|
||||
}
|
||||
|
||||
var prev_gap gapValue[uint64]
|
||||
var prev_float_gap gapValue[float64]
|
||||
has_prev_gap := false
|
||||
has_prev_float_gap := false
|
||||
|
||||
prev_gap, has_prev_gap = tI.gap.(gapValue[uint64])
|
||||
if !has_prev_gap {
|
||||
prev_float_gap, has_prev_float_gap = tI.gap.(gapValue[float64])
|
||||
}
|
||||
|
||||
if has_prev_gap {
|
||||
if maximize {
|
||||
should_send = is_greater_than(gap, prev_gap)
|
||||
} else {
|
||||
should_send = is_less_than(gap, prev_gap)
|
||||
}
|
||||
}
|
||||
|
||||
if has_prev_float_gap {
|
||||
if maximize {
|
||||
should_send = is_greater_than(float_gap, prev_float_gap)
|
||||
} else {
|
||||
should_send = is_less_than(float_gap, prev_float_gap)
|
||||
}
|
||||
}
|
||||
|
||||
if should_send {
|
||||
if tI.is_integer_gap() {
|
||||
tI.gap = gap
|
||||
} else {
|
||||
tI.gap = float_gap
|
||||
}
|
||||
emitGuidance(gI)
|
||||
}
|
||||
}
|
||||
|
||||
func emitGuidance(gI *guidanceInfo) error {
|
||||
return internal.Json_data(map[string]any{"antithesis_guidance": gI})
|
||||
}
|
||||
|
||||
// When left and right are the same sign (both negative, or both non-negative)
|
||||
// Calculate: <result> = (left - right). The gap_size is abs(<result>) and
|
||||
// gap_is_negative is (right > left)
|
||||
func makeGap[Op operandConstraint](operand numericOperands[Op]) gapValue[uint64] {
|
||||
|
||||
var gap_size uint64
|
||||
var gap_is_negative bool
|
||||
|
||||
switch this_op := any(operand).(type) {
|
||||
case numericOperands[int32]:
|
||||
result := int64(this_op.Left) - int64(this_op.Right)
|
||||
gap_size = abs_int64(result)
|
||||
gap_is_negative = result < 0
|
||||
|
||||
case numericOperands[int64]:
|
||||
if is_same_sign(this_op.Left, this_op.Right) {
|
||||
result := int64(this_op.Left) - int64(this_op.Right)
|
||||
gap_size = abs_int64(result)
|
||||
gap_is_negative = result < 0
|
||||
break
|
||||
}
|
||||
|
||||
// Otherwise left and right are opposite signs
|
||||
// gap = abs(left) + abs(right)
|
||||
// gap_is_negative = left < right
|
||||
left_gap_size := abs_int64(this_op.Left)
|
||||
right_gap_size := abs_int64(this_op.Right)
|
||||
gap_size = left_gap_size + right_gap_size
|
||||
gap_is_negative = this_op.Left < this_op.Right
|
||||
|
||||
case numericOperands[uint64]:
|
||||
left_val := this_op.Left
|
||||
right_val := this_op.Right
|
||||
gap_is_negative = false
|
||||
if left_val < right_val {
|
||||
gap_is_negative = true
|
||||
gap_size = right_val - left_val
|
||||
} else {
|
||||
gap_size = left_val - right_val
|
||||
}
|
||||
|
||||
default:
|
||||
zero_gap, _ := newGapValue(uint64(0), false).(gapValue[uint64])
|
||||
return zero_gap
|
||||
}
|
||||
|
||||
this_gap, _ := newGapValue(gap_size, gap_is_negative).(gapValue[uint64])
|
||||
return this_gap
|
||||
} // MakeGap
|
||||
|
||||
func makeFloatGap[Op operandConstraint](operand numericOperands[Op]) gapValue[float64] {
|
||||
switch this_op := any(operand).(type) {
|
||||
case numericOperands[float64]:
|
||||
left_val := this_op.Left
|
||||
right_val := this_op.Right
|
||||
gap_is_negative := false
|
||||
var gap_size float64
|
||||
if left_val < right_val {
|
||||
gap_is_negative = true
|
||||
gap_size = right_val - left_val
|
||||
} else {
|
||||
gap_size = left_val - right_val
|
||||
}
|
||||
|
||||
this_gap, _ := newGapValue(gap_size, gap_is_negative).(gapValue[float64])
|
||||
return this_gap
|
||||
|
||||
default:
|
||||
zero_gap, _ := newGapValue(float64(0.0), false).(gapValue[float64])
|
||||
return zero_gap
|
||||
}
|
||||
} // MakeFloatGap
|
||||
-330
@@ -1,330 +0,0 @@
|
||||
//go:build enable_antithesis_sdk
|
||||
|
||||
package assert
|
||||
|
||||
// A type for writing raw assertions.
|
||||
// guidanceFnType allows the assertion to provide guidance to
|
||||
// the Antithesis platform when testing in Antithesis.
|
||||
// Regular users of the assert package should not use it.
|
||||
type guidanceFnType int
|
||||
|
||||
const (
|
||||
guidanceFnMaximize guidanceFnType = iota // Maximize (left - right) values
|
||||
guidanceFnMinimize // Minimize (left - right) values
|
||||
guidanceFnWantAll // Encourages fuzzing explorations where boolean values are true
|
||||
guidanceFnWantNone // Encourages fuzzing explorations where boolean values are false
|
||||
guidanceFnExplore
|
||||
)
|
||||
|
||||
// guidanceFnExplore
|
||||
|
||||
func get_guidance_type_string(gt guidanceFnType) string {
|
||||
switch gt {
|
||||
case guidanceFnMaximize, guidanceFnMinimize:
|
||||
return "numeric"
|
||||
case guidanceFnWantAll, guidanceFnWantNone:
|
||||
return "boolean"
|
||||
case guidanceFnExplore:
|
||||
return "json"
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
type numericOperands[T operandConstraint] struct {
|
||||
Left T `json:"left"`
|
||||
Right T `json:"right"`
|
||||
}
|
||||
|
||||
type guidanceInfo struct {
|
||||
Data any `json:"guidance_data,omitempty"`
|
||||
Location *locationInfo `json:"location"`
|
||||
GuidanceType string `json:"guidance_type"`
|
||||
Message string `json:"message"`
|
||||
Id string `json:"id"`
|
||||
Maximize bool `json:"maximize"`
|
||||
Hit bool `json:"hit"`
|
||||
}
|
||||
|
||||
type booleanGuidanceInfo struct {
|
||||
Data any `json:"guidance_data,omitempty"`
|
||||
Location *locationInfo `json:"location"`
|
||||
GuidanceType string `json:"guidance_type"`
|
||||
Message string `json:"message"`
|
||||
Id string `json:"id"`
|
||||
Maximize bool `json:"maximize"`
|
||||
Hit bool `json:"hit"`
|
||||
}
|
||||
|
||||
func uses_maximize(gt guidanceFnType) bool {
|
||||
return gt == guidanceFnMaximize || gt == guidanceFnWantAll
|
||||
}
|
||||
|
||||
func newOperands[T Number](left, right T) any {
|
||||
switch any(left).(type) {
|
||||
case int8, int16, int32:
|
||||
return numericOperands[int32]{int32(left), int32(right)}
|
||||
case int, int64:
|
||||
return numericOperands[int64]{int64(left), int64(right)}
|
||||
case uint8, uint16, uint32, uint, uint64, uintptr:
|
||||
return numericOperands[uint64]{uint64(left), uint64(right)}
|
||||
case float32, float64:
|
||||
return numericOperands[float64]{float64(left), float64(right)}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func build_numeric_guidance[T Number](gt guidanceFnType, message string, left, right T, loc *locationInfo, id string, hit bool) *guidanceInfo {
|
||||
|
||||
operands := newOperands(left, right)
|
||||
if !hit {
|
||||
operands = nil
|
||||
}
|
||||
|
||||
gI := guidanceInfo{
|
||||
GuidanceType: get_guidance_type_string(gt),
|
||||
Message: message,
|
||||
Id: id,
|
||||
Location: loc,
|
||||
Maximize: uses_maximize(gt),
|
||||
Data: operands,
|
||||
Hit: hit,
|
||||
}
|
||||
|
||||
return &gI
|
||||
}
|
||||
|
||||
type namedBoolDictionary map[string]bool
|
||||
|
||||
func build_boolean_guidance(gt guidanceFnType, message string, named_bools []NamedBool,
|
||||
loc *locationInfo,
|
||||
id string, hit bool) *booleanGuidanceInfo {
|
||||
|
||||
var guidance_data any
|
||||
|
||||
// To ensure the sequence and naming for the named_bool values
|
||||
if hit {
|
||||
named_bool_dictionary := namedBoolDictionary{}
|
||||
for _, named_bool := range named_bools {
|
||||
named_bool_dictionary[named_bool.First] = named_bool.Second
|
||||
}
|
||||
guidance_data = named_bool_dictionary
|
||||
}
|
||||
|
||||
bgI := booleanGuidanceInfo{
|
||||
GuidanceType: get_guidance_type_string(gt),
|
||||
Message: message,
|
||||
Id: id,
|
||||
Location: loc,
|
||||
Maximize: uses_maximize(gt),
|
||||
Data: guidance_data,
|
||||
Hit: hit,
|
||||
}
|
||||
|
||||
return &bgI
|
||||
}
|
||||
|
||||
func behavior_to_guidance(behavior string) guidanceFnType {
|
||||
guidance := guidanceFnExplore
|
||||
switch behavior {
|
||||
case "maximize":
|
||||
guidance = guidanceFnMaximize
|
||||
case "minimize":
|
||||
guidance = guidanceFnMinimize
|
||||
case "all":
|
||||
guidance = guidanceFnWantAll
|
||||
case "none":
|
||||
guidance = guidanceFnWantNone
|
||||
}
|
||||
return guidance
|
||||
}
|
||||
|
||||
func numericGuidanceImpl[T Number](left, right T, message, id string, loc *locationInfo, guidanceFn guidanceFnType, hit bool) {
|
||||
tI := numeric_guidance_tracker.getTrackerEntry(id, gapTypeForOperand(left), uses_maximize(guidanceFn))
|
||||
gI := build_numeric_guidance(guidanceFn, message, left, right, loc, id, hit)
|
||||
send_value_if_needed(tI, gI)
|
||||
}
|
||||
|
||||
func booleanGuidanceImpl(named_bools []NamedBool, message, id string, loc *locationInfo, guidanceFn guidanceFnType, hit bool) {
|
||||
tI := boolean_guidance_tracker.getTrackerEntry(id)
|
||||
bgI := build_boolean_guidance(guidanceFn, message, named_bools, loc, id, hit)
|
||||
tI.send_value(bgI)
|
||||
}
|
||||
|
||||
// NumericGuidanceRaw is a low-level method designed to be used by third-party frameworks. Regular users of the assert package should not call it.
|
||||
func NumericGuidanceRaw[T Number](
|
||||
left, right T,
|
||||
message, id string,
|
||||
classname, funcname, filename string,
|
||||
line int,
|
||||
behavior string,
|
||||
hit bool,
|
||||
) {
|
||||
loc := &locationInfo{classname, funcname, filename, line, columnUnknown}
|
||||
guidanceFn := behavior_to_guidance(behavior)
|
||||
numericGuidanceImpl(left, right, message, id, loc, guidanceFn, hit)
|
||||
}
|
||||
|
||||
// BooleanGuidanceRaw is a low-level method designed to be used by third-party frameworks. Regular users of the assert package should not call it.
|
||||
func BooleanGuidanceRaw(
|
||||
named_bools []NamedBool,
|
||||
message, id string,
|
||||
classname, funcname, filename string,
|
||||
line int,
|
||||
behavior string,
|
||||
hit bool,
|
||||
) {
|
||||
loc := &locationInfo{classname, funcname, filename, line, columnUnknown}
|
||||
guidanceFn := behavior_to_guidance(behavior)
|
||||
booleanGuidanceImpl(named_bools, message, id, loc, guidanceFn, hit)
|
||||
}
|
||||
|
||||
func add_numeric_details[T Number](details map[string]any, left, right T) map[string]any {
|
||||
// ----------------------------------------------------
|
||||
// Can not use maps.Clone() until go 1.21.0 or above
|
||||
// enhancedDetails := maps.Clone(details)
|
||||
// ----------------------------------------------------
|
||||
enhancedDetails := map[string]any{}
|
||||
for k, v := range details {
|
||||
enhancedDetails[k] = v
|
||||
}
|
||||
enhancedDetails["left"] = left
|
||||
enhancedDetails["right"] = right
|
||||
return enhancedDetails
|
||||
}
|
||||
|
||||
func add_boolean_details(details map[string]any, named_bools []NamedBool) map[string]any {
|
||||
// ----------------------------------------------------
|
||||
// Can not use maps.Clone() until go 1.21.0 or above
|
||||
// enhancedDetails := maps.Clone(details)
|
||||
// ----------------------------------------------------
|
||||
enhancedDetails := map[string]any{}
|
||||
for k, v := range details {
|
||||
enhancedDetails[k] = v
|
||||
}
|
||||
for _, named_bool := range named_bools {
|
||||
enhancedDetails[named_bool.First] = named_bool.Second
|
||||
}
|
||||
return enhancedDetails
|
||||
}
|
||||
|
||||
// Equivalent to asserting Always(left > right, message, details). Information about left and right will automatically be added to the details parameter, with keys left and right. If you use this function for assertions that compare numeric quantities, you may help Antithesis find more bugs.
|
||||
func AlwaysGreaterThan[T Number](left, right T, message string, details map[string]any) {
|
||||
loc := newLocationInfo(offsetAPICaller)
|
||||
id := makeKey(message, loc)
|
||||
condition := left > right
|
||||
all_details := add_numeric_details(details, left, right)
|
||||
assertImpl(condition, message, all_details, loc, wasHit, mustBeHit, universalTest, alwaysDisplay, id)
|
||||
|
||||
numericGuidanceImpl(left, right, message, id, loc, guidanceFnMinimize, wasHit)
|
||||
}
|
||||
|
||||
// Equivalent to asserting Always(left >= right, message, details). Information about left and right will automatically be added to the details parameter, with keys left and right. If you use this function for assertions that compare numeric quantities, you may help Antithesis find more bugs.
|
||||
func AlwaysGreaterThanOrEqualTo[T Number](left, right T, message string, details map[string]any) {
|
||||
loc := newLocationInfo(offsetAPICaller)
|
||||
id := makeKey(message, loc)
|
||||
condition := left >= right
|
||||
all_details := add_numeric_details(details, left, right)
|
||||
assertImpl(condition, message, all_details, loc, wasHit, mustBeHit, universalTest, alwaysDisplay, id)
|
||||
|
||||
numericGuidanceImpl(left, right, message, id, loc, guidanceFnMinimize, wasHit)
|
||||
}
|
||||
|
||||
// Equivalent to asserting Sometimes(T left > T right, message, details). Information about left and right will automatically be added to the details parameter, with keys left and right. If you use this function for assertions that compare numeric quantities, you may help Antithesis find more bugs.
|
||||
func SometimesGreaterThan[T Number](left, right T, message string, details map[string]any) {
|
||||
loc := newLocationInfo(offsetAPICaller)
|
||||
id := makeKey(message, loc)
|
||||
condition := left > right
|
||||
all_details := add_numeric_details(details, left, right)
|
||||
assertImpl(condition, message, all_details, loc, wasHit, mustBeHit, existentialTest, sometimesDisplay, id)
|
||||
|
||||
numericGuidanceImpl(left, right, message, id, loc, guidanceFnMaximize, wasHit)
|
||||
}
|
||||
|
||||
// Equivalent to asserting Sometimes(T left >= T right, message, details). Information about left and right will automatically be added to the details parameter, with keys left and right. If you use this function for assertions that compare numeric quantities, you may help Antithesis find more bugs.
|
||||
func SometimesGreaterThanOrEqualTo[T Number](left, right T, message string, details map[string]any) {
|
||||
loc := newLocationInfo(offsetAPICaller)
|
||||
id := makeKey(message, loc)
|
||||
condition := left >= right
|
||||
all_details := add_numeric_details(details, left, right)
|
||||
assertImpl(condition, message, all_details, loc, wasHit, mustBeHit, existentialTest, sometimesDisplay, id)
|
||||
|
||||
numericGuidanceImpl(left, right, message, id, loc, guidanceFnMaximize, wasHit)
|
||||
}
|
||||
|
||||
// Equivalent to asserting Always(left < right, message, details). Information about left and right will automatically be added to the details parameter, with keys left and right. If you use this function for assertions that compare numeric quantities, you may help Antithesis find more bugs.
|
||||
func AlwaysLessThan[T Number](left, right T, message string, details map[string]any) {
|
||||
loc := newLocationInfo(offsetAPICaller)
|
||||
id := makeKey(message, loc)
|
||||
condition := left < right
|
||||
all_details := add_numeric_details(details, left, right)
|
||||
assertImpl(condition, message, all_details, loc, wasHit, mustBeHit, universalTest, alwaysDisplay, id)
|
||||
|
||||
numericGuidanceImpl(left, right, message, id, loc, guidanceFnMaximize, wasHit)
|
||||
}
|
||||
|
||||
// Equivalent to asserting Always(left <= right, message, details). Information about left and right will automatically be added to the details parameter, with keys left and right. If you use this function for assertions that compare numeric quantities, you may help Antithesis find more bugs.
|
||||
func AlwaysLessThanOrEqualTo[T Number](left, right T, message string, details map[string]any) {
|
||||
loc := newLocationInfo(offsetAPICaller)
|
||||
id := makeKey(message, loc)
|
||||
condition := left <= right
|
||||
all_details := add_numeric_details(details, left, right)
|
||||
assertImpl(condition, message, all_details, loc, wasHit, mustBeHit, universalTest, alwaysDisplay, id)
|
||||
|
||||
numericGuidanceImpl(left, right, message, id, loc, guidanceFnMaximize, wasHit)
|
||||
}
|
||||
|
||||
// Equivalent to asserting Sometimes(T left < T right, message, details). Information about left and right will automatically be added to the details parameter, with keys left and right. If you use this function for assertions that compare numeric quantities, you may help Antithesis find more bugs.
|
||||
func SometimesLessThan[T Number](left, right T, message string, details map[string]any) {
|
||||
loc := newLocationInfo(offsetAPICaller)
|
||||
id := makeKey(message, loc)
|
||||
condition := left < right
|
||||
all_details := add_numeric_details(details, left, right)
|
||||
assertImpl(condition, message, all_details, loc, wasHit, mustBeHit, existentialTest, sometimesDisplay, id)
|
||||
|
||||
numericGuidanceImpl(left, right, message, id, loc, guidanceFnMinimize, wasHit)
|
||||
}
|
||||
|
||||
// Equivalent to asserting Sometimes(T left <= T right, message, details). Information about left and right will automatically be added to the details parameter, with keys left and right. If you use this function for assertions that compare numeric quantities, you may help Antithesis find more bugs.
|
||||
func SometimesLessThanOrEqualTo[T Number](left, right T, message string, details map[string]any) {
|
||||
loc := newLocationInfo(offsetAPICaller)
|
||||
id := makeKey(message, loc)
|
||||
condition := left <= right
|
||||
all_details := add_numeric_details(details, left, right)
|
||||
assertImpl(condition, message, all_details, loc, wasHit, mustBeHit, existentialTest, sometimesDisplay, id)
|
||||
|
||||
numericGuidanceImpl(left, right, message, id, loc, guidanceFnMinimize, wasHit)
|
||||
}
|
||||
|
||||
// Asserts that every time this is called, at least one bool in named_bools is true. Equivalent to Always(named_bools[0].second || named_bools[1].second || ..., message, details). If you use this for assertions about the behavior of booleans, you may help Antithesis find more bugs. Information about named_bools will automatically be added to the details parameter, and the keys will be the names of the bools.
|
||||
func AlwaysSome(named_bools []NamedBool, message string, details map[string]any) {
|
||||
loc := newLocationInfo(offsetAPICaller)
|
||||
id := makeKey(message, loc)
|
||||
disjunction := false
|
||||
for _, named_bool := range named_bools {
|
||||
if named_bool.Second {
|
||||
disjunction = true
|
||||
break
|
||||
}
|
||||
}
|
||||
all_details := add_boolean_details(details, named_bools)
|
||||
assertImpl(disjunction, message, all_details, loc, wasHit, mustBeHit, universalTest, alwaysDisplay, id)
|
||||
|
||||
booleanGuidanceImpl(named_bools, message, id, loc, guidanceFnWantNone, wasHit)
|
||||
}
|
||||
|
||||
// Asserts that at least one time this is called, every bool in named_bools is true. Equivalent to Sometimes(named_bools[0].second && named_bools[1].second && ..., message, details). If you use this for assertions about the behavior of booleans, you may help Antithesis find more bugs. Information about named_bools will automatically be added to the details parameter, and the keys will be the names of the bools.
|
||||
func SometimesAll(named_bools []NamedBool, message string, details map[string]any) {
|
||||
loc := newLocationInfo(offsetAPICaller)
|
||||
id := makeKey(message, loc)
|
||||
conjunction := true
|
||||
for _, named_bool := range named_bools {
|
||||
if !named_bool.Second {
|
||||
conjunction = false
|
||||
break
|
||||
}
|
||||
}
|
||||
all_details := add_boolean_details(details, named_bools)
|
||||
assertImpl(conjunction, message, all_details, loc, wasHit, mustBeHit, existentialTest, sometimesDisplay, id)
|
||||
|
||||
booleanGuidanceImpl(named_bools, message, id, loc, guidanceFnWantAll, wasHit)
|
||||
}
|
||||
-34
@@ -1,34 +0,0 @@
|
||||
//go:build !enable_antithesis_sdk
|
||||
|
||||
package assert
|
||||
|
||||
func AlwaysGreaterThan[T Number](left, right T, message string, details map[string]any) {}
|
||||
func AlwaysGreaterThanOrEqualTo[T Number](left, right T, message string, details map[string]any) {}
|
||||
func SometimesGreaterThan[T Number](left, right T, message string, details map[string]any) {}
|
||||
func SometimesGreaterThanOrEqualTo[T Number](left, right T, message string, details map[string]any) {}
|
||||
func AlwaysLessThan[T Number](left, right T, message string, details map[string]any) {}
|
||||
func AlwaysLessThanOrEqualTo[T Number](left, right T, message string, details map[string]any) {}
|
||||
func SometimesLessThan[T Number](left, right T, message string, details map[string]any) {}
|
||||
func SometimesLessThanOrEqualTo[T Number](left, right T, message string, details map[string]any) {}
|
||||
|
||||
func AlwaysSome(named_bool []NamedBool, message string, details map[string]any) {}
|
||||
func SometimesAll(named_bool []NamedBool, message string, details map[string]any) {}
|
||||
|
||||
func NumericGuidanceRaw[T Number](left, right T,
|
||||
message, id string,
|
||||
classname, funcname, filename string,
|
||||
line int,
|
||||
behavior string,
|
||||
hit bool,
|
||||
) {
|
||||
}
|
||||
|
||||
func BooleanGuidanceRaw(
|
||||
named_bools []NamedBool,
|
||||
message, id string,
|
||||
classname, funcname, filename string,
|
||||
line int,
|
||||
behavior string,
|
||||
hit bool,
|
||||
) {
|
||||
}
|
||||
-111
@@ -1,111 +0,0 @@
|
||||
//go:build enable_antithesis_sdk
|
||||
|
||||
package assert
|
||||
|
||||
import (
|
||||
"runtime"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
|
||||
"github.com/antithesishq/antithesis-sdk-go/internal"
|
||||
)
|
||||
|
||||
type trackerInfo struct {
|
||||
Filename string
|
||||
Classname string
|
||||
PassCount int
|
||||
FailCount int
|
||||
}
|
||||
|
||||
type emitTracker map[string]*trackerInfo
|
||||
|
||||
// assert_tracker (global) keeps track of the unique asserts evaluated
|
||||
var (
|
||||
assertTracker emitTracker = make(emitTracker)
|
||||
trackerMutex sync.Mutex
|
||||
trackerInfoMutex sync.Mutex
|
||||
)
|
||||
|
||||
func (tracker emitTracker) getTrackerEntry(messageKey string, filename, classname string) *trackerInfo {
|
||||
var trackerEntry *trackerInfo
|
||||
var ok bool
|
||||
|
||||
if tracker == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
trackerMutex.Lock()
|
||||
defer trackerMutex.Unlock()
|
||||
if trackerEntry, ok = tracker[messageKey]; !ok {
|
||||
trackerEntry = newTrackerInfo(filename, classname)
|
||||
tracker[messageKey] = trackerEntry
|
||||
}
|
||||
return trackerEntry
|
||||
}
|
||||
|
||||
func newTrackerInfo(filename, classname string) *trackerInfo {
|
||||
trackerInfo := trackerInfo{
|
||||
PassCount: 0,
|
||||
FailCount: 0,
|
||||
Filename: filename,
|
||||
Classname: classname,
|
||||
}
|
||||
return &trackerInfo
|
||||
}
|
||||
|
||||
func (ti *trackerInfo) emit(ai *assertInfo) {
|
||||
if ti == nil || ai == nil {
|
||||
return
|
||||
}
|
||||
|
||||
// Registrations are just sent to voidstar
|
||||
hit := ai.Hit
|
||||
if !hit {
|
||||
emitAssert(ai)
|
||||
return
|
||||
}
|
||||
|
||||
var err error
|
||||
cond := ai.Condition
|
||||
|
||||
trackerInfoMutex.Lock()
|
||||
defer trackerInfoMutex.Unlock()
|
||||
if cond {
|
||||
if ti.PassCount == 0 {
|
||||
err = emitAssert(ai)
|
||||
}
|
||||
if err == nil {
|
||||
ti.PassCount++
|
||||
}
|
||||
return
|
||||
}
|
||||
if ti.FailCount == 0 {
|
||||
err = emitAssert(ai)
|
||||
}
|
||||
if err == nil {
|
||||
ti.FailCount++
|
||||
}
|
||||
}
|
||||
|
||||
func versionMessage() {
|
||||
languageBlock := map[string]any{
|
||||
"name": "Go",
|
||||
"version": runtime.Version(),
|
||||
}
|
||||
versionBlock := map[string]any{
|
||||
"language": languageBlock,
|
||||
"sdk_version": internal.SDK_Version,
|
||||
"protocol_version": internal.Protocol_Version,
|
||||
}
|
||||
internal.Json_data(map[string]any{"antithesis_sdk": versionBlock})
|
||||
}
|
||||
|
||||
// package-level flag
|
||||
var hasEmitted atomic.Bool // initialzed to false
|
||||
|
||||
func emitAssert(ai *assertInfo) error {
|
||||
if hasEmitted.CompareAndSwap(false, true) {
|
||||
versionMessage()
|
||||
}
|
||||
return internal.Json_data(wrappedAssertInfo{ai})
|
||||
}
|
||||
-100
@@ -1,100 +0,0 @@
|
||||
//go:build enable_antithesis_sdk
|
||||
|
||||
package internal
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"log"
|
||||
"math/rand"
|
||||
"os"
|
||||
)
|
||||
|
||||
func Json_data(v any) error {
|
||||
if data, err := json.Marshal(v); err != nil {
|
||||
return err
|
||||
} else {
|
||||
handler.output(string(data))
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
func Get_random() uint64 {
|
||||
return handler.random()
|
||||
}
|
||||
|
||||
func Notify(edge uint64) bool {
|
||||
return handler.notify(edge)
|
||||
}
|
||||
|
||||
func InitCoverage(num_edges uint64, symbols string) uint64 {
|
||||
return handler.init_coverage(num_edges, symbols)
|
||||
}
|
||||
|
||||
type libHandler interface {
|
||||
output(message string)
|
||||
random() uint64
|
||||
notify(edge uint64) bool
|
||||
init_coverage(num_edges uint64, symbols string) uint64
|
||||
}
|
||||
|
||||
const (
|
||||
errorLogLinePrefix = "[* antithesis-sdk-go *]"
|
||||
)
|
||||
|
||||
var handler libHandler
|
||||
|
||||
type localHandler struct {
|
||||
outputFile *os.File // can be nil
|
||||
}
|
||||
|
||||
func (h *localHandler) output(message string) {
|
||||
msg_len := len(message)
|
||||
if msg_len == 0 {
|
||||
return
|
||||
}
|
||||
if h.outputFile != nil {
|
||||
h.outputFile.WriteString(message + "\n")
|
||||
}
|
||||
}
|
||||
|
||||
func (h *localHandler) random() uint64 {
|
||||
return rand.Uint64()
|
||||
}
|
||||
|
||||
func (h *localHandler) notify(edge uint64) bool {
|
||||
return false
|
||||
}
|
||||
|
||||
func (h *localHandler) init_coverage(num_edges uint64, symbols string) uint64 {
|
||||
return 0
|
||||
}
|
||||
|
||||
func init() {
|
||||
handler = init_in_antithesis()
|
||||
if handler == nil {
|
||||
// Otherwise fallback to the local handler.
|
||||
handler = openLocalHandler()
|
||||
}
|
||||
}
|
||||
|
||||
// If `localOutputEnvVar` is set to a non-empty path, attempt to open that path and truncate the file
|
||||
// to serve as the log file of the local handler.
|
||||
// Otherwise, we don't have a log file, and logging is a no-op in the local handler.
|
||||
func openLocalHandler() *localHandler {
|
||||
path, is_set := os.LookupEnv(localOutputEnvVar)
|
||||
if !is_set || len(path) == 0 {
|
||||
return &localHandler{nil}
|
||||
}
|
||||
|
||||
// Open the file R/W (create if needed and possible)
|
||||
file, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE, 0644)
|
||||
if err != nil {
|
||||
log.Printf("%s Failed to open path %s: %v", errorLogLinePrefix, path, err)
|
||||
file = nil
|
||||
} else if err = file.Truncate(0); err != nil {
|
||||
log.Printf("%s Failed to truncate file at %s: %v", errorLogLinePrefix, path, err)
|
||||
file = nil
|
||||
}
|
||||
|
||||
return &localHandler{file}
|
||||
}
|
||||
-12
@@ -1,12 +0,0 @@
|
||||
package internal
|
||||
|
||||
// --------------------------------------------------------------------------------
|
||||
// Versions
|
||||
// --------------------------------------------------------------------------------
|
||||
const SDK_Version = "0.7.0"
|
||||
const Protocol_Version = "1.1.0"
|
||||
|
||||
// --------------------------------------------------------------------------------
|
||||
// Environment Vars
|
||||
// --------------------------------------------------------------------------------
|
||||
const localOutputEnvVar = "ANTITHESIS_SDK_LOCAL_OUTPUT"
|
||||
-160
@@ -1,160 +0,0 @@
|
||||
//go:build enable_antithesis_sdk && linux && amd64 && cgo
|
||||
|
||||
package internal
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"unsafe"
|
||||
"os"
|
||||
)
|
||||
|
||||
// --------------------------------------------------------------------------------
|
||||
// To build and run an executable with this package
|
||||
//
|
||||
// CC=clang CGO_ENABLED=1 go run ./main.go
|
||||
// --------------------------------------------------------------------------------
|
||||
|
||||
// \/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/
|
||||
//
|
||||
// The commented lines below, and the `import "C"` line which must directly follow
|
||||
// the commented lines are used by CGO. They are load-bearing, and should not be
|
||||
// changed without first understanding how CGO uses them.
|
||||
//
|
||||
// \/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/
|
||||
|
||||
// #cgo LDFLAGS: -ldl
|
||||
//
|
||||
// #include <dlfcn.h>
|
||||
// #include <stdbool.h>
|
||||
// #include <stdint.h>
|
||||
// #include <stdlib.h>
|
||||
//
|
||||
// typedef void (*go_fuzz_json_data_fn)(const char *data, size_t size);
|
||||
// void
|
||||
// go_fuzz_json_data(void *f, const char *data, size_t size) {
|
||||
// ((go_fuzz_json_data_fn)f)(data, size);
|
||||
// }
|
||||
//
|
||||
// typedef void (*go_fuzz_flush_fn)(void);
|
||||
// void
|
||||
// go_fuzz_flush(void *f) {
|
||||
// ((go_fuzz_flush_fn)f)();
|
||||
// }
|
||||
//
|
||||
// typedef uint64_t (*go_fuzz_get_random_fn)(void);
|
||||
// uint64_t
|
||||
// go_fuzz_get_random(void *f) {
|
||||
// return ((go_fuzz_get_random_fn)f)();
|
||||
// }
|
||||
//
|
||||
// typedef bool (*go_notify_coverage_fn)(size_t);
|
||||
// int
|
||||
// go_notify_coverage(void *f, size_t edges) {
|
||||
// bool b = ((go_notify_coverage_fn)f)(edges);
|
||||
// return b ? 1 : 0;
|
||||
// }
|
||||
//
|
||||
// typedef uint64_t (*go_init_coverage_fn)(size_t num_edges, const char *symbols);
|
||||
// uint64_t
|
||||
// go_init_coverage(void *f, size_t num_edges, const char *symbols) {
|
||||
// return ((go_init_coverage_fn)f)(num_edges, symbols);
|
||||
// }
|
||||
//
|
||||
import "C"
|
||||
|
||||
const (
|
||||
defaultNativeLibraryPath = "/usr/lib/libvoidstar.so"
|
||||
)
|
||||
|
||||
type voidstarHandler struct {
|
||||
fuzzJsonData unsafe.Pointer
|
||||
fuzzFlush unsafe.Pointer
|
||||
fuzzGetRandom unsafe.Pointer
|
||||
initCoverage unsafe.Pointer
|
||||
notifyCoverage unsafe.Pointer
|
||||
}
|
||||
|
||||
func (h *voidstarHandler) output(message string) {
|
||||
msg_len := len(message)
|
||||
if msg_len == 0 {
|
||||
return
|
||||
}
|
||||
cstrMessage := C.CString(message)
|
||||
defer C.free(unsafe.Pointer(cstrMessage))
|
||||
C.go_fuzz_json_data(h.fuzzJsonData, cstrMessage, C.ulong(msg_len))
|
||||
C.go_fuzz_flush(h.fuzzFlush)
|
||||
}
|
||||
|
||||
func (h *voidstarHandler) random() uint64 {
|
||||
return uint64(C.go_fuzz_get_random(h.fuzzGetRandom))
|
||||
}
|
||||
|
||||
func (h *voidstarHandler) init_coverage(num_edge uint64, symbols string) uint64 {
|
||||
cstrSymbols := C.CString(symbols)
|
||||
defer C.free(unsafe.Pointer(cstrSymbols))
|
||||
return uint64(C.go_init_coverage(h.initCoverage, C.ulong(num_edge), cstrSymbols))
|
||||
}
|
||||
|
||||
func (h *voidstarHandler) notify(edge uint64) bool {
|
||||
ival := int(C.go_notify_coverage(h.notifyCoverage, C.ulong(edge)))
|
||||
return ival == 1
|
||||
}
|
||||
|
||||
// Attempt to load libvoidstar and some symbols from `path`
|
||||
func openSharedLib(path string) (*voidstarHandler, error) {
|
||||
cstrPath := C.CString(path)
|
||||
defer C.free(unsafe.Pointer(cstrPath))
|
||||
|
||||
dlError := func(message string) error {
|
||||
return fmt.Errorf("%s: (%s)", message, C.GoString(C.dlerror()))
|
||||
}
|
||||
|
||||
sharedLib := C.dlopen(cstrPath, C.int(C.RTLD_NOW))
|
||||
if sharedLib == nil {
|
||||
return nil, dlError("Can not load the Antithesis native library")
|
||||
}
|
||||
|
||||
loadFunc := func(name string) (symbol unsafe.Pointer, err error) {
|
||||
cstrName := C.CString(name)
|
||||
defer C.free(unsafe.Pointer(cstrName))
|
||||
if symbol = C.dlsym(sharedLib, cstrName); symbol == nil {
|
||||
err = dlError(fmt.Sprintf("Can not access symbol %s", name))
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
fuzzJsonData, err := loadFunc("fuzz_json_data")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
fuzzFlush, err := loadFunc("fuzz_flush")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
fuzzGetRandom, err := loadFunc("fuzz_get_random")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
notifyCoverage, err := loadFunc("notify_coverage")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
initCoverage, err := loadFunc("init_coverage_module")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &voidstarHandler{fuzzJsonData, fuzzFlush, fuzzGetRandom, initCoverage, notifyCoverage}, nil
|
||||
}
|
||||
|
||||
// If we have a file at `defaultNativeLibraryPath`, we load the shared library
|
||||
// (and panic on any error encountered during load).
|
||||
func init_in_antithesis() libHandler {
|
||||
if _, err := os.Stat(defaultNativeLibraryPath); err == nil {
|
||||
handler, err := openSharedLib(defaultNativeLibraryPath)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return handler
|
||||
}
|
||||
return nil
|
||||
}
|
||||
Generated
Vendored
-7
@@ -1,7 +0,0 @@
|
||||
//go:build enable_antithesis_sdk && (!linux || !amd64 || !cgo)
|
||||
|
||||
package internal
|
||||
|
||||
func init_in_antithesis() libHandler {
|
||||
return nil
|
||||
}
|
||||
-202
@@ -1,202 +0,0 @@
|
||||
|
||||
Apache License
|
||||
Version 2.0, January 2004
|
||||
http://www.apache.org/licenses/
|
||||
|
||||
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
||||
|
||||
1. Definitions.
|
||||
|
||||
"License" shall mean the terms and conditions for use, reproduction,
|
||||
and distribution as defined by Sections 1 through 9 of this document.
|
||||
|
||||
"Licensor" shall mean the copyright owner or entity authorized by
|
||||
the copyright owner that is granting the License.
|
||||
|
||||
"Legal Entity" shall mean the union of the acting entity and all
|
||||
other entities that control, are controlled by, or are under common
|
||||
control with that entity. For the purposes of this definition,
|
||||
"control" means (i) the power, direct or indirect, to cause the
|
||||
direction or management of such entity, whether by contract or
|
||||
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
||||
outstanding shares, or (iii) beneficial ownership of such entity.
|
||||
|
||||
"You" (or "Your") shall mean an individual or Legal Entity
|
||||
exercising permissions granted by this License.
|
||||
|
||||
"Source" form shall mean the preferred form for making modifications,
|
||||
including but not limited to software source code, documentation
|
||||
source, and configuration files.
|
||||
|
||||
"Object" form shall mean any form resulting from mechanical
|
||||
transformation or translation of a Source form, including but
|
||||
not limited to compiled object code, generated documentation,
|
||||
and conversions to other media types.
|
||||
|
||||
"Work" shall mean the work of authorship, whether in Source or
|
||||
Object form, made available under the License, as indicated by a
|
||||
copyright notice that is included in or attached to the work
|
||||
(an example is provided in the Appendix below).
|
||||
|
||||
"Derivative Works" shall mean any work, whether in Source or Object
|
||||
form, that is based on (or derived from) the Work and for which the
|
||||
editorial revisions, annotations, elaborations, or other modifications
|
||||
represent, as a whole, an original work of authorship. For the purposes
|
||||
of this License, Derivative Works shall not include works that remain
|
||||
separable from, or merely link (or bind by name) to the interfaces of,
|
||||
the Work and Derivative Works thereof.
|
||||
|
||||
"Contribution" shall mean any work of authorship, including
|
||||
the original version of the Work and any modifications or additions
|
||||
to that Work or Derivative Works thereof, that is intentionally
|
||||
submitted to Licensor for inclusion in the Work by the copyright owner
|
||||
or by an individual or Legal Entity authorized to submit on behalf of
|
||||
the copyright owner. For the purposes of this definition, "submitted"
|
||||
means any form of electronic, verbal, or written communication sent
|
||||
to the Licensor or its representatives, including but not limited to
|
||||
communication on electronic mailing lists, source code control systems,
|
||||
and issue tracking systems that are managed by, or on behalf of, the
|
||||
Licensor for the purpose of discussing and improving the Work, but
|
||||
excluding communication that is conspicuously marked or otherwise
|
||||
designated in writing by the copyright owner as "Not a Contribution."
|
||||
|
||||
"Contributor" shall mean Licensor and any individual or Legal Entity
|
||||
on behalf of whom a Contribution has been received by Licensor and
|
||||
subsequently incorporated within the Work.
|
||||
|
||||
2. Grant of Copyright License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
copyright license to reproduce, prepare Derivative Works of,
|
||||
publicly display, publicly perform, sublicense, and distribute the
|
||||
Work and such Derivative Works in Source or Object form.
|
||||
|
||||
3. Grant of Patent License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
(except as stated in this section) patent license to make, have made,
|
||||
use, offer to sell, sell, import, and otherwise transfer the Work,
|
||||
where such license applies only to those patent claims licensable
|
||||
by such Contributor that are necessarily infringed by their
|
||||
Contribution(s) alone or by combination of their Contribution(s)
|
||||
with the Work to which such Contribution(s) was submitted. If You
|
||||
institute patent litigation against any entity (including a
|
||||
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
||||
or a Contribution incorporated within the Work constitutes direct
|
||||
or contributory patent infringement, then any patent licenses
|
||||
granted to You under this License for that Work shall terminate
|
||||
as of the date such litigation is filed.
|
||||
|
||||
4. Redistribution. You may reproduce and distribute copies of the
|
||||
Work or Derivative Works thereof in any medium, with or without
|
||||
modifications, and in Source or Object form, provided that You
|
||||
meet the following conditions:
|
||||
|
||||
(a) You must give any other recipients of the Work or
|
||||
Derivative Works a copy of this License; and
|
||||
|
||||
(b) You must cause any modified files to carry prominent notices
|
||||
stating that You changed the files; and
|
||||
|
||||
(c) You must retain, in the Source form of any Derivative Works
|
||||
that You distribute, all copyright, patent, trademark, and
|
||||
attribution notices from the Source form of the Work,
|
||||
excluding those notices that do not pertain to any part of
|
||||
the Derivative Works; and
|
||||
|
||||
(d) If the Work includes a "NOTICE" text file as part of its
|
||||
distribution, then any Derivative Works that You distribute must
|
||||
include a readable copy of the attribution notices contained
|
||||
within such NOTICE file, excluding those notices that do not
|
||||
pertain to any part of the Derivative Works, in at least one
|
||||
of the following places: within a NOTICE text file distributed
|
||||
as part of the Derivative Works; within the Source form or
|
||||
documentation, if provided along with the Derivative Works; or,
|
||||
within a display generated by the Derivative Works, if and
|
||||
wherever such third-party notices normally appear. The contents
|
||||
of the NOTICE file are for informational purposes only and
|
||||
do not modify the License. You may add Your own attribution
|
||||
notices within Derivative Works that You distribute, alongside
|
||||
or as an addendum to the NOTICE text from the Work, provided
|
||||
that such additional attribution notices cannot be construed
|
||||
as modifying the License.
|
||||
|
||||
You may add Your own copyright statement to Your modifications and
|
||||
may provide additional or different license terms and conditions
|
||||
for use, reproduction, or distribution of Your modifications, or
|
||||
for any such Derivative Works as a whole, provided Your use,
|
||||
reproduction, and distribution of the Work otherwise complies with
|
||||
the conditions stated in this License.
|
||||
|
||||
5. Submission of Contributions. Unless You explicitly state otherwise,
|
||||
any Contribution intentionally submitted for inclusion in the Work
|
||||
by You to the Licensor shall be under the terms and conditions of
|
||||
this License, without any additional terms or conditions.
|
||||
Notwithstanding the above, nothing herein shall supersede or modify
|
||||
the terms of any separate license agreement you may have executed
|
||||
with Licensor regarding such Contributions.
|
||||
|
||||
6. Trademarks. This License does not grant permission to use the trade
|
||||
names, trademarks, service marks, or product names of the Licensor,
|
||||
except as required for reasonable and customary use in describing the
|
||||
origin of the Work and reproducing the content of the NOTICE file.
|
||||
|
||||
7. Disclaimer of Warranty. Unless required by applicable law or
|
||||
agreed to in writing, Licensor provides the Work (and each
|
||||
Contributor provides its Contributions) on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
implied, including, without limitation, any warranties or conditions
|
||||
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
||||
PARTICULAR PURPOSE. You are solely responsible for determining the
|
||||
appropriateness of using or redistributing the Work and assume any
|
||||
risks associated with Your exercise of permissions under this License.
|
||||
|
||||
8. Limitation of Liability. In no event and under no legal theory,
|
||||
whether in tort (including negligence), contract, or otherwise,
|
||||
unless required by applicable law (such as deliberate and grossly
|
||||
negligent acts) or agreed to in writing, shall any Contributor be
|
||||
liable to You for damages, including any direct, indirect, special,
|
||||
incidental, or consequential damages of any character arising as a
|
||||
result of this License or out of the use or inability to use the
|
||||
Work (including but not limited to damages for loss of goodwill,
|
||||
work stoppage, computer failure or malfunction, or any and all
|
||||
other commercial damages or losses), even if such Contributor
|
||||
has been advised of the possibility of such damages.
|
||||
|
||||
9. Accepting Warranty or Additional Liability. While redistributing
|
||||
the Work or Derivative Works thereof, You may choose to offer,
|
||||
and charge a fee for, acceptance of support, warranty, indemnity,
|
||||
or other liability obligations and/or rights consistent with this
|
||||
License. However, in accepting such obligations, You may act only
|
||||
on Your own behalf and on Your sole responsibility, not on behalf
|
||||
of any other Contributor, and only if You agree to indemnify,
|
||||
defend, and hold each Contributor harmless for any liability
|
||||
incurred by, or claims asserted against, such Contributor by reason
|
||||
of your accepting any such warranty or additional liability.
|
||||
|
||||
END OF TERMS AND CONDITIONS
|
||||
|
||||
APPENDIX: How to apply the Apache License to your work.
|
||||
|
||||
To apply the Apache License to your work, attach the following
|
||||
boilerplate notice, with the fields enclosed by brackets "[]"
|
||||
replaced with your own identifying information. (Don't include
|
||||
the brackets!) The text should be enclosed in the appropriate
|
||||
comment syntax for the file format. We also recommend that a
|
||||
file or class name and description of purpose be included on the
|
||||
same "printed page" as the copyright notice for easier
|
||||
identification within third-party archives.
|
||||
|
||||
Copyright [yyyy] [name of copyright owner]
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
-35
@@ -1,35 +0,0 @@
|
||||
# TPM 2.0 client library
|
||||
|
||||
## Tests
|
||||
|
||||
This library contains unit tests in `github.com/google/go-tpm/tpm2`, which just
|
||||
tests that various encoding and error checking functions work correctly. It also
|
||||
contains more comprehensive integration tests in
|
||||
`github.com/google/go-tpm/tpm2/test`, which run actual commands on a TPM.
|
||||
|
||||
By default, these integration tests are run against the
|
||||
[`go-tpm-tools`](https://github.com/google/go-tpm-tools)
|
||||
simulator, which is baesed on the
|
||||
[Microsoft Reference TPM2 code](https://github.com/microsoft/ms-tpm-20-ref). To
|
||||
run both the unit and integration tests, run (in this directory)
|
||||
```bash
|
||||
go test . ./test
|
||||
```
|
||||
|
||||
These integration tests can also be run against a real TPM device. This is
|
||||
slightly more complex as the tests often need to be built as a normal user and
|
||||
then executed as root. For example,
|
||||
```bash
|
||||
# Build the test binary without running it
|
||||
go test -c github.com/google/go-tpm/tpm2/test
|
||||
# Execute the test binary as root
|
||||
sudo ./test.test --tpm-path=/dev/tpmrm0
|
||||
```
|
||||
On Linux, The `--tpm-path` causes the integration tests to be run against a
|
||||
real TPM located at that path (usually `/dev/tpmrm0` or `/dev/tpm0`). On Windows, the story is similar, execept that
|
||||
the `--use-tbs` flag is used instead.
|
||||
|
||||
Tip: if your TPM host is remote and you don't want to install Go on it, this
|
||||
same two-step process can be used. The test binary can be copied to a remote
|
||||
host and run without extra installation (as the test binary has very few
|
||||
*runtime* dependancies).
|
||||
-576
@@ -1,576 +0,0 @@
|
||||
// Copyright (c) 2018, Google LLC All rights reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package tpm2
|
||||
|
||||
import (
|
||||
"crypto"
|
||||
"crypto/elliptic"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
// Register the relevant hash implementations to prevent a runtime failure.
|
||||
_ "crypto/sha1"
|
||||
_ "crypto/sha256"
|
||||
_ "crypto/sha512"
|
||||
|
||||
"github.com/google/go-tpm/tpmutil"
|
||||
)
|
||||
|
||||
var hashInfo = []struct {
|
||||
alg Algorithm
|
||||
hash crypto.Hash
|
||||
}{
|
||||
{AlgSHA1, crypto.SHA1},
|
||||
{AlgSHA256, crypto.SHA256},
|
||||
{AlgSHA384, crypto.SHA384},
|
||||
{AlgSHA512, crypto.SHA512},
|
||||
{AlgSHA3_256, crypto.SHA3_256},
|
||||
{AlgSHA3_384, crypto.SHA3_384},
|
||||
{AlgSHA3_512, crypto.SHA3_512},
|
||||
}
|
||||
|
||||
// MAX_DIGEST_BUFFER is the maximum size of []byte request or response fields.
|
||||
// Typically used for chunking of big blobs of data (such as for hashing or
|
||||
// encryption).
|
||||
const maxDigestBuffer = 1024
|
||||
|
||||
// Algorithm represents a TPM_ALG_ID value.
|
||||
type Algorithm uint16
|
||||
|
||||
// HashToAlgorithm looks up the TPM2 algorithm corresponding to the provided crypto.Hash
|
||||
func HashToAlgorithm(hash crypto.Hash) (Algorithm, error) {
|
||||
for _, info := range hashInfo {
|
||||
if info.hash == hash {
|
||||
return info.alg, nil
|
||||
}
|
||||
}
|
||||
return AlgUnknown, fmt.Errorf("go hash algorithm #%d has no TPM2 algorithm", hash)
|
||||
}
|
||||
|
||||
// IsNull returns true if a is AlgNull or zero (unset).
|
||||
func (a Algorithm) IsNull() bool {
|
||||
return a == AlgNull || a == AlgUnknown
|
||||
}
|
||||
|
||||
// UsesCount returns true if a signature algorithm uses count value.
|
||||
func (a Algorithm) UsesCount() bool {
|
||||
return a == AlgECDAA
|
||||
}
|
||||
|
||||
// UsesHash returns true if the algorithm requires the use of a hash.
|
||||
func (a Algorithm) UsesHash() bool {
|
||||
return a == AlgOAEP
|
||||
}
|
||||
|
||||
// Hash returns a crypto.Hash based on the given TPM_ALG_ID.
|
||||
// An error is returned if the given algorithm is not a hash algorithm or is not available.
|
||||
func (a Algorithm) Hash() (crypto.Hash, error) {
|
||||
for _, info := range hashInfo {
|
||||
if info.alg == a {
|
||||
if !info.hash.Available() {
|
||||
return crypto.Hash(0), fmt.Errorf("go hash algorithm #%d not available", info.hash)
|
||||
}
|
||||
return info.hash, nil
|
||||
}
|
||||
}
|
||||
return crypto.Hash(0), fmt.Errorf("hash algorithm not supported: 0x%x", a)
|
||||
}
|
||||
|
||||
func (a Algorithm) String() string {
|
||||
var s strings.Builder
|
||||
var err error
|
||||
switch a {
|
||||
case AlgUnknown:
|
||||
_, err = s.WriteString("AlgUnknown")
|
||||
case AlgRSA:
|
||||
_, err = s.WriteString("RSA")
|
||||
case AlgSHA1:
|
||||
_, err = s.WriteString("SHA1")
|
||||
case AlgHMAC:
|
||||
_, err = s.WriteString("HMAC")
|
||||
case AlgAES:
|
||||
_, err = s.WriteString("AES")
|
||||
case AlgKeyedHash:
|
||||
_, err = s.WriteString("KeyedHash")
|
||||
case AlgXOR:
|
||||
_, err = s.WriteString("XOR")
|
||||
case AlgSHA256:
|
||||
_, err = s.WriteString("SHA256")
|
||||
case AlgSHA384:
|
||||
_, err = s.WriteString("SHA384")
|
||||
case AlgSHA512:
|
||||
_, err = s.WriteString("SHA512")
|
||||
case AlgNull:
|
||||
_, err = s.WriteString("AlgNull")
|
||||
case AlgRSASSA:
|
||||
_, err = s.WriteString("RSASSA")
|
||||
case AlgRSAES:
|
||||
_, err = s.WriteString("RSAES")
|
||||
case AlgRSAPSS:
|
||||
_, err = s.WriteString("RSAPSS")
|
||||
case AlgOAEP:
|
||||
_, err = s.WriteString("OAEP")
|
||||
case AlgECDSA:
|
||||
_, err = s.WriteString("ECDSA")
|
||||
case AlgECDH:
|
||||
_, err = s.WriteString("ECDH")
|
||||
case AlgECDAA:
|
||||
_, err = s.WriteString("ECDAA")
|
||||
case AlgKDF2:
|
||||
_, err = s.WriteString("KDF2")
|
||||
case AlgECC:
|
||||
_, err = s.WriteString("ECC")
|
||||
case AlgSymCipher:
|
||||
_, err = s.WriteString("SymCipher")
|
||||
case AlgSHA3_256:
|
||||
_, err = s.WriteString("SHA3_256")
|
||||
case AlgSHA3_384:
|
||||
_, err = s.WriteString("SHA3_384")
|
||||
case AlgSHA3_512:
|
||||
_, err = s.WriteString("SHA3_512")
|
||||
case AlgCTR:
|
||||
_, err = s.WriteString("CTR")
|
||||
case AlgOFB:
|
||||
_, err = s.WriteString("OFB")
|
||||
case AlgCBC:
|
||||
_, err = s.WriteString("CBC")
|
||||
case AlgCFB:
|
||||
_, err = s.WriteString("CFB")
|
||||
case AlgECB:
|
||||
_, err = s.WriteString("ECB")
|
||||
default:
|
||||
return fmt.Sprintf("Alg?<%d>", int(a))
|
||||
}
|
||||
if err != nil {
|
||||
return fmt.Sprintf("Writing to string builder failed: %v", err)
|
||||
}
|
||||
return s.String()
|
||||
}
|
||||
|
||||
// Supported Algorithms.
|
||||
const (
|
||||
AlgUnknown Algorithm = 0x0000
|
||||
AlgRSA Algorithm = 0x0001
|
||||
AlgSHA1 Algorithm = 0x0004
|
||||
AlgHMAC Algorithm = 0x0005
|
||||
AlgAES Algorithm = 0x0006
|
||||
AlgKeyedHash Algorithm = 0x0008
|
||||
AlgXOR Algorithm = 0x000A
|
||||
AlgSHA256 Algorithm = 0x000B
|
||||
AlgSHA384 Algorithm = 0x000C
|
||||
AlgSHA512 Algorithm = 0x000D
|
||||
AlgNull Algorithm = 0x0010
|
||||
AlgRSASSA Algorithm = 0x0014
|
||||
AlgRSAES Algorithm = 0x0015
|
||||
AlgRSAPSS Algorithm = 0x0016
|
||||
AlgOAEP Algorithm = 0x0017
|
||||
AlgECDSA Algorithm = 0x0018
|
||||
AlgECDH Algorithm = 0x0019
|
||||
AlgECDAA Algorithm = 0x001A
|
||||
AlgKDF2 Algorithm = 0x0021
|
||||
AlgECC Algorithm = 0x0023
|
||||
AlgSymCipher Algorithm = 0x0025
|
||||
AlgSHA3_256 Algorithm = 0x0027
|
||||
AlgSHA3_384 Algorithm = 0x0028
|
||||
AlgSHA3_512 Algorithm = 0x0029
|
||||
AlgCTR Algorithm = 0x0040
|
||||
AlgOFB Algorithm = 0x0041
|
||||
AlgCBC Algorithm = 0x0042
|
||||
AlgCFB Algorithm = 0x0043
|
||||
AlgECB Algorithm = 0x0044
|
||||
)
|
||||
|
||||
// HandleType defines a type of handle.
|
||||
type HandleType uint8
|
||||
|
||||
// Supported handle types
|
||||
const (
|
||||
HandleTypePCR HandleType = 0x00
|
||||
HandleTypeNVIndex HandleType = 0x01
|
||||
HandleTypeHMACSession HandleType = 0x02
|
||||
HandleTypeLoadedSession HandleType = 0x02
|
||||
HandleTypePolicySession HandleType = 0x03
|
||||
HandleTypeSavedSession HandleType = 0x03
|
||||
HandleTypePermanent HandleType = 0x40
|
||||
HandleTypeTransient HandleType = 0x80
|
||||
HandleTypePersistent HandleType = 0x81
|
||||
)
|
||||
|
||||
// SessionType defines the type of session created in StartAuthSession.
|
||||
type SessionType uint8
|
||||
|
||||
// Supported session types.
|
||||
const (
|
||||
SessionHMAC SessionType = 0x00
|
||||
SessionPolicy SessionType = 0x01
|
||||
SessionTrial SessionType = 0x03
|
||||
)
|
||||
|
||||
// SessionAttributes represents an attribute of a session.
|
||||
type SessionAttributes byte
|
||||
|
||||
// Session Attributes (Structures 8.4 TPMA_SESSION)
|
||||
const (
|
||||
AttrContinueSession SessionAttributes = 1 << iota
|
||||
AttrAuditExclusive
|
||||
AttrAuditReset
|
||||
_ // bit 3 reserved
|
||||
_ // bit 4 reserved
|
||||
AttrDecrypt
|
||||
AttrEcrypt
|
||||
AttrAudit
|
||||
)
|
||||
|
||||
// EmptyAuth represents the empty authorization value.
|
||||
var EmptyAuth []byte
|
||||
|
||||
// KeyProp is a bitmask used in Attributes field of key templates. Individual
|
||||
// flags should be OR-ed to form a full mask.
|
||||
type KeyProp uint32
|
||||
|
||||
// Key properties.
|
||||
const (
|
||||
FlagFixedTPM KeyProp = 0x00000002
|
||||
FlagStClear KeyProp = 0x00000004
|
||||
FlagFixedParent KeyProp = 0x00000010
|
||||
FlagSensitiveDataOrigin KeyProp = 0x00000020
|
||||
FlagUserWithAuth KeyProp = 0x00000040
|
||||
FlagAdminWithPolicy KeyProp = 0x00000080
|
||||
FlagNoDA KeyProp = 0x00000400
|
||||
FlagRestricted KeyProp = 0x00010000
|
||||
FlagDecrypt KeyProp = 0x00020000
|
||||
FlagSign KeyProp = 0x00040000
|
||||
|
||||
FlagSealDefault = FlagFixedTPM | FlagFixedParent
|
||||
FlagSignerDefault = FlagSign | FlagRestricted | FlagFixedTPM |
|
||||
FlagFixedParent | FlagSensitiveDataOrigin | FlagUserWithAuth
|
||||
FlagStorageDefault = FlagDecrypt | FlagRestricted | FlagFixedTPM |
|
||||
FlagFixedParent | FlagSensitiveDataOrigin | FlagUserWithAuth
|
||||
)
|
||||
|
||||
// TPMProp represents a Property Tag (TPM_PT) used with calls to GetCapability(CapabilityTPMProperties).
|
||||
type TPMProp uint32
|
||||
|
||||
// TPM Capability Properties, see TPM 2.0 Spec, Rev 1.38, Table 23.
|
||||
// Fixed TPM Properties (PT_FIXED)
|
||||
const (
|
||||
FamilyIndicator TPMProp = 0x100 + iota
|
||||
SpecLevel
|
||||
SpecRevision
|
||||
SpecDayOfYear
|
||||
SpecYear
|
||||
Manufacturer
|
||||
VendorString1
|
||||
VendorString2
|
||||
VendorString3
|
||||
VendorString4
|
||||
VendorTPMType
|
||||
FirmwareVersion1
|
||||
FirmwareVersion2
|
||||
InputMaxBufferSize
|
||||
TransientObjectsMin
|
||||
PersistentObjectsMin
|
||||
LoadedObjectsMin
|
||||
ActiveSessionsMax
|
||||
PCRCount
|
||||
PCRSelectMin
|
||||
ContextGapMax
|
||||
_ // (PT_FIXED + 21) is skipped
|
||||
NVCountersMax
|
||||
NVIndexMax
|
||||
MemoryMethod
|
||||
ClockUpdate
|
||||
ContextHash
|
||||
ContextSym
|
||||
ContextSymSize
|
||||
OrderlyCount
|
||||
CommandMaxSize
|
||||
ResponseMaxSize
|
||||
DigestMaxSize
|
||||
ObjectContextMaxSize
|
||||
SessionContextMaxSize
|
||||
PSFamilyIndicator
|
||||
PSSpecLevel
|
||||
PSSpecRevision
|
||||
PSSpecDayOfYear
|
||||
PSSpecYear
|
||||
SplitSigningMax
|
||||
TotalCommands
|
||||
LibraryCommands
|
||||
VendorCommands
|
||||
NVMaxBufferSize
|
||||
TPMModes
|
||||
CapabilityMaxBufferSize
|
||||
)
|
||||
|
||||
// Variable TPM Properties (PT_VAR)
|
||||
const (
|
||||
TPMAPermanent TPMProp = 0x200 + iota
|
||||
TPMAStartupClear
|
||||
HRNVIndex
|
||||
HRLoaded
|
||||
HRLoadedAvail
|
||||
HRActive
|
||||
HRActiveAvail
|
||||
HRTransientAvail
|
||||
CurrentPersistent
|
||||
AvailPersistent
|
||||
NVCounters
|
||||
NVCountersAvail
|
||||
AlgorithmSet
|
||||
LoadedCurves
|
||||
LockoutCounter
|
||||
MaxAuthFail
|
||||
LockoutInterval
|
||||
LockoutRecovery
|
||||
NVWriteRecovery
|
||||
AuditCounter0
|
||||
AuditCounter1
|
||||
)
|
||||
|
||||
// Allowed ranges of different kinds of Handles (TPM_HANDLE)
|
||||
// These constants have type TPMProp for backwards compatibility.
|
||||
const (
|
||||
PCRFirst TPMProp = 0x00000000
|
||||
HMACSessionFirst TPMProp = 0x02000000
|
||||
LoadedSessionFirst TPMProp = 0x02000000
|
||||
PolicySessionFirst TPMProp = 0x03000000
|
||||
ActiveSessionFirst TPMProp = 0x03000000
|
||||
TransientFirst TPMProp = 0x80000000
|
||||
PersistentFirst TPMProp = 0x81000000
|
||||
PersistentLast TPMProp = 0x81FFFFFF
|
||||
PlatformPersistent TPMProp = 0x81800000
|
||||
NVIndexFirst TPMProp = 0x01000000
|
||||
NVIndexLast TPMProp = 0x01FFFFFF
|
||||
PermanentFirst TPMProp = 0x40000000
|
||||
PermanentLast TPMProp = 0x4000010F
|
||||
)
|
||||
|
||||
// Reserved Handles.
|
||||
const (
|
||||
HandleOwner tpmutil.Handle = 0x40000001 + iota
|
||||
HandleRevoke
|
||||
HandleTransport
|
||||
HandleOperator
|
||||
HandleAdmin
|
||||
HandleEK
|
||||
HandleNull
|
||||
HandleUnassigned
|
||||
HandlePasswordSession
|
||||
HandleLockout
|
||||
HandleEndorsement
|
||||
HandlePlatform
|
||||
)
|
||||
|
||||
// Capability identifies some TPM property or state type.
|
||||
type Capability uint32
|
||||
|
||||
// TPM Capabilities.
|
||||
const (
|
||||
CapabilityAlgs Capability = iota
|
||||
CapabilityHandles
|
||||
CapabilityCommands
|
||||
CapabilityPPCommands
|
||||
CapabilityAuditCommands
|
||||
CapabilityPCRs
|
||||
CapabilityTPMProperties
|
||||
CapabilityPCRProperties
|
||||
CapabilityECCCurves
|
||||
CapabilityAuthPolicies
|
||||
)
|
||||
|
||||
// TPM Structure Tags. Tags are used to disambiguate structures, similar to Alg
|
||||
// values: tag value defines what kind of data lives in a nested field.
|
||||
const (
|
||||
TagNull tpmutil.Tag = 0x8000
|
||||
TagNoSessions tpmutil.Tag = 0x8001
|
||||
TagSessions tpmutil.Tag = 0x8002
|
||||
TagAttestCertify tpmutil.Tag = 0x8017
|
||||
TagAttestQuote tpmutil.Tag = 0x8018
|
||||
TagAttestCreation tpmutil.Tag = 0x801a
|
||||
TagAuthSecret tpmutil.Tag = 0x8023
|
||||
TagHashCheck tpmutil.Tag = 0x8024
|
||||
TagAuthSigned tpmutil.Tag = 0x8025
|
||||
)
|
||||
|
||||
// StartupType instructs the TPM on how to handle its state during Shutdown or
|
||||
// Startup.
|
||||
type StartupType uint16
|
||||
|
||||
// Startup types
|
||||
const (
|
||||
StartupClear StartupType = iota
|
||||
StartupState
|
||||
)
|
||||
|
||||
// EllipticCurve identifies specific EC curves.
|
||||
type EllipticCurve uint16
|
||||
|
||||
// ECC curves supported by TPM 2.0 spec.
|
||||
const (
|
||||
CurveNISTP192 = EllipticCurve(iota + 1)
|
||||
CurveNISTP224
|
||||
CurveNISTP256
|
||||
CurveNISTP384
|
||||
CurveNISTP521
|
||||
|
||||
CurveBNP256 = EllipticCurve(iota + 10)
|
||||
CurveBNP638
|
||||
|
||||
CurveSM2P256 = EllipticCurve(0x0020)
|
||||
)
|
||||
|
||||
var toGoCurve = map[EllipticCurve]elliptic.Curve{
|
||||
CurveNISTP224: elliptic.P224(),
|
||||
CurveNISTP256: elliptic.P256(),
|
||||
CurveNISTP384: elliptic.P384(),
|
||||
CurveNISTP521: elliptic.P521(),
|
||||
}
|
||||
|
||||
// Supported TPM operations.
|
||||
const (
|
||||
CmdNVUndefineSpaceSpecial tpmutil.Command = 0x0000011F
|
||||
CmdEvictControl tpmutil.Command = 0x00000120
|
||||
CmdUndefineSpace tpmutil.Command = 0x00000122
|
||||
CmdClear tpmutil.Command = 0x00000126
|
||||
CmdHierarchyChangeAuth tpmutil.Command = 0x00000129
|
||||
CmdDefineSpace tpmutil.Command = 0x0000012A
|
||||
CmdPCRAllocate tpmutil.Command = 0x0000012B
|
||||
CmdCreatePrimary tpmutil.Command = 0x00000131
|
||||
CmdIncrementNVCounter tpmutil.Command = 0x00000134
|
||||
CmdWriteNV tpmutil.Command = 0x00000137
|
||||
CmdWriteLockNV tpmutil.Command = 0x00000138
|
||||
CmdDictionaryAttackLockReset tpmutil.Command = 0x00000139
|
||||
CmdDictionaryAttackParameters tpmutil.Command = 0x0000013A
|
||||
CmdPCREvent tpmutil.Command = 0x0000013C
|
||||
CmdPCRReset tpmutil.Command = 0x0000013D
|
||||
CmdSequenceComplete tpmutil.Command = 0x0000013E
|
||||
CmdStartup tpmutil.Command = 0x00000144
|
||||
CmdShutdown tpmutil.Command = 0x00000145
|
||||
CmdActivateCredential tpmutil.Command = 0x00000147
|
||||
CmdCertify tpmutil.Command = 0x00000148
|
||||
CmdCertifyCreation tpmutil.Command = 0x0000014A
|
||||
CmdReadNV tpmutil.Command = 0x0000014E
|
||||
CmdReadLockNV tpmutil.Command = 0x0000014F
|
||||
CmdPolicySecret tpmutil.Command = 0x00000151
|
||||
CmdCreate tpmutil.Command = 0x00000153
|
||||
CmdECDHZGen tpmutil.Command = 0x00000154
|
||||
CmdImport tpmutil.Command = 0x00000156
|
||||
CmdLoad tpmutil.Command = 0x00000157
|
||||
CmdQuote tpmutil.Command = 0x00000158
|
||||
CmdRSADecrypt tpmutil.Command = 0x00000159
|
||||
CmdSequenceUpdate tpmutil.Command = 0x0000015C
|
||||
CmdSign tpmutil.Command = 0x0000015D
|
||||
CmdUnseal tpmutil.Command = 0x0000015E
|
||||
CmdPolicySigned tpmutil.Command = 0x00000160
|
||||
CmdContextLoad tpmutil.Command = 0x00000161
|
||||
CmdContextSave tpmutil.Command = 0x00000162
|
||||
CmdECDHKeyGen tpmutil.Command = 0x00000163
|
||||
CmdEncryptDecrypt tpmutil.Command = 0x00000164
|
||||
CmdFlushContext tpmutil.Command = 0x00000165
|
||||
CmdLoadExternal tpmutil.Command = 0x00000167
|
||||
CmdMakeCredential tpmutil.Command = 0x00000168
|
||||
CmdReadPublicNV tpmutil.Command = 0x00000169
|
||||
CmdPolicyCommandCode tpmutil.Command = 0x0000016C
|
||||
CmdPolicyOr tpmutil.Command = 0x00000171
|
||||
CmdReadPublic tpmutil.Command = 0x00000173
|
||||
CmdRSAEncrypt tpmutil.Command = 0x00000174
|
||||
CmdStartAuthSession tpmutil.Command = 0x00000176
|
||||
CmdGetCapability tpmutil.Command = 0x0000017A
|
||||
CmdGetRandom tpmutil.Command = 0x0000017B
|
||||
CmdHash tpmutil.Command = 0x0000017D
|
||||
CmdPCRRead tpmutil.Command = 0x0000017E
|
||||
CmdPolicyPCR tpmutil.Command = 0x0000017F
|
||||
CmdReadClock tpmutil.Command = 0x00000181
|
||||
CmdPCRExtend tpmutil.Command = 0x00000182
|
||||
CmdEventSequenceComplete tpmutil.Command = 0x00000185
|
||||
CmdHashSequenceStart tpmutil.Command = 0x00000186
|
||||
CmdPolicyGetDigest tpmutil.Command = 0x00000189
|
||||
CmdPolicyPassword tpmutil.Command = 0x0000018C
|
||||
CmdEncryptDecrypt2 tpmutil.Command = 0x00000193
|
||||
)
|
||||
|
||||
// Regular TPM 2.0 devices use 24-bit mask (3 bytes) for PCR selection.
|
||||
const sizeOfPCRSelect = 3
|
||||
|
||||
const defaultRSAExponent = 1<<16 + 1
|
||||
|
||||
// NVAttr is a bitmask used in Attributes field of NV indexes. Individual
|
||||
// flags should be OR-ed to form a full mask.
|
||||
type NVAttr uint32
|
||||
|
||||
// NV Attributes
|
||||
const (
|
||||
AttrPPWrite NVAttr = 0x00000001
|
||||
AttrOwnerWrite NVAttr = 0x00000002
|
||||
AttrAuthWrite NVAttr = 0x00000004
|
||||
AttrPolicyWrite NVAttr = 0x00000008
|
||||
AttrPolicyDelete NVAttr = 0x00000400
|
||||
AttrWriteLocked NVAttr = 0x00000800
|
||||
AttrWriteAll NVAttr = 0x00001000
|
||||
AttrWriteDefine NVAttr = 0x00002000
|
||||
AttrWriteSTClear NVAttr = 0x00004000
|
||||
AttrGlobalLock NVAttr = 0x00008000
|
||||
AttrPPRead NVAttr = 0x00010000
|
||||
AttrOwnerRead NVAttr = 0x00020000
|
||||
AttrAuthRead NVAttr = 0x00040000
|
||||
AttrPolicyRead NVAttr = 0x00080000
|
||||
AttrNoDA NVAttr = 0x02000000
|
||||
AttrOrderly NVAttr = 0x04000000
|
||||
AttrClearSTClear NVAttr = 0x08000000
|
||||
AttrReadLocked NVAttr = 0x10000000
|
||||
AttrWritten NVAttr = 0x20000000
|
||||
AttrPlatformCreate NVAttr = 0x40000000
|
||||
AttrReadSTClear NVAttr = 0x80000000
|
||||
)
|
||||
|
||||
var permMap = map[NVAttr]string{
|
||||
AttrPPWrite: "PPWrite",
|
||||
AttrOwnerWrite: "OwnerWrite",
|
||||
AttrAuthWrite: "AuthWrite",
|
||||
AttrPolicyWrite: "PolicyWrite",
|
||||
AttrPolicyDelete: "PolicyDelete",
|
||||
AttrWriteLocked: "WriteLocked",
|
||||
AttrWriteAll: "WriteAll",
|
||||
AttrWriteDefine: "WriteDefine",
|
||||
AttrWriteSTClear: "WriteSTClear",
|
||||
AttrGlobalLock: "GlobalLock",
|
||||
AttrPPRead: "PPRead",
|
||||
AttrOwnerRead: "OwnerRead",
|
||||
AttrAuthRead: "AuthRead",
|
||||
AttrPolicyRead: "PolicyRead",
|
||||
AttrNoDA: "No Do",
|
||||
AttrOrderly: "Oderly",
|
||||
AttrClearSTClear: "ClearSTClear",
|
||||
AttrReadLocked: "ReadLocked",
|
||||
AttrWritten: "Writte",
|
||||
AttrPlatformCreate: "PlatformCreate",
|
||||
AttrReadSTClear: "ReadSTClear",
|
||||
}
|
||||
|
||||
// String returns a textual representation of the set of NVAttr
|
||||
func (p NVAttr) String() string {
|
||||
var retString strings.Builder
|
||||
for iterator, item := range permMap {
|
||||
if (p & iterator) != 0 {
|
||||
retString.WriteString(item + " + ")
|
||||
}
|
||||
}
|
||||
if retString.String() == "" {
|
||||
return "Permission/s not found"
|
||||
}
|
||||
return strings.TrimSuffix(retString.String(), " + ")
|
||||
|
||||
}
|
||||
-362
@@ -1,362 +0,0 @@
|
||||
package tpm2
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"github.com/google/go-tpm/tpmutil"
|
||||
)
|
||||
|
||||
type (
|
||||
// RCFmt0 holds Format 0 error codes
|
||||
RCFmt0 uint8
|
||||
|
||||
// RCFmt1 holds Format 1 error codes
|
||||
RCFmt1 uint8
|
||||
|
||||
// RCWarn holds error codes used in warnings
|
||||
RCWarn uint8
|
||||
|
||||
// RCIndex is used to reference arguments, handles and sessions in errors
|
||||
RCIndex uint8
|
||||
)
|
||||
|
||||
// Format 0 error codes.
|
||||
const (
|
||||
RCInitialize RCFmt0 = 0x00
|
||||
RCFailure RCFmt0 = 0x01
|
||||
RCSequence RCFmt0 = 0x03
|
||||
RCPrivate RCFmt0 = 0x0B
|
||||
RCHMAC RCFmt0 = 0x19
|
||||
RCDisabled RCFmt0 = 0x20
|
||||
RCExclusive RCFmt0 = 0x21
|
||||
RCAuthType RCFmt0 = 0x24
|
||||
RCAuthMissing RCFmt0 = 0x25
|
||||
RCPolicy RCFmt0 = 0x26
|
||||
RCPCR RCFmt0 = 0x27
|
||||
RCPCRChanged RCFmt0 = 0x28
|
||||
RCUpgrade RCFmt0 = 0x2D
|
||||
RCTooManyContexts RCFmt0 = 0x2E
|
||||
RCAuthUnavailable RCFmt0 = 0x2F
|
||||
RCReboot RCFmt0 = 0x30
|
||||
RCUnbalanced RCFmt0 = 0x31
|
||||
RCCommandSize RCFmt0 = 0x42
|
||||
RCCommandCode RCFmt0 = 0x43
|
||||
RCAuthSize RCFmt0 = 0x44
|
||||
RCAuthContext RCFmt0 = 0x45
|
||||
RCNVRange RCFmt0 = 0x46
|
||||
RCNVSize RCFmt0 = 0x47
|
||||
RCNVLocked RCFmt0 = 0x48
|
||||
RCNVAuthorization RCFmt0 = 0x49
|
||||
RCNVUninitialized RCFmt0 = 0x4A
|
||||
RCNVSpace RCFmt0 = 0x4B
|
||||
RCNVDefined RCFmt0 = 0x4C
|
||||
RCBadContext RCFmt0 = 0x50
|
||||
RCCPHash RCFmt0 = 0x51
|
||||
RCParent RCFmt0 = 0x52
|
||||
RCNeedsTest RCFmt0 = 0x53
|
||||
RCNoResult RCFmt0 = 0x54
|
||||
RCSensitive RCFmt0 = 0x55
|
||||
)
|
||||
|
||||
var fmt0Msg = map[RCFmt0]string{
|
||||
RCInitialize: "TPM not initialized by TPM2_Startup or already initialized",
|
||||
RCFailure: "commands not being accepted because of a TPM failure",
|
||||
RCSequence: "improper use of a sequence handle",
|
||||
RCPrivate: "not currently used",
|
||||
RCHMAC: "not currently used",
|
||||
RCDisabled: "the command is disabled",
|
||||
RCExclusive: "command failed because audit sequence required exclusivity",
|
||||
RCAuthType: "authorization handle is not correct for command",
|
||||
RCAuthMissing: "5 command requires an authorization session for handle and it is not present",
|
||||
RCPolicy: "policy failure in math operation or an invalid authPolicy value",
|
||||
RCPCR: "PCR check fail",
|
||||
RCPCRChanged: "PCR have changed since checked",
|
||||
RCUpgrade: "TPM is in field upgrade mode unless called via TPM2_FieldUpgradeData(), then it is not in field upgrade mode",
|
||||
RCTooManyContexts: "context ID counter is at maximum",
|
||||
RCAuthUnavailable: "authValue or authPolicy is not available for selected entity",
|
||||
RCReboot: "a _TPM_Init and Startup(CLEAR) is required before the TPM can resume operation",
|
||||
RCUnbalanced: "the protection algorithms (hash and symmetric) are not reasonably balanced; the digest size of the hash must be larger than the key size of the symmetric algorithm",
|
||||
RCCommandSize: "command commandSize value is inconsistent with contents of the command buffer; either the size is not the same as the octets loaded by the hardware interface layer or the value is not large enough to hold a command header",
|
||||
RCCommandCode: "command code not supported",
|
||||
RCAuthSize: "the value of authorizationSize is out of range or the number of octets in the Authorization Area is greater than required",
|
||||
RCAuthContext: "use of an authorization session with a context command or another command that cannot have an authorization session",
|
||||
RCNVRange: "NV offset+size is out of range",
|
||||
RCNVSize: "Requested allocation size is larger than allowed",
|
||||
RCNVLocked: "NV access locked",
|
||||
RCNVAuthorization: "NV access authorization fails in command actions",
|
||||
RCNVUninitialized: "an NV Index is used before being initialized or the state saved by TPM2_Shutdown(STATE) could not be restored",
|
||||
RCNVSpace: "insufficient space for NV allocation",
|
||||
RCNVDefined: "NV Index or persistent object already defined",
|
||||
RCBadContext: "context in TPM2_ContextLoad() is not valid",
|
||||
RCCPHash: "cpHash value already set or not correct for use",
|
||||
RCParent: "handle for parent is not a valid parent",
|
||||
RCNeedsTest: "some function needs testing",
|
||||
RCNoResult: "returned when an internal function cannot process a request due to an unspecified problem; this code is usually related to invalid parameters that are not properly filtered by the input unmarshaling code",
|
||||
RCSensitive: "the sensitive area did not unmarshal correctly after decryption",
|
||||
}
|
||||
|
||||
// Format 1 error codes.
|
||||
const (
|
||||
RCAsymmetric = 0x01
|
||||
RCAttributes = 0x02
|
||||
RCHash = 0x03
|
||||
RCValue = 0x04
|
||||
RCHierarchy = 0x05
|
||||
RCKeySize = 0x07
|
||||
RCMGF = 0x08
|
||||
RCMode = 0x09
|
||||
RCType = 0x0A
|
||||
RCHandle = 0x0B
|
||||
RCKDF = 0x0C
|
||||
RCRange = 0x0D
|
||||
RCAuthFail = 0x0E
|
||||
RCNonce = 0x0F
|
||||
RCPP = 0x10
|
||||
RCScheme = 0x12
|
||||
RCSize = 0x15
|
||||
RCSymmetric = 0x16
|
||||
RCTag = 0x17
|
||||
RCSelector = 0x18
|
||||
RCInsufficient = 0x1A
|
||||
RCSignature = 0x1B
|
||||
RCKey = 0x1C
|
||||
RCPolicyFail = 0x1D
|
||||
RCIntegrity = 0x1F
|
||||
RCTicket = 0x20
|
||||
RCReservedBits = 0x21
|
||||
RCBadAuth = 0x22
|
||||
RCExpired = 0x23
|
||||
RCPolicyCC = 0x24
|
||||
RCBinding = 0x25
|
||||
RCCurve = 0x26
|
||||
RCECCPoint = 0x27
|
||||
)
|
||||
|
||||
var fmt1Msg = map[RCFmt1]string{
|
||||
RCAsymmetric: "asymmetric algorithm not supported or not correct",
|
||||
RCAttributes: "inconsistent attributes",
|
||||
RCHash: "hash algorithm not supported or not appropriate",
|
||||
RCValue: "value is out of range or is not correct for the context",
|
||||
RCHierarchy: "hierarchy is not enabled or is not correct for the use",
|
||||
RCKeySize: "key size is not supported",
|
||||
RCMGF: "mask generation function not supported",
|
||||
RCMode: "mode of operation not supported",
|
||||
RCType: "the type of the value is not appropriate for the use",
|
||||
RCHandle: "the handle is not correct for the use",
|
||||
RCKDF: "unsupported key derivation function or function not appropriate for use",
|
||||
RCRange: "value was out of allowed range",
|
||||
RCAuthFail: "the authorization HMAC check failed and DA counter incremented",
|
||||
RCNonce: "invalid nonce size or nonce value mismatch",
|
||||
RCPP: "authorization requires assertion of PP",
|
||||
RCScheme: "unsupported or incompatible scheme",
|
||||
RCSize: "structure is the wrong size",
|
||||
RCSymmetric: "unsupported symmetric algorithm or key size, or not appropriate for instance",
|
||||
RCTag: "incorrect structure tag",
|
||||
RCSelector: "union selector is incorrect",
|
||||
RCInsufficient: "the TPM was unable to unmarshal a value because there were not enough octets in the input buffer",
|
||||
RCSignature: "the signature is not valid",
|
||||
RCKey: "key fields are not compatible with the selected use",
|
||||
RCPolicyFail: "a policy check failed",
|
||||
RCIntegrity: "integrity check failed",
|
||||
RCTicket: "invalid ticket",
|
||||
RCReservedBits: "reserved bits not set to zero as required",
|
||||
RCBadAuth: "authorization failure without DA implications",
|
||||
RCExpired: "the policy has expired",
|
||||
RCPolicyCC: "the commandCode in the policy is not the commandCode of the command or the command code in a policy command references a command that is not implemented",
|
||||
RCBinding: "public and sensitive portions of an object are not cryptographically bound",
|
||||
RCCurve: "curve not supported",
|
||||
RCECCPoint: "point is not on the required curve",
|
||||
}
|
||||
|
||||
// Warning codes.
|
||||
const (
|
||||
RCContextGap RCWarn = 0x01
|
||||
RCObjectMemory RCWarn = 0x02
|
||||
RCSessionMemory RCWarn = 0x03
|
||||
RCMemory RCWarn = 0x04
|
||||
RCSessionHandles RCWarn = 0x05
|
||||
RCObjectHandles RCWarn = 0x06
|
||||
RCLocality RCWarn = 0x07
|
||||
RCYielded RCWarn = 0x08
|
||||
RCCanceled RCWarn = 0x09
|
||||
RCTesting RCWarn = 0x0A
|
||||
RCReferenceH0 RCWarn = 0x10
|
||||
RCReferenceH1 RCWarn = 0x11
|
||||
RCReferenceH2 RCWarn = 0x12
|
||||
RCReferenceH3 RCWarn = 0x13
|
||||
RCReferenceH4 RCWarn = 0x14
|
||||
RCReferenceH5 RCWarn = 0x15
|
||||
RCReferenceH6 RCWarn = 0x16
|
||||
RCReferenceS0 RCWarn = 0x18
|
||||
RCReferenceS1 RCWarn = 0x19
|
||||
RCReferenceS2 RCWarn = 0x1A
|
||||
RCReferenceS3 RCWarn = 0x1B
|
||||
RCReferenceS4 RCWarn = 0x1C
|
||||
RCReferenceS5 RCWarn = 0x1D
|
||||
RCReferenceS6 RCWarn = 0x1E
|
||||
RCNVRate RCWarn = 0x20
|
||||
RCLockout RCWarn = 0x21
|
||||
RCRetry RCWarn = 0x22
|
||||
RCNVUnavailable RCWarn = 0x23
|
||||
)
|
||||
|
||||
var warnMsg = map[RCWarn]string{
|
||||
RCContextGap: "gap for context ID is too large",
|
||||
RCObjectMemory: "out of memory for object contexts",
|
||||
RCSessionMemory: "out of memory for session contexts",
|
||||
RCMemory: "out of shared object/session memory or need space for internal operations",
|
||||
RCSessionHandles: "out of session handles",
|
||||
RCObjectHandles: "out of object handles",
|
||||
RCLocality: "bad locality",
|
||||
RCYielded: "the TPM has suspended operation on the command; forward progress was made and the command may be retried",
|
||||
RCCanceled: "the command was canceled",
|
||||
RCTesting: "TPM is performing self-tests",
|
||||
RCReferenceH0: "the 1st handle in the handle area references a transient object or session that is not loaded",
|
||||
RCReferenceH1: "the 2nd handle in the handle area references a transient object or session that is not loaded",
|
||||
RCReferenceH2: "the 3rd handle in the handle area references a transient object or session that is not loaded",
|
||||
RCReferenceH3: "the 4th handle in the handle area references a transient object or session that is not loaded",
|
||||
RCReferenceH4: "the 5th handle in the handle area references a transient object or session that is not loaded",
|
||||
RCReferenceH5: "the 6th handle in the handle area references a transient object or session that is not loaded",
|
||||
RCReferenceH6: "the 7th handle in the handle area references a transient object or session that is not loaded",
|
||||
RCReferenceS0: "the 1st authorization session handle references a session that is not loaded",
|
||||
RCReferenceS1: "the 2nd authorization session handle references a session that is not loaded",
|
||||
RCReferenceS2: "the 3rd authorization session handle references a session that is not loaded",
|
||||
RCReferenceS3: "the 4th authorization session handle references a session that is not loaded",
|
||||
RCReferenceS4: "the 5th authorization session handle references a session that is not loaded",
|
||||
RCReferenceS5: "the 6th authorization session handle references a session that is not loaded",
|
||||
RCReferenceS6: "the 7th authorization session handle references a session that is not loaded",
|
||||
RCNVRate: "the TPM is rate-limiting accesses to prevent wearout of NV",
|
||||
RCLockout: "authorizations for objects subject to DA protection are not allowed at this time because the TPM is in DA lockout mode",
|
||||
RCRetry: "the TPM was not able to start the command",
|
||||
RCNVUnavailable: "the command may require writing of NV and NV is not current accessible",
|
||||
}
|
||||
|
||||
// Indexes for arguments, handles and sessions.
|
||||
const (
|
||||
RC1 RCIndex = iota + 1
|
||||
RC2
|
||||
RC3
|
||||
RC4
|
||||
RC5
|
||||
RC6
|
||||
RC7
|
||||
RC8
|
||||
RC9
|
||||
RCA
|
||||
RCB
|
||||
RCC
|
||||
RCD
|
||||
RCE
|
||||
RCF
|
||||
)
|
||||
|
||||
const unknownCode = "unknown error code"
|
||||
|
||||
// Error is returned for all Format 0 errors from the TPM. It is used for general
|
||||
// errors not specific to a parameter, handle or session.
|
||||
type Error struct {
|
||||
Code RCFmt0
|
||||
}
|
||||
|
||||
func (e Error) Error() string {
|
||||
msg := fmt0Msg[e.Code]
|
||||
if msg == "" {
|
||||
msg = unknownCode
|
||||
}
|
||||
return fmt.Sprintf("error code 0x%x : %s", e.Code, msg)
|
||||
}
|
||||
|
||||
// VendorError represents a vendor-specific error response. These types of responses
|
||||
// are not decoded and Code contains the complete response code.
|
||||
type VendorError struct {
|
||||
Code uint32
|
||||
}
|
||||
|
||||
func (e VendorError) Error() string {
|
||||
return fmt.Sprintf("vendor error code 0x%x", e.Code)
|
||||
}
|
||||
|
||||
// Warning is typically used to report transient errors.
|
||||
type Warning struct {
|
||||
Code RCWarn
|
||||
}
|
||||
|
||||
func (w Warning) Error() string {
|
||||
msg := warnMsg[w.Code]
|
||||
if msg == "" {
|
||||
msg = unknownCode
|
||||
}
|
||||
return fmt.Sprintf("warning code 0x%x : %s", w.Code, msg)
|
||||
}
|
||||
|
||||
// ParameterError describes an error related to a parameter, and the parameter number.
|
||||
type ParameterError struct {
|
||||
Code RCFmt1
|
||||
Parameter RCIndex
|
||||
}
|
||||
|
||||
func (e ParameterError) Error() string {
|
||||
msg := fmt1Msg[e.Code]
|
||||
if msg == "" {
|
||||
msg = unknownCode
|
||||
}
|
||||
return fmt.Sprintf("parameter %d, error code 0x%x : %s", e.Parameter, e.Code, msg)
|
||||
}
|
||||
|
||||
// HandleError describes an error related to a handle, and the handle number.
|
||||
type HandleError struct {
|
||||
Code RCFmt1
|
||||
Handle RCIndex
|
||||
}
|
||||
|
||||
func (e HandleError) Error() string {
|
||||
msg := fmt1Msg[e.Code]
|
||||
if msg == "" {
|
||||
msg = unknownCode
|
||||
}
|
||||
return fmt.Sprintf("handle %d, error code 0x%x : %s", e.Handle, e.Code, msg)
|
||||
}
|
||||
|
||||
// SessionError describes an error related to a session, and the session number.
|
||||
type SessionError struct {
|
||||
Code RCFmt1
|
||||
Session RCIndex
|
||||
}
|
||||
|
||||
func (e SessionError) Error() string {
|
||||
msg := fmt1Msg[e.Code]
|
||||
if msg == "" {
|
||||
msg = unknownCode
|
||||
}
|
||||
return fmt.Sprintf("session %d, error code 0x%x : %s", e.Session, e.Code, msg)
|
||||
}
|
||||
|
||||
// Decode a TPM2 response code and return the appropriate error. Logic
|
||||
// according to the "Response Code Evaluation" chart in Part 1 of the TPM 2.0
|
||||
// spec.
|
||||
func decodeResponse(code tpmutil.ResponseCode) error {
|
||||
if code == tpmutil.RCSuccess {
|
||||
return nil
|
||||
}
|
||||
if code&0x180 == 0 { // Bits 7:8 == 0 is a TPM1 error
|
||||
return fmt.Errorf("response status 0x%x", code)
|
||||
}
|
||||
if code&0x80 == 0 { // Bit 7 unset
|
||||
if code&0x400 > 0 { // Bit 10 set, vendor specific code
|
||||
return VendorError{uint32(code)}
|
||||
}
|
||||
if code&0x800 > 0 { // Bit 11 set, warning with code in bit 0:6
|
||||
return Warning{RCWarn(code & 0x7f)}
|
||||
}
|
||||
// error with code in bit 0:6
|
||||
return Error{RCFmt0(code & 0x7f)}
|
||||
}
|
||||
if code&0x40 > 0 { // Bit 6 set, code in 0:5, parameter number in 8:11
|
||||
return ParameterError{RCFmt1(code & 0x3f), RCIndex((code & 0xf00) >> 8)}
|
||||
}
|
||||
if code&0x800 == 0 { // Bit 11 unset, code in 0:5, handle in 8:10
|
||||
return HandleError{RCFmt1(code & 0x3f), RCIndex((code & 0x700) >> 8)}
|
||||
}
|
||||
// Code in 0:5, Session in 8:10
|
||||
return SessionError{RCFmt1(code & 0x3f), RCIndex((code & 0x700) >> 8)}
|
||||
}
|
||||
-116
@@ -1,116 +0,0 @@
|
||||
// Copyright (c) 2018, Google LLC All rights reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package tpm2
|
||||
|
||||
import (
|
||||
"crypto"
|
||||
"crypto/hmac"
|
||||
"encoding/binary"
|
||||
"hash"
|
||||
)
|
||||
|
||||
// KDFa implements TPM 2.0's default key derivation function, as defined in
|
||||
// section 11.4.9.2 of the TPM revision 2 specification part 1.
|
||||
// See: https://trustedcomputinggroup.org/resource/tpm-library-specification/
|
||||
// The key & label parameters must not be zero length.
|
||||
// The label parameter is a non-null-terminated string.
|
||||
// The contextU & contextV parameters are optional.
|
||||
// Deprecated: Use KDFaHash.
|
||||
func KDFa(hashAlg Algorithm, key []byte, label string, contextU, contextV []byte, bits int) ([]byte, error) {
|
||||
h, err := hashAlg.Hash()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return KDFaHash(h, key, label, contextU, contextV, bits), nil
|
||||
}
|
||||
|
||||
// KDFe implements TPM 2.0's ECDH key derivation function, as defined in
|
||||
// section 11.4.9.3 of the TPM revision 2 specification part 1.
|
||||
// See: https://trustedcomputinggroup.org/resource/tpm-library-specification/
|
||||
// The z parameter is the x coordinate of one party's private ECC key multiplied
|
||||
// by the other party's public ECC point.
|
||||
// The use parameter is a non-null-terminated string.
|
||||
// The partyUInfo and partyVInfo are the x coordinates of the initiator's and
|
||||
// Deprecated: Use KDFeHash.
|
||||
func KDFe(hashAlg Algorithm, z []byte, use string, partyUInfo, partyVInfo []byte, bits int) ([]byte, error) {
|
||||
h, err := hashAlg.Hash()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return KDFeHash(h, z, use, partyUInfo, partyVInfo, bits), nil
|
||||
}
|
||||
|
||||
// KDFaHash implements TPM 2.0's default key derivation function, as defined in
|
||||
// section 11.4.9.2 of the TPM revision 2 specification part 1.
|
||||
// See: https://trustedcomputinggroup.org/resource/tpm-library-specification/
|
||||
// The key & label parameters must not be zero length.
|
||||
// The label parameter is a non-null-terminated string.
|
||||
// The contextU & contextV parameters are optional.
|
||||
func KDFaHash(h crypto.Hash, key []byte, label string, contextU, contextV []byte, bits int) []byte {
|
||||
mac := hmac.New(h.New, key)
|
||||
|
||||
out := kdf(mac, bits, func() {
|
||||
mac.Write([]byte(label))
|
||||
mac.Write([]byte{0}) // Terminating null character for C-string.
|
||||
mac.Write(contextU)
|
||||
mac.Write(contextV)
|
||||
binary.Write(mac, binary.BigEndian, uint32(bits))
|
||||
})
|
||||
return out
|
||||
}
|
||||
|
||||
// KDFeHash implements TPM 2.0's ECDH key derivation function, as defined in
|
||||
// section 11.4.9.3 of the TPM revision 2 specification part 1.
|
||||
// See: https://trustedcomputinggroup.org/resource/tpm-library-specification/
|
||||
// The z parameter is the x coordinate of one party's private ECC key multiplied
|
||||
// by the other party's public ECC point.
|
||||
// The use parameter is a non-null-terminated string.
|
||||
// The partyUInfo and partyVInfo are the x coordinates of the initiator's and
|
||||
// the responder's ECC points, respectively.
|
||||
func KDFeHash(h crypto.Hash, z []byte, use string, partyUInfo, partyVInfo []byte, bits int) []byte {
|
||||
hash := h.New()
|
||||
|
||||
out := kdf(hash, bits, func() {
|
||||
hash.Write(z)
|
||||
hash.Write([]byte(use))
|
||||
hash.Write([]byte{0}) // Terminating null character for C-string.
|
||||
hash.Write(partyUInfo)
|
||||
hash.Write(partyVInfo)
|
||||
})
|
||||
return out
|
||||
}
|
||||
|
||||
func kdf(h hash.Hash, bits int, update func()) []byte {
|
||||
bytes := (bits + 7) / 8
|
||||
out := []byte{}
|
||||
|
||||
for counter := 1; len(out) < bytes; counter++ {
|
||||
h.Reset()
|
||||
binary.Write(h, binary.BigEndian, uint32(counter))
|
||||
update()
|
||||
|
||||
out = h.Sum(out)
|
||||
}
|
||||
// out's length is a multiple of hash size, so there will be excess
|
||||
// bytes if bytes isn't a multiple of hash size.
|
||||
out = out[:bytes]
|
||||
|
||||
// As mentioned in the KDFa and KDFe specs mentioned above,
|
||||
// the unused bits of the most significant octet are masked off.
|
||||
if maskBits := uint8(bits % 8); maskBits > 0 {
|
||||
out[0] &= (1 << maskBits) - 1
|
||||
}
|
||||
return out
|
||||
}
|
||||
-57
@@ -1,57 +0,0 @@
|
||||
//go:build !windows
|
||||
|
||||
// Copyright (c) 2019, Google LLC All rights reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package tpm2
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
|
||||
"github.com/google/go-tpm/tpmutil"
|
||||
)
|
||||
|
||||
// OpenTPM opens a channel to the TPM at the given path. If the file is a
|
||||
// device, then it treats it like a normal TPM device, and if the file is a
|
||||
// Unix domain socket, then it opens a connection to the socket.
|
||||
//
|
||||
// This function may also be invoked with no paths, as tpm2.OpenTPM(). In this
|
||||
// case, the default paths on Linux (/dev/tpmrm0 then /dev/tpm0), will be used.
|
||||
func OpenTPM(path ...string) (tpm io.ReadWriteCloser, err error) {
|
||||
switch len(path) {
|
||||
case 0:
|
||||
tpm, err = tpmutil.OpenTPM("/dev/tpmrm0")
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
tpm, err = tpmutil.OpenTPM("/dev/tpm0")
|
||||
}
|
||||
case 1:
|
||||
tpm, err = tpmutil.OpenTPM(path[0])
|
||||
default:
|
||||
return nil, errors.New("cannot specify multiple paths to tpm2.OpenTPM")
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Make sure this is a TPM 2.0
|
||||
_, err = GetManufacturer(tpm)
|
||||
if err != nil {
|
||||
tpm.Close()
|
||||
return nil, fmt.Errorf("open %s: device is not a TPM 2.0", path)
|
||||
}
|
||||
return tpm, nil
|
||||
}
|
||||
-39
@@ -1,39 +0,0 @@
|
||||
//go:build windows
|
||||
|
||||
// Copyright (c) 2018, Google LLC All rights reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package tpm2
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
|
||||
"github.com/google/go-tpm/tpmutil"
|
||||
"github.com/google/go-tpm/tpmutil/tbs"
|
||||
)
|
||||
|
||||
// OpenTPM opens a channel to the TPM.
|
||||
func OpenTPM() (io.ReadWriteCloser, error) {
|
||||
info, err := tbs.GetDeviceInfo()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if info.TPMVersion != tbs.TPMVersion20 {
|
||||
return nil, fmt.Errorf("openTPM: device is not a TPM 2.0")
|
||||
}
|
||||
|
||||
return tpmutil.OpenTPM()
|
||||
}
|
||||
-1112
File diff suppressed because it is too large
Load Diff
-2354
File diff suppressed because it is too large
Load Diff
-211
@@ -1,211 +0,0 @@
|
||||
// Copyright (c) 2018, Google LLC All rights reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package tpmutil
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/binary"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"reflect"
|
||||
)
|
||||
|
||||
var (
|
||||
selfMarshalerType = reflect.TypeOf((*SelfMarshaler)(nil)).Elem()
|
||||
handlesAreaType = reflect.TypeOf((*[]Handle)(nil))
|
||||
)
|
||||
|
||||
// packWithHeader takes a header and a sequence of elements that are either of
|
||||
// fixed length or slices of fixed-length types and packs them into a single
|
||||
// byte array using binary.Write. It updates the CommandHeader to have the right
|
||||
// length.
|
||||
func packWithHeader(ch commandHeader, cmd ...interface{}) ([]byte, error) {
|
||||
hdrSize := binary.Size(ch)
|
||||
body, err := Pack(cmd...)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("couldn't pack message body: %v", err)
|
||||
}
|
||||
bodySize := len(body)
|
||||
ch.Size = uint32(hdrSize + bodySize)
|
||||
header, err := Pack(ch)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("couldn't pack message header: %v", err)
|
||||
}
|
||||
return append(header, body...), nil
|
||||
}
|
||||
|
||||
// Pack encodes a set of elements into a single byte array, using
|
||||
// encoding/binary. This means that all the elements must be encodeable
|
||||
// according to the rules of encoding/binary.
|
||||
//
|
||||
// It has one difference from encoding/binary: it encodes byte slices with a
|
||||
// prepended length, to match how the TPM encodes variable-length arrays. If
|
||||
// you wish to add a byte slice without length prefix, use RawBytes.
|
||||
func Pack(elts ...interface{}) ([]byte, error) {
|
||||
buf := new(bytes.Buffer)
|
||||
if err := packType(buf, elts...); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return buf.Bytes(), nil
|
||||
}
|
||||
|
||||
// tryMarshal attempts to use a TPMMarshal() method defined on the type
|
||||
// to pack v into buf. True is returned if the method exists and the
|
||||
// marshal was attempted.
|
||||
func tryMarshal(buf io.Writer, v reflect.Value) (bool, error) {
|
||||
t := v.Type()
|
||||
if t.Implements(selfMarshalerType) {
|
||||
if v.Kind() == reflect.Ptr && v.IsNil() {
|
||||
return true, fmt.Errorf("cannot call TPMMarshal on a nil pointer of type %T", v)
|
||||
}
|
||||
return true, v.Interface().(SelfMarshaler).TPMMarshal(buf)
|
||||
}
|
||||
|
||||
// We might have a non-pointer struct field, but we dont have a
|
||||
// pointer with which to implement the interface.
|
||||
// If the pointer of the type implements the interface, we should be
|
||||
// able to construct a value to call TPMMarshal() with.
|
||||
// TODO(awly): Try and avoid blowing away private data by using Addr() instead of Set()
|
||||
if reflect.PtrTo(t).Implements(selfMarshalerType) {
|
||||
tmp := reflect.New(t)
|
||||
tmp.Elem().Set(v)
|
||||
return true, tmp.Interface().(SelfMarshaler).TPMMarshal(buf)
|
||||
}
|
||||
|
||||
return false, nil
|
||||
}
|
||||
|
||||
func packValue(buf io.Writer, v reflect.Value) error {
|
||||
if v.Type() == handlesAreaType {
|
||||
v = v.Convert(reflect.TypeOf((*handleList)(nil)))
|
||||
}
|
||||
if canMarshal, err := tryMarshal(buf, v); canMarshal {
|
||||
return err
|
||||
}
|
||||
|
||||
switch v.Kind() {
|
||||
case reflect.Ptr:
|
||||
if v.IsNil() {
|
||||
return fmt.Errorf("cannot pack nil %s", v.Type().String())
|
||||
}
|
||||
return packValue(buf, v.Elem())
|
||||
case reflect.Struct:
|
||||
for i := 0; i < v.NumField(); i++ {
|
||||
f := v.Field(i)
|
||||
if err := packValue(buf, f); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
default:
|
||||
return binary.Write(buf, binary.BigEndian, v.Interface())
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func packType(buf io.Writer, elts ...interface{}) error {
|
||||
for _, e := range elts {
|
||||
if err := packValue(buf, reflect.ValueOf(e)); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// tryUnmarshal attempts to use TPMUnmarshal() to perform the
|
||||
// unpack, if the given value implements SelfMarshaler.
|
||||
// True is returned if v implements SelfMarshaler & TPMUnmarshal
|
||||
// was called, along with an error returned from TPMUnmarshal.
|
||||
func tryUnmarshal(buf io.Reader, v reflect.Value) (bool, error) {
|
||||
t := v.Type()
|
||||
if t.Implements(selfMarshalerType) {
|
||||
if v.Kind() == reflect.Ptr && v.IsNil() {
|
||||
return true, fmt.Errorf("cannot call TPMUnmarshal on a nil pointer")
|
||||
}
|
||||
return true, v.Interface().(SelfMarshaler).TPMUnmarshal(buf)
|
||||
}
|
||||
|
||||
// We might have a non-pointer struct field, which is addressable,
|
||||
// If the pointer of the type implements the interface, and the
|
||||
// value is addressable, we should be able to call TPMUnmarshal().
|
||||
if v.CanAddr() && reflect.PtrTo(t).Implements(selfMarshalerType) {
|
||||
return true, v.Addr().Interface().(SelfMarshaler).TPMUnmarshal(buf)
|
||||
}
|
||||
|
||||
return false, nil
|
||||
}
|
||||
|
||||
// Unpack is a convenience wrapper around UnpackBuf. Unpack returns the number
|
||||
// of bytes read from b to fill elts and error, if any.
|
||||
func Unpack(b []byte, elts ...interface{}) (int, error) {
|
||||
buf := bytes.NewBuffer(b)
|
||||
err := UnpackBuf(buf, elts...)
|
||||
read := len(b) - buf.Len()
|
||||
return read, err
|
||||
}
|
||||
|
||||
func unpackValue(buf io.Reader, v reflect.Value) error {
|
||||
if v.Type() == handlesAreaType {
|
||||
v = v.Convert(reflect.TypeOf((*handleList)(nil)))
|
||||
}
|
||||
if didUnmarshal, err := tryUnmarshal(buf, v); didUnmarshal {
|
||||
return err
|
||||
}
|
||||
|
||||
switch v.Kind() {
|
||||
case reflect.Ptr:
|
||||
if v.IsNil() {
|
||||
return fmt.Errorf("cannot unpack nil %s", v.Type().String())
|
||||
}
|
||||
return unpackValue(buf, v.Elem())
|
||||
case reflect.Struct:
|
||||
for i := 0; i < v.NumField(); i++ {
|
||||
f := v.Field(i)
|
||||
if err := unpackValue(buf, f); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
default:
|
||||
// binary.Read can only set pointer values, so we need to take the address.
|
||||
if !v.CanAddr() {
|
||||
return fmt.Errorf("cannot unpack unaddressable leaf type %q", v.Type().String())
|
||||
}
|
||||
return binary.Read(buf, binary.BigEndian, v.Addr().Interface())
|
||||
}
|
||||
}
|
||||
|
||||
// UnpackBuf recursively unpacks types from a reader just as encoding/binary
|
||||
// does under binary.BigEndian, but with one difference: it unpacks a byte
|
||||
// slice by first reading an integer with lengthPrefixSize bytes, then reading
|
||||
// that many bytes. It assumes that incoming values are pointers to values so
|
||||
// that, e.g., underlying slices can be resized as needed.
|
||||
func UnpackBuf(buf io.Reader, elts ...interface{}) error {
|
||||
for _, e := range elts {
|
||||
v := reflect.ValueOf(e)
|
||||
if v.Kind() != reflect.Ptr {
|
||||
return fmt.Errorf("non-pointer value %q passed to UnpackBuf", v.Type().String())
|
||||
}
|
||||
if v.IsNil() {
|
||||
return errors.New("nil pointer passed to UnpackBuf")
|
||||
}
|
||||
|
||||
if err := unpackValue(buf, v); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
-10
@@ -1,10 +0,0 @@
|
||||
//go:build !linux && !darwin
|
||||
|
||||
package tpmutil
|
||||
|
||||
import (
|
||||
"os"
|
||||
)
|
||||
|
||||
// Not implemented on Windows.
|
||||
func poll(_ *os.File) error { return nil }
|
||||
-32
@@ -1,32 +0,0 @@
|
||||
//go:build linux || darwin
|
||||
|
||||
package tpmutil
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
|
||||
"golang.org/x/sys/unix"
|
||||
)
|
||||
|
||||
// poll blocks until the file descriptor is ready for reading or an error occurs.
|
||||
func poll(f *os.File) error {
|
||||
var (
|
||||
fds = []unix.PollFd{{
|
||||
Fd: int32(f.Fd()),
|
||||
Events: 0x1, // POLLIN
|
||||
}}
|
||||
timeout = -1 // Indefinite timeout
|
||||
)
|
||||
|
||||
if _, err := unix.Poll(fds, timeout); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Revents is filled in by the kernel.
|
||||
// If the expected event happened, Revents should match Events.
|
||||
if fds[0].Revents != fds[0].Events {
|
||||
return fmt.Errorf("unexpected poll Revents 0x%x", fds[0].Revents)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
-113
@@ -1,113 +0,0 @@
|
||||
// Copyright (c) 2018, Google LLC All rights reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
// Package tpmutil provides common utility functions for both TPM 1.2 and TPM
|
||||
// 2.0 devices.
|
||||
package tpmutil
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"io"
|
||||
"os"
|
||||
"time"
|
||||
)
|
||||
|
||||
// maxTPMResponse is the largest possible response from the TPM. We need to know
|
||||
// this because we don't always know the length of the TPM response, and
|
||||
// /dev/tpm insists on giving it all back in a single value rather than
|
||||
// returning a header and a body in separate responses.
|
||||
const maxTPMResponse = 4096
|
||||
|
||||
// RunCommandRaw executes the given raw command and returns the raw response.
|
||||
// Does not check the response code except to execute retry logic.
|
||||
func RunCommandRaw(rw io.ReadWriter, inb []byte) ([]byte, error) {
|
||||
if rw == nil {
|
||||
return nil, errors.New("nil TPM handle")
|
||||
}
|
||||
|
||||
// f(t) = (2^t)ms, up to 2s
|
||||
var backoffFac uint
|
||||
var rh responseHeader
|
||||
var outb []byte
|
||||
|
||||
for {
|
||||
if _, err := rw.Write(inb); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// If the TPM is a real device, it may not be ready for reading
|
||||
// immediately after writing the command. Wait until the file
|
||||
// descriptor is ready to be read from.
|
||||
if f, ok := rw.(*os.File); ok {
|
||||
if err := poll(f); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
outb = make([]byte, maxTPMResponse)
|
||||
outlen, err := rw.Read(outb)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// Resize the buffer to match the amount read from the TPM.
|
||||
outb = outb[:outlen]
|
||||
|
||||
_, err = Unpack(outb, &rh)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// If TPM is busy, retry the command after waiting a few ms.
|
||||
if rh.Res == RCRetry {
|
||||
if backoffFac < 11 {
|
||||
dur := (1 << backoffFac) * time.Millisecond
|
||||
time.Sleep(dur)
|
||||
backoffFac++
|
||||
} else {
|
||||
return nil, err
|
||||
}
|
||||
} else {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
return outb, nil
|
||||
}
|
||||
|
||||
// RunCommand executes cmd with given tag and arguments. Returns TPM response
|
||||
// body (without response header) and response code from the header. Returned
|
||||
// error may be nil if response code is not RCSuccess; caller should check
|
||||
// both.
|
||||
func RunCommand(rw io.ReadWriter, tag Tag, cmd Command, in ...interface{}) ([]byte, ResponseCode, error) {
|
||||
inb, err := packWithHeader(commandHeader{tag, 0, cmd}, in...)
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
|
||||
outb, err := RunCommandRaw(rw, inb)
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
|
||||
var rh responseHeader
|
||||
read, err := Unpack(outb, &rh)
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
if rh.Res != RCSuccess {
|
||||
return nil, rh.Res, nil
|
||||
}
|
||||
|
||||
return outb[read:], rh.Res, nil
|
||||
}
|
||||
-111
@@ -1,111 +0,0 @@
|
||||
//go:build !windows
|
||||
|
||||
// Copyright (c) 2018, Google LLC All rights reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package tpmutil
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"os"
|
||||
)
|
||||
|
||||
// OpenTPM opens a channel to the TPM at the given path. If the file is a
|
||||
// device, then it treats it like a normal TPM device, and if the file is a
|
||||
// Unix domain socket, then it opens a connection to the socket.
|
||||
func OpenTPM(path string) (io.ReadWriteCloser, error) {
|
||||
// If it's a regular file, then open it
|
||||
var rwc io.ReadWriteCloser
|
||||
fi, err := os.Stat(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if fi.Mode()&os.ModeDevice != 0 {
|
||||
var f *os.File
|
||||
f, err = os.OpenFile(path, os.O_RDWR, 0600)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
rwc = io.ReadWriteCloser(f)
|
||||
} else if fi.Mode()&os.ModeSocket != 0 {
|
||||
rwc = NewEmulatorReadWriteCloser(path)
|
||||
} else {
|
||||
return nil, fmt.Errorf("unsupported TPM file mode %s", fi.Mode().String())
|
||||
}
|
||||
|
||||
return rwc, nil
|
||||
}
|
||||
|
||||
// dialer abstracts the net.Dial call so test code can provide its own net.Conn
|
||||
// implementation.
|
||||
type dialer func(network, path string) (net.Conn, error)
|
||||
|
||||
// EmulatorReadWriteCloser manages connections with a TPM emulator over a Unix
|
||||
// domain socket. These emulators often operate in a write/read/disconnect
|
||||
// sequence, so the Write method always connects, and the Read method always
|
||||
// closes. EmulatorReadWriteCloser is not thread safe.
|
||||
type EmulatorReadWriteCloser struct {
|
||||
path string
|
||||
conn net.Conn
|
||||
dialer dialer
|
||||
}
|
||||
|
||||
// NewEmulatorReadWriteCloser stores information about a Unix domain socket to
|
||||
// write to and read from.
|
||||
func NewEmulatorReadWriteCloser(path string) *EmulatorReadWriteCloser {
|
||||
return &EmulatorReadWriteCloser{
|
||||
path: path,
|
||||
dialer: net.Dial,
|
||||
}
|
||||
}
|
||||
|
||||
// Read implements io.Reader by reading from the Unix domain socket and closing
|
||||
// it.
|
||||
func (erw *EmulatorReadWriteCloser) Read(p []byte) (int, error) {
|
||||
// Read is always the second operation in a Write/Read sequence.
|
||||
if erw.conn == nil {
|
||||
return 0, fmt.Errorf("must call Write then Read in an alternating sequence")
|
||||
}
|
||||
n, err := erw.conn.Read(p)
|
||||
erw.conn.Close()
|
||||
erw.conn = nil
|
||||
return n, err
|
||||
}
|
||||
|
||||
// Write implements io.Writer by connecting to the Unix domain socket and
|
||||
// writing.
|
||||
func (erw *EmulatorReadWriteCloser) Write(p []byte) (int, error) {
|
||||
if erw.conn != nil {
|
||||
return 0, fmt.Errorf("must call Write then Read in an alternating sequence")
|
||||
}
|
||||
var err error
|
||||
erw.conn, err = erw.dialer("unix", erw.path)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return erw.conn.Write(p)
|
||||
}
|
||||
|
||||
// Close implements io.Closer by closing the Unix domain socket if one is open.
|
||||
func (erw *EmulatorReadWriteCloser) Close() error {
|
||||
if erw.conn == nil {
|
||||
return fmt.Errorf("cannot call Close when no connection is open")
|
||||
}
|
||||
err := erw.conn.Close()
|
||||
erw.conn = nil
|
||||
return err
|
||||
}
|
||||
-84
@@ -1,84 +0,0 @@
|
||||
// Copyright (c) 2018, Google LLC All rights reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package tpmutil
|
||||
|
||||
import (
|
||||
"io"
|
||||
|
||||
"github.com/google/go-tpm/tpmutil/tbs"
|
||||
)
|
||||
|
||||
// winTPMBuffer is a ReadWriteCloser to access the TPM in Windows.
|
||||
type winTPMBuffer struct {
|
||||
context tbs.Context
|
||||
outBuffer []byte
|
||||
}
|
||||
|
||||
// Executes the TPM command specified by commandBuffer (at Normal Priority), returning the number
|
||||
// of bytes in the command and any error code returned by executing the TPM command. Command
|
||||
// response can be read by calling Read().
|
||||
func (rwc *winTPMBuffer) Write(commandBuffer []byte) (int, error) {
|
||||
// TPM spec defines longest possible response to be maxTPMResponse.
|
||||
rwc.outBuffer = rwc.outBuffer[:maxTPMResponse]
|
||||
|
||||
outBufferLen, err := rwc.context.SubmitCommand(
|
||||
tbs.NormalPriority,
|
||||
commandBuffer,
|
||||
rwc.outBuffer,
|
||||
)
|
||||
|
||||
if err != nil {
|
||||
rwc.outBuffer = rwc.outBuffer[:0]
|
||||
return 0, err
|
||||
}
|
||||
// Shrink outBuffer so it is length of response.
|
||||
rwc.outBuffer = rwc.outBuffer[:outBufferLen]
|
||||
return len(commandBuffer), nil
|
||||
}
|
||||
|
||||
// Provides TPM response from the command called in the last Write call.
|
||||
func (rwc *winTPMBuffer) Read(responseBuffer []byte) (int, error) {
|
||||
if len(rwc.outBuffer) == 0 {
|
||||
return 0, io.EOF
|
||||
}
|
||||
lenCopied := copy(responseBuffer, rwc.outBuffer)
|
||||
// Cut out the piece of slice which was just read out, maintaining original slice capacity.
|
||||
rwc.outBuffer = append(rwc.outBuffer[:0], rwc.outBuffer[lenCopied:]...)
|
||||
return lenCopied, nil
|
||||
}
|
||||
|
||||
func (rwc *winTPMBuffer) Close() error {
|
||||
return rwc.context.Close()
|
||||
}
|
||||
|
||||
// OpenTPM creates a new instance of a ReadWriteCloser which can interact with a
|
||||
// Windows TPM.
|
||||
func OpenTPM() (io.ReadWriteCloser, error) {
|
||||
tpmContext, err := tbs.CreateContext(tbs.TPMVersion20, tbs.IncludeTPM12|tbs.IncludeTPM20)
|
||||
rwc := &winTPMBuffer{
|
||||
context: tpmContext,
|
||||
outBuffer: make([]byte, 0, maxTPMResponse),
|
||||
}
|
||||
return rwc, err
|
||||
}
|
||||
|
||||
// FromContext creates a new instance of a ReadWriteCloser which can
|
||||
// interact with a Windows TPM, using the specified TBS handle.
|
||||
func FromContext(ctx tbs.Context) io.ReadWriteCloser {
|
||||
return &winTPMBuffer{
|
||||
context: ctx,
|
||||
outBuffer: make([]byte, 0, maxTPMResponse),
|
||||
}
|
||||
}
|
||||
-195
@@ -1,195 +0,0 @@
|
||||
// Copyright (c) 2018, Google LLC All rights reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package tpmutil
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/binary"
|
||||
"fmt"
|
||||
"io"
|
||||
)
|
||||
|
||||
// maxBytesBufferSize sets a sane upper bound on the size of a U32Bytes
|
||||
// buffer. This limit exists to prevent a maliciously large size prefix
|
||||
// from resulting in a massive memory allocation, potentially causing
|
||||
// an OOM condition on the system.
|
||||
// We expect no buffer from a TPM to approach 1Mb in size.
|
||||
const maxBytesBufferSize uint32 = 1024 * 1024 // 1Mb.
|
||||
|
||||
// RawBytes is for Pack and RunCommand arguments that are already encoded.
|
||||
// Compared to []byte, RawBytes will not be prepended with slice length during
|
||||
// encoding.
|
||||
type RawBytes []byte
|
||||
|
||||
// U16Bytes is a byte slice with a 16-bit header
|
||||
type U16Bytes []byte
|
||||
|
||||
// TPMMarshal packs U16Bytes
|
||||
func (b *U16Bytes) TPMMarshal(out io.Writer) error {
|
||||
size := len([]byte(*b))
|
||||
if err := binary.Write(out, binary.BigEndian, uint16(size)); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
n, err := out.Write(*b)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if n != size {
|
||||
return fmt.Errorf("unable to write all contents of U16Bytes")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// TPMUnmarshal unpacks a U16Bytes
|
||||
func (b *U16Bytes) TPMUnmarshal(in io.Reader) error {
|
||||
var tmpSize uint16
|
||||
if err := binary.Read(in, binary.BigEndian, &tmpSize); err != nil {
|
||||
return err
|
||||
}
|
||||
size := int(tmpSize)
|
||||
|
||||
if len(*b) >= size {
|
||||
*b = (*b)[:size]
|
||||
} else {
|
||||
*b = append(*b, make([]byte, size-len(*b))...)
|
||||
}
|
||||
|
||||
n, err := in.Read(*b)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if n != size {
|
||||
return io.ErrUnexpectedEOF
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// U32Bytes is a byte slice with a 32-bit header
|
||||
type U32Bytes []byte
|
||||
|
||||
// TPMMarshal packs U32Bytes
|
||||
func (b *U32Bytes) TPMMarshal(out io.Writer) error {
|
||||
size := len([]byte(*b))
|
||||
if err := binary.Write(out, binary.BigEndian, uint32(size)); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
n, err := out.Write(*b)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if n != size {
|
||||
return fmt.Errorf("unable to write all contents of U32Bytes")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// TPMUnmarshal unpacks a U32Bytes
|
||||
func (b *U32Bytes) TPMUnmarshal(in io.Reader) error {
|
||||
var tmpSize uint32
|
||||
if err := binary.Read(in, binary.BigEndian, &tmpSize); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if tmpSize > maxBytesBufferSize {
|
||||
return bytes.ErrTooLarge
|
||||
}
|
||||
// We can now safely cast to an int on 32-bit or 64-bit machines
|
||||
size := int(tmpSize)
|
||||
|
||||
if len(*b) >= size {
|
||||
*b = (*b)[:size]
|
||||
} else {
|
||||
*b = append(*b, make([]byte, size-len(*b))...)
|
||||
}
|
||||
|
||||
n, err := in.Read(*b)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if n != size {
|
||||
return fmt.Errorf("unable to read all contents in to U32Bytes")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Tag is a command tag.
|
||||
type Tag uint16
|
||||
|
||||
// Command is an identifier of a TPM command.
|
||||
type Command uint32
|
||||
|
||||
// A commandHeader is the header for a TPM command.
|
||||
type commandHeader struct {
|
||||
Tag Tag
|
||||
Size uint32
|
||||
Cmd Command
|
||||
}
|
||||
|
||||
// ResponseCode is a response code returned by TPM.
|
||||
type ResponseCode uint32
|
||||
|
||||
// RCSuccess is response code for successful command. Identical for TPM 1.2 and
|
||||
// 2.0.
|
||||
const RCSuccess ResponseCode = 0x000
|
||||
|
||||
// RCRetry is response code for TPM is busy.
|
||||
const RCRetry ResponseCode = 0x922
|
||||
|
||||
// A responseHeader is a header for TPM responses.
|
||||
type responseHeader struct {
|
||||
Tag Tag
|
||||
Size uint32
|
||||
Res ResponseCode
|
||||
}
|
||||
|
||||
// A Handle is a reference to a TPM object.
|
||||
type Handle uint32
|
||||
|
||||
// HandleValue returns the handle value. This behavior is intended to satisfy
|
||||
// an interface that can be implemented by other, more complex types as well.
|
||||
func (h Handle) HandleValue() uint32 {
|
||||
return uint32(h)
|
||||
}
|
||||
|
||||
type handleList []Handle
|
||||
|
||||
func (l *handleList) TPMMarshal(_ io.Writer) error {
|
||||
return fmt.Errorf("TPMMarhsal on []Handle is not supported yet")
|
||||
}
|
||||
|
||||
func (l *handleList) TPMUnmarshal(in io.Reader) error {
|
||||
var numHandles uint16
|
||||
if err := binary.Read(in, binary.BigEndian, &numHandles); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Make len(e) match size exactly.
|
||||
size := int(numHandles)
|
||||
if len(*l) >= size {
|
||||
*l = (*l)[:size]
|
||||
} else {
|
||||
*l = append(*l, make([]Handle, size-len(*l))...)
|
||||
}
|
||||
return binary.Read(in, binary.BigEndian, *l)
|
||||
}
|
||||
|
||||
// SelfMarshaler allows custom types to override default encoding/decoding
|
||||
// behavior in Pack, Unpack and UnpackBuf.
|
||||
type SelfMarshaler interface {
|
||||
TPMMarshal(out io.Writer) error
|
||||
TPMUnmarshal(in io.Reader) error
|
||||
}
|
||||
-267
@@ -1,267 +0,0 @@
|
||||
// Copyright (c) 2018, Google LLC All rights reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
// Package tbs provides an low-level interface directly mapping to Windows
|
||||
// Tbs.dll system library commands:
|
||||
// https://docs.microsoft.com/en-us/windows/desktop/TBS/tpm-base-services-portal
|
||||
// Public field descriptions contain links to the high-level Windows documentation.
|
||||
package tbs
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"syscall"
|
||||
"unsafe"
|
||||
)
|
||||
|
||||
// Context references the current TPM context
|
||||
type Context uintptr
|
||||
|
||||
// Version of TPM being used by the application.
|
||||
type Version uint32
|
||||
|
||||
// Flag indicates TPM versions that are supported by the application.
|
||||
type Flag uint32
|
||||
|
||||
// CommandPriority is used to determine which pending command to submit whenever the TPM is free.
|
||||
type CommandPriority uint32
|
||||
|
||||
// Command parameters:
|
||||
// https://github.com/tpn/winsdk-10/blob/master/Include/10.0.10240.0/shared/tbs.h
|
||||
const (
|
||||
// https://docs.microsoft.com/en-us/windows/desktop/api/Tbs/ns-tbs-tdtbs_context_params2
|
||||
// OR flags to use multiple.
|
||||
RequestRaw Flag = 1 << iota // Add flag to request raw context
|
||||
IncludeTPM12 // Add flag to support TPM 1.2
|
||||
IncludeTPM20 // Add flag to support TPM 2
|
||||
|
||||
TPMVersion12 Version = 1 // For TPM 1.2 applications
|
||||
TPMVersion20 Version = 2 // For TPM 2 applications or applications using multiple TPM versions
|
||||
|
||||
// https://docs.microsoft.com/en-us/windows/desktop/tbs/command-scheduling
|
||||
// https://docs.microsoft.com/en-us/windows/desktop/api/Tbs/nf-tbs-tbsip_submit_command#parameters
|
||||
LowPriority CommandPriority = 100 // For low priority application use
|
||||
NormalPriority CommandPriority = 200 // For normal priority application use
|
||||
HighPriority CommandPriority = 300 // For high priority application use
|
||||
SystemPriority CommandPriority = 400 // For system tasks that access the TPM
|
||||
|
||||
commandLocalityZero uint32 = 0 // Windows currently only supports TBS_COMMAND_LOCALITY_ZERO.
|
||||
)
|
||||
|
||||
// Error is the return type of all functions in this package.
|
||||
type Error uint32
|
||||
|
||||
func (err Error) Error() string {
|
||||
if description, ok := errorDescriptions[err]; ok {
|
||||
return fmt.Sprintf("TBS Error 0x%X: %s", uint32(err), description)
|
||||
}
|
||||
return fmt.Sprintf("Unrecognized TBS Error 0x%X", uint32(err))
|
||||
}
|
||||
|
||||
func getError(err uintptr) error {
|
||||
// tbs.dll uses 0x0 as the return value for success.
|
||||
if err == 0 {
|
||||
return nil
|
||||
}
|
||||
return Error(err)
|
||||
}
|
||||
|
||||
// TBS Return Codes:
|
||||
// https://docs.microsoft.com/en-us/windows/desktop/TBS/tbs-return-codes
|
||||
const (
|
||||
ErrInternalError Error = 0x80284001
|
||||
ErrBadParameter Error = 0x80284002
|
||||
ErrInvalidOutputPointer Error = 0x80284003
|
||||
ErrInvalidContext Error = 0x80284004
|
||||
ErrInsufficientBuffer Error = 0x80284005
|
||||
ErrIOError Error = 0x80284006
|
||||
ErrInvalidContextParam Error = 0x80284007
|
||||
ErrServiceNotRunning Error = 0x80284008
|
||||
ErrTooManyTBSContexts Error = 0x80284009
|
||||
ErrTooManyResources Error = 0x8028400A
|
||||
ErrServiceStartPending Error = 0x8028400B
|
||||
ErrPPINotSupported Error = 0x8028400C
|
||||
ErrCommandCanceled Error = 0x8028400D
|
||||
ErrBufferTooLarge Error = 0x8028400E
|
||||
ErrTPMNotFound Error = 0x8028400F
|
||||
ErrServiceDisabled Error = 0x80284010
|
||||
ErrNoEventLog Error = 0x80284011
|
||||
ErrAccessDenied Error = 0x80284012
|
||||
ErrProvisioningNotAllowed Error = 0x80284013
|
||||
ErrPPIFunctionUnsupported Error = 0x80284014
|
||||
ErrOwnerauthNotFound Error = 0x80284015
|
||||
)
|
||||
|
||||
var errorDescriptions = map[Error]string{
|
||||
ErrInternalError: "An internal software error occurred.",
|
||||
ErrBadParameter: "One or more parameter values are not valid.",
|
||||
ErrInvalidOutputPointer: "A specified output pointer is bad.",
|
||||
ErrInvalidContext: "The specified context handle does not refer to a valid context.",
|
||||
ErrInsufficientBuffer: "The specified output buffer is too small.",
|
||||
ErrIOError: "An error occurred while communicating with the TPM.",
|
||||
ErrInvalidContextParam: "A context parameter that is not valid was passed when attempting to create a TBS context.",
|
||||
ErrServiceNotRunning: "The TBS service is not running and could not be started.",
|
||||
ErrTooManyTBSContexts: "A new context could not be created because there are too many open contexts.",
|
||||
ErrTooManyResources: "A new virtual resource could not be created because there are too many open virtual resources.",
|
||||
ErrServiceStartPending: "The TBS service has been started but is not yet running.",
|
||||
ErrPPINotSupported: "The physical presence interface is not supported.",
|
||||
ErrCommandCanceled: "The command was canceled.",
|
||||
ErrBufferTooLarge: "The input or output buffer is too large.",
|
||||
ErrTPMNotFound: "A compatible Trusted Platform Module (TPM) Security Device cannot be found on this computer.",
|
||||
ErrServiceDisabled: "The TBS service has been disabled.",
|
||||
ErrNoEventLog: "The TBS event log is not available.",
|
||||
ErrAccessDenied: "The caller does not have the appropriate rights to perform the requested operation.",
|
||||
ErrProvisioningNotAllowed: "The TPM provisioning action is not allowed by the specified flags.",
|
||||
ErrPPIFunctionUnsupported: "The Physical Presence Interface of this firmware does not support the requested method.",
|
||||
ErrOwnerauthNotFound: "The requested TPM OwnerAuth value was not found.",
|
||||
}
|
||||
|
||||
// Tbs.dll provides an API for making calls to the TPM:
|
||||
// https://docs.microsoft.com/en-us/windows/desktop/TBS/tpm-base-services-portal
|
||||
var (
|
||||
tbsDLL = syscall.NewLazyDLL("Tbs.dll")
|
||||
tbsGetDeviceInfo = tbsDLL.NewProc("Tbsi_GetDeviceInfo")
|
||||
tbsCreateContext = tbsDLL.NewProc("Tbsi_Context_Create")
|
||||
tbsContextClose = tbsDLL.NewProc("Tbsip_Context_Close")
|
||||
tbsSubmitCommand = tbsDLL.NewProc("Tbsip_Submit_Command")
|
||||
tbsGetTCGLog = tbsDLL.NewProc("Tbsi_Get_TCG_Log")
|
||||
)
|
||||
|
||||
// Returns the address of the beginning of a slice or 0 for a nil slice.
|
||||
func sliceAddress(s []byte) uintptr {
|
||||
if len(s) == 0 {
|
||||
return 0
|
||||
}
|
||||
return uintptr(unsafe.Pointer(&(s[0])))
|
||||
}
|
||||
|
||||
// DeviceInfo is TPM_DEVICE_INFO from tbs.h
|
||||
type DeviceInfo struct {
|
||||
StructVersion uint32
|
||||
TPMVersion Version
|
||||
TPMInterfaceType uint32
|
||||
TPMImpRevision uint32
|
||||
}
|
||||
|
||||
// GetDeviceInfo gets the DeviceInfo of the current TPM:
|
||||
// https://docs.microsoft.com/en-us/windows/win32/api/tbs/nf-tbs-tbsi_getdeviceinfo
|
||||
func GetDeviceInfo() (*DeviceInfo, error) {
|
||||
info := DeviceInfo{}
|
||||
// TBS_RESULT Tbsi_GetDeviceInfo(
|
||||
// UINT32 Size,
|
||||
// PVOID Info
|
||||
// );
|
||||
if err := tbsGetDeviceInfo.Find(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
result, _, _ := tbsGetDeviceInfo.Call(
|
||||
unsafe.Sizeof(info),
|
||||
uintptr(unsafe.Pointer(&info)),
|
||||
)
|
||||
return &info, getError(result)
|
||||
}
|
||||
|
||||
// CreateContext creates a new TPM context:
|
||||
// https://docs.microsoft.com/en-us/windows/desktop/api/Tbs/nf-tbs-tbsi_context_create
|
||||
func CreateContext(version Version, flag Flag) (Context, error) {
|
||||
var context Context
|
||||
params := struct {
|
||||
Version
|
||||
Flag
|
||||
}{version, flag}
|
||||
// TBS_RESULT Tbsi_Context_Create(
|
||||
// _In_ PCTBS_CONTEXT_PARAMS pContextParams,
|
||||
// _Out_ PTBS_HCONTEXT *phContext
|
||||
// );
|
||||
if err := tbsCreateContext.Find(); err != nil {
|
||||
return context, err
|
||||
}
|
||||
result, _, _ := tbsCreateContext.Call(
|
||||
uintptr(unsafe.Pointer(¶ms)),
|
||||
uintptr(unsafe.Pointer(&context)),
|
||||
)
|
||||
return context, getError(result)
|
||||
}
|
||||
|
||||
// Close closes an existing TPM context:
|
||||
// https://docs.microsoft.com/en-us/windows/desktop/api/Tbs/nf-tbs-tbsip_context_close
|
||||
func (context Context) Close() error {
|
||||
// TBS_RESULT Tbsip_Context_Close(
|
||||
// _In_ TBS_HCONTEXT hContext
|
||||
// );
|
||||
if err := tbsContextClose.Find(); err != nil {
|
||||
return err
|
||||
}
|
||||
result, _, _ := tbsContextClose.Call(uintptr(context))
|
||||
return getError(result)
|
||||
}
|
||||
|
||||
// SubmitCommand sends commandBuffer to the TPM, returning the number of bytes
|
||||
// written to responseBuffer. ErrInsufficientBuffer is returned if the
|
||||
// responseBuffer is too short. ErrInvalidOutputPointer is returned if the
|
||||
// responseBuffer is nil. On failure, the returned length is unspecified.
|
||||
// https://docs.microsoft.com/en-us/windows/desktop/api/Tbs/nf-tbs-tbsip_submit_command
|
||||
func (context Context) SubmitCommand(
|
||||
priority CommandPriority,
|
||||
commandBuffer []byte,
|
||||
responseBuffer []byte,
|
||||
) (uint32, error) {
|
||||
responseBufferLen := uint32(len(responseBuffer))
|
||||
|
||||
// TBS_RESULT Tbsip_Submit_Command(
|
||||
// _In_ TBS_HCONTEXT hContext,
|
||||
// _In_ TBS_COMMAND_LOCALITY Locality,
|
||||
// _In_ TBS_COMMAND_PRIORITY Priority,
|
||||
// _In_ const PCBYTE *pabCommand,
|
||||
// _In_ UINT32 cbCommand,
|
||||
// _Out_ PBYTE *pabResult,
|
||||
// _Inout_ UINT32 *pcbOutput
|
||||
// );
|
||||
if err := tbsSubmitCommand.Find(); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
result, _, _ := tbsSubmitCommand.Call(
|
||||
uintptr(context),
|
||||
uintptr(commandLocalityZero),
|
||||
uintptr(priority),
|
||||
sliceAddress(commandBuffer),
|
||||
uintptr(len(commandBuffer)),
|
||||
sliceAddress(responseBuffer),
|
||||
uintptr(unsafe.Pointer(&responseBufferLen)),
|
||||
)
|
||||
return responseBufferLen, getError(result)
|
||||
}
|
||||
|
||||
// GetTCGLog gets the system event log, returning the number of bytes written
|
||||
// to logBuffer. If logBuffer is nil, the size of the TCG log is returned.
|
||||
// ErrInsufficientBuffer is returned if the logBuffer is too short. On failure,
|
||||
// the returned length is unspecified.
|
||||
// https://docs.microsoft.com/en-us/windows/desktop/api/Tbs/nf-tbs-tbsi_get_tcg_log
|
||||
func (context Context) GetTCGLog(logBuffer []byte) (uint32, error) {
|
||||
logBufferLen := uint32(len(logBuffer))
|
||||
|
||||
// TBS_RESULT Tbsi_Get_TCG_Log(
|
||||
// TBS_HCONTEXT hContext,
|
||||
// PBYTE pOutputBuf,
|
||||
// PUINT32 pOutputBufLen
|
||||
// );
|
||||
if err := tbsGetTCGLog.Find(); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
result, _, _ := tbsGetTCGLog.Call(
|
||||
uintptr(context),
|
||||
sliceAddress(logBuffer),
|
||||
uintptr(unsafe.Pointer(&logBufferLen)),
|
||||
)
|
||||
return logBufferLen, getError(result)
|
||||
}
|
||||
-304
@@ -1,304 +0,0 @@
|
||||
Copyright (c) 2012 The Go Authors. All rights reserved.
|
||||
Copyright (c) 2019 Klaus Post. All rights reserved.
|
||||
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
modification, are permitted provided that the following conditions are
|
||||
met:
|
||||
|
||||
* Redistributions of source code must retain the above copyright
|
||||
notice, this list of conditions and the following disclaimer.
|
||||
* Redistributions in binary form must reproduce the above
|
||||
copyright notice, this list of conditions and the following disclaimer
|
||||
in the documentation and/or other materials provided with the
|
||||
distribution.
|
||||
* Neither the name of Google Inc. nor the names of its
|
||||
contributors may be used to endorse or promote products derived from
|
||||
this software without specific prior written permission.
|
||||
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||||
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
------------------
|
||||
|
||||
Files: gzhttp/*
|
||||
|
||||
Apache License
|
||||
Version 2.0, January 2004
|
||||
http://www.apache.org/licenses/
|
||||
|
||||
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
||||
|
||||
1. Definitions.
|
||||
|
||||
"License" shall mean the terms and conditions for use, reproduction,
|
||||
and distribution as defined by Sections 1 through 9 of this document.
|
||||
|
||||
"Licensor" shall mean the copyright owner or entity authorized by
|
||||
the copyright owner that is granting the License.
|
||||
|
||||
"Legal Entity" shall mean the union of the acting entity and all
|
||||
other entities that control, are controlled by, or are under common
|
||||
control with that entity. For the purposes of this definition,
|
||||
"control" means (i) the power, direct or indirect, to cause the
|
||||
direction or management of such entity, whether by contract or
|
||||
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
||||
outstanding shares, or (iii) beneficial ownership of such entity.
|
||||
|
||||
"You" (or "Your") shall mean an individual or Legal Entity
|
||||
exercising permissions granted by this License.
|
||||
|
||||
"Source" form shall mean the preferred form for making modifications,
|
||||
including but not limited to software source code, documentation
|
||||
source, and configuration files.
|
||||
|
||||
"Object" form shall mean any form resulting from mechanical
|
||||
transformation or translation of a Source form, including but
|
||||
not limited to compiled object code, generated documentation,
|
||||
and conversions to other media types.
|
||||
|
||||
"Work" shall mean the work of authorship, whether in Source or
|
||||
Object form, made available under the License, as indicated by a
|
||||
copyright notice that is included in or attached to the work
|
||||
(an example is provided in the Appendix below).
|
||||
|
||||
"Derivative Works" shall mean any work, whether in Source or Object
|
||||
form, that is based on (or derived from) the Work and for which the
|
||||
editorial revisions, annotations, elaborations, or other modifications
|
||||
represent, as a whole, an original work of authorship. For the purposes
|
||||
of this License, Derivative Works shall not include works that remain
|
||||
separable from, or merely link (or bind by name) to the interfaces of,
|
||||
the Work and Derivative Works thereof.
|
||||
|
||||
"Contribution" shall mean any work of authorship, including
|
||||
the original version of the Work and any modifications or additions
|
||||
to that Work or Derivative Works thereof, that is intentionally
|
||||
submitted to Licensor for inclusion in the Work by the copyright owner
|
||||
or by an individual or Legal Entity authorized to submit on behalf of
|
||||
the copyright owner. For the purposes of this definition, "submitted"
|
||||
means any form of electronic, verbal, or written communication sent
|
||||
to the Licensor or its representatives, including but not limited to
|
||||
communication on electronic mailing lists, source code control systems,
|
||||
and issue tracking systems that are managed by, or on behalf of, the
|
||||
Licensor for the purpose of discussing and improving the Work, but
|
||||
excluding communication that is conspicuously marked or otherwise
|
||||
designated in writing by the copyright owner as "Not a Contribution."
|
||||
|
||||
"Contributor" shall mean Licensor and any individual or Legal Entity
|
||||
on behalf of whom a Contribution has been received by Licensor and
|
||||
subsequently incorporated within the Work.
|
||||
|
||||
2. Grant of Copyright License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
copyright license to reproduce, prepare Derivative Works of,
|
||||
publicly display, publicly perform, sublicense, and distribute the
|
||||
Work and such Derivative Works in Source or Object form.
|
||||
|
||||
3. Grant of Patent License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
(except as stated in this section) patent license to make, have made,
|
||||
use, offer to sell, sell, import, and otherwise transfer the Work,
|
||||
where such license applies only to those patent claims licensable
|
||||
by such Contributor that are necessarily infringed by their
|
||||
Contribution(s) alone or by combination of their Contribution(s)
|
||||
with the Work to which such Contribution(s) was submitted. If You
|
||||
institute patent litigation against any entity (including a
|
||||
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
||||
or a Contribution incorporated within the Work constitutes direct
|
||||
or contributory patent infringement, then any patent licenses
|
||||
granted to You under this License for that Work shall terminate
|
||||
as of the date such litigation is filed.
|
||||
|
||||
4. Redistribution. You may reproduce and distribute copies of the
|
||||
Work or Derivative Works thereof in any medium, with or without
|
||||
modifications, and in Source or Object form, provided that You
|
||||
meet the following conditions:
|
||||
|
||||
(a) You must give any other recipients of the Work or
|
||||
Derivative Works a copy of this License; and
|
||||
|
||||
(b) You must cause any modified files to carry prominent notices
|
||||
stating that You changed the files; and
|
||||
|
||||
(c) You must retain, in the Source form of any Derivative Works
|
||||
that You distribute, all copyright, patent, trademark, and
|
||||
attribution notices from the Source form of the Work,
|
||||
excluding those notices that do not pertain to any part of
|
||||
the Derivative Works; and
|
||||
|
||||
(d) If the Work includes a "NOTICE" text file as part of its
|
||||
distribution, then any Derivative Works that You distribute must
|
||||
include a readable copy of the attribution notices contained
|
||||
within such NOTICE file, excluding those notices that do not
|
||||
pertain to any part of the Derivative Works, in at least one
|
||||
of the following places: within a NOTICE text file distributed
|
||||
as part of the Derivative Works; within the Source form or
|
||||
documentation, if provided along with the Derivative Works; or,
|
||||
within a display generated by the Derivative Works, if and
|
||||
wherever such third-party notices normally appear. The contents
|
||||
of the NOTICE file are for informational purposes only and
|
||||
do not modify the License. You may add Your own attribution
|
||||
notices within Derivative Works that You distribute, alongside
|
||||
or as an addendum to the NOTICE text from the Work, provided
|
||||
that such additional attribution notices cannot be construed
|
||||
as modifying the License.
|
||||
|
||||
You may add Your own copyright statement to Your modifications and
|
||||
may provide additional or different license terms and conditions
|
||||
for use, reproduction, or distribution of Your modifications, or
|
||||
for any such Derivative Works as a whole, provided Your use,
|
||||
reproduction, and distribution of the Work otherwise complies with
|
||||
the conditions stated in this License.
|
||||
|
||||
5. Submission of Contributions. Unless You explicitly state otherwise,
|
||||
any Contribution intentionally submitted for inclusion in the Work
|
||||
by You to the Licensor shall be under the terms and conditions of
|
||||
this License, without any additional terms or conditions.
|
||||
Notwithstanding the above, nothing herein shall supersede or modify
|
||||
the terms of any separate license agreement you may have executed
|
||||
with Licensor regarding such Contributions.
|
||||
|
||||
6. Trademarks. This License does not grant permission to use the trade
|
||||
names, trademarks, service marks, or product names of the Licensor,
|
||||
except as required for reasonable and customary use in describing the
|
||||
origin of the Work and reproducing the content of the NOTICE file.
|
||||
|
||||
7. Disclaimer of Warranty. Unless required by applicable law or
|
||||
agreed to in writing, Licensor provides the Work (and each
|
||||
Contributor provides its Contributions) on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
implied, including, without limitation, any warranties or conditions
|
||||
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
||||
PARTICULAR PURPOSE. You are solely responsible for determining the
|
||||
appropriateness of using or redistributing the Work and assume any
|
||||
risks associated with Your exercise of permissions under this License.
|
||||
|
||||
8. Limitation of Liability. In no event and under no legal theory,
|
||||
whether in tort (including negligence), contract, or otherwise,
|
||||
unless required by applicable law (such as deliberate and grossly
|
||||
negligent acts) or agreed to in writing, shall any Contributor be
|
||||
liable to You for damages, including any direct, indirect, special,
|
||||
incidental, or consequential damages of any character arising as a
|
||||
result of this License or out of the use or inability to use the
|
||||
Work (including but not limited to damages for loss of goodwill,
|
||||
work stoppage, computer failure or malfunction, or any and all
|
||||
other commercial damages or losses), even if such Contributor
|
||||
has been advised of the possibility of such damages.
|
||||
|
||||
9. Accepting Warranty or Additional Liability. While redistributing
|
||||
the Work or Derivative Works thereof, You may choose to offer,
|
||||
and charge a fee for, acceptance of support, warranty, indemnity,
|
||||
or other liability obligations and/or rights consistent with this
|
||||
License. However, in accepting such obligations, You may act only
|
||||
on Your own behalf and on Your sole responsibility, not on behalf
|
||||
of any other Contributor, and only if You agree to indemnify,
|
||||
defend, and hold each Contributor harmless for any liability
|
||||
incurred by, or claims asserted against, such Contributor by reason
|
||||
of your accepting any such warranty or additional liability.
|
||||
|
||||
END OF TERMS AND CONDITIONS
|
||||
|
||||
APPENDIX: How to apply the Apache License to your work.
|
||||
|
||||
To apply the Apache License to your work, attach the following
|
||||
boilerplate notice, with the fields enclosed by brackets "[]"
|
||||
replaced with your own identifying information. (Don't include
|
||||
the brackets!) The text should be enclosed in the appropriate
|
||||
comment syntax for the file format. We also recommend that a
|
||||
file or class name and description of purpose be included on the
|
||||
same "printed page" as the copyright notice for easier
|
||||
identification within third-party archives.
|
||||
|
||||
Copyright 2016-2017 The New York Times Company
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
|
||||
------------------
|
||||
|
||||
Files: s2/cmd/internal/readahead/*
|
||||
|
||||
The MIT License (MIT)
|
||||
|
||||
Copyright (c) 2015 Klaus Post
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all
|
||||
copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
SOFTWARE.
|
||||
|
||||
---------------------
|
||||
Files: snappy/*
|
||||
Files: internal/snapref/*
|
||||
|
||||
Copyright (c) 2011 The Snappy-Go Authors. All rights reserved.
|
||||
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
modification, are permitted provided that the following conditions are
|
||||
met:
|
||||
|
||||
* Redistributions of source code must retain the above copyright
|
||||
notice, this list of conditions and the following disclaimer.
|
||||
* Redistributions in binary form must reproduce the above
|
||||
copyright notice, this list of conditions and the following disclaimer
|
||||
in the documentation and/or other materials provided with the
|
||||
distribution.
|
||||
* Neither the name of Google Inc. nor the names of its
|
||||
contributors may be used to endorse or promote products derived from
|
||||
this software without specific prior written permission.
|
||||
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||||
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
-----------------
|
||||
|
||||
Files: s2/cmd/internal/filepathx/*
|
||||
|
||||
Copyright 2016 The filepathx Authors
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
-997
@@ -1,997 +0,0 @@
|
||||
// Copyright 2009 The Go Authors. All rights reserved.
|
||||
// Copyright (c) 2015 Klaus Post
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package flate
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"math"
|
||||
|
||||
"github.com/klauspost/compress/internal/le"
|
||||
)
|
||||
|
||||
const (
|
||||
NoCompression = 0
|
||||
BestSpeed = 1
|
||||
BestCompression = 9
|
||||
DefaultCompression = -1
|
||||
|
||||
// HuffmanOnly disables Lempel-Ziv match searching and only performs Huffman
|
||||
// entropy encoding. This mode is useful in compressing data that has
|
||||
// already been compressed with an LZ style algorithm (e.g. Snappy or LZ4)
|
||||
// that lacks an entropy encoder. Compression gains are achieved when
|
||||
// certain bytes in the input stream occur more frequently than others.
|
||||
//
|
||||
// Note that HuffmanOnly produces a compressed output that is
|
||||
// RFC 1951 compliant. That is, any valid DEFLATE decompressor will
|
||||
// continue to be able to decompress this output.
|
||||
HuffmanOnly = -2
|
||||
ConstantCompression = HuffmanOnly // compatibility alias.
|
||||
|
||||
logWindowSize = 15
|
||||
windowSize = 1 << logWindowSize
|
||||
windowMask = windowSize - 1
|
||||
logMaxOffsetSize = 15 // Standard DEFLATE
|
||||
minMatchLength = 4 // The smallest match that the compressor looks for
|
||||
maxMatchLength = 258 // The longest match for the compressor
|
||||
minOffsetSize = 1 // The shortest offset that makes any sense
|
||||
|
||||
// The maximum number of tokens we will encode at the time.
|
||||
// Smaller sizes usually creates less optimal blocks.
|
||||
// Bigger can make context switching slow.
|
||||
// We use this for levels 7-9, so we make it big.
|
||||
maxFlateBlockTokens = 1 << 15
|
||||
maxStoreBlockSize = 65535
|
||||
hashBits = 17 // After 17 performance degrades
|
||||
hashSize = 1 << hashBits
|
||||
hashMask = (1 << hashBits) - 1
|
||||
hashShift = (hashBits + minMatchLength - 1) / minMatchLength
|
||||
maxHashOffset = 1 << 28
|
||||
|
||||
skipNever = math.MaxInt32
|
||||
|
||||
debugDeflate = false
|
||||
)
|
||||
|
||||
type compressionLevel struct {
|
||||
good, lazy, nice, chain, fastSkipHashing, level int
|
||||
}
|
||||
|
||||
// Compression levels have been rebalanced from zlib deflate defaults
|
||||
// to give a bigger spread in speed and compression.
|
||||
// See https://blog.klauspost.com/rebalancing-deflate-compression-levels/
|
||||
var levels = []compressionLevel{
|
||||
{}, // 0
|
||||
// Level 1-6 uses specialized algorithm - values not used
|
||||
{0, 0, 0, 0, 0, 1},
|
||||
{0, 0, 0, 0, 0, 2},
|
||||
{0, 0, 0, 0, 0, 3},
|
||||
{0, 0, 0, 0, 0, 4},
|
||||
{0, 0, 0, 0, 0, 5},
|
||||
{0, 0, 0, 0, 0, 6},
|
||||
// Levels 7-9 use increasingly more lazy matching
|
||||
// and increasingly stringent conditions for "good enough".
|
||||
{8, 12, 16, 24, skipNever, 7},
|
||||
{16, 30, 40, 64, skipNever, 8},
|
||||
{32, 258, 258, 1024, skipNever, 9},
|
||||
}
|
||||
|
||||
// advancedState contains state for the advanced levels, with bigger hash tables, etc.
|
||||
type advancedState struct {
|
||||
// deflate state
|
||||
length int
|
||||
offset int
|
||||
maxInsertIndex int
|
||||
chainHead int
|
||||
hashOffset int
|
||||
|
||||
ii uint16 // position of last match, intended to overflow to reset.
|
||||
|
||||
// input window: unprocessed data is window[index:windowEnd]
|
||||
index int
|
||||
hashMatch [maxMatchLength + minMatchLength]uint32
|
||||
|
||||
// Input hash chains
|
||||
// hashHead[hashValue] contains the largest inputIndex with the specified hash value
|
||||
// If hashHead[hashValue] is within the current window, then
|
||||
// hashPrev[hashHead[hashValue] & windowMask] contains the previous index
|
||||
// with the same hash value.
|
||||
hashHead [hashSize]uint32
|
||||
hashPrev [windowSize]uint32
|
||||
}
|
||||
|
||||
type compressor struct {
|
||||
compressionLevel
|
||||
|
||||
h *huffmanEncoder
|
||||
w *huffmanBitWriter
|
||||
|
||||
// compression algorithm
|
||||
fill func(*compressor, []byte) int // copy data to window
|
||||
step func(*compressor) // process window
|
||||
|
||||
window []byte
|
||||
windowEnd int
|
||||
blockStart int // window index where current tokens start
|
||||
err error
|
||||
|
||||
// queued output tokens
|
||||
tokens tokens
|
||||
fast fastEnc
|
||||
state *advancedState
|
||||
|
||||
sync bool // requesting flush
|
||||
byteAvailable bool // if true, still need to process window[index-1].
|
||||
}
|
||||
|
||||
func (d *compressor) fillDeflate(b []byte) int {
|
||||
s := d.state
|
||||
if s.index >= 2*windowSize-(minMatchLength+maxMatchLength) {
|
||||
// shift the window by windowSize
|
||||
//copy(d.window[:], d.window[windowSize:2*windowSize])
|
||||
*(*[windowSize]byte)(d.window) = *(*[windowSize]byte)(d.window[windowSize:])
|
||||
s.index -= windowSize
|
||||
d.windowEnd -= windowSize
|
||||
if d.blockStart >= windowSize {
|
||||
d.blockStart -= windowSize
|
||||
} else {
|
||||
d.blockStart = math.MaxInt32
|
||||
}
|
||||
s.hashOffset += windowSize
|
||||
if s.hashOffset > maxHashOffset {
|
||||
delta := s.hashOffset - 1
|
||||
s.hashOffset -= delta
|
||||
s.chainHead -= delta
|
||||
// Iterate over slices instead of arrays to avoid copying
|
||||
// the entire table onto the stack (Issue #18625).
|
||||
for i, v := range s.hashPrev[:] {
|
||||
if int(v) > delta {
|
||||
s.hashPrev[i] = uint32(int(v) - delta)
|
||||
} else {
|
||||
s.hashPrev[i] = 0
|
||||
}
|
||||
}
|
||||
for i, v := range s.hashHead[:] {
|
||||
if int(v) > delta {
|
||||
s.hashHead[i] = uint32(int(v) - delta)
|
||||
} else {
|
||||
s.hashHead[i] = 0
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
n := copy(d.window[d.windowEnd:], b)
|
||||
d.windowEnd += n
|
||||
return n
|
||||
}
|
||||
|
||||
func (d *compressor) writeBlock(tok *tokens, index int, eof bool) error {
|
||||
if index > 0 || eof {
|
||||
var window []byte
|
||||
if d.blockStart <= index {
|
||||
window = d.window[d.blockStart:index]
|
||||
}
|
||||
d.blockStart = index
|
||||
//d.w.writeBlock(tok, eof, window)
|
||||
d.w.writeBlockDynamic(tok, eof, window, d.sync)
|
||||
return d.w.err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// writeBlockSkip writes the current block and uses the number of tokens
|
||||
// to determine if the block should be stored on no matches, or
|
||||
// only huffman encoded.
|
||||
func (d *compressor) writeBlockSkip(tok *tokens, index int, eof bool) error {
|
||||
if index > 0 || eof {
|
||||
if d.blockStart <= index {
|
||||
window := d.window[d.blockStart:index]
|
||||
// If we removed less than a 64th of all literals
|
||||
// we huffman compress the block.
|
||||
if int(tok.n) > len(window)-int(tok.n>>6) {
|
||||
d.w.writeBlockHuff(eof, window, d.sync)
|
||||
} else {
|
||||
// Write a dynamic huffman block.
|
||||
d.w.writeBlockDynamic(tok, eof, window, d.sync)
|
||||
}
|
||||
} else {
|
||||
d.w.writeBlock(tok, eof, nil)
|
||||
}
|
||||
d.blockStart = index
|
||||
return d.w.err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// fillWindow will fill the current window with the supplied
|
||||
// dictionary and calculate all hashes.
|
||||
// This is much faster than doing a full encode.
|
||||
// Should only be used after a start/reset.
|
||||
func (d *compressor) fillWindow(b []byte) {
|
||||
// Do not fill window if we are in store-only or huffman mode.
|
||||
if d.level <= 0 && d.level > -MinCustomWindowSize {
|
||||
return
|
||||
}
|
||||
if d.fast != nil {
|
||||
// encode the last data, but discard the result
|
||||
if len(b) > maxMatchOffset {
|
||||
b = b[len(b)-maxMatchOffset:]
|
||||
}
|
||||
d.fast.Encode(&d.tokens, b)
|
||||
d.tokens.Reset()
|
||||
return
|
||||
}
|
||||
s := d.state
|
||||
// If we are given too much, cut it.
|
||||
if len(b) > windowSize {
|
||||
b = b[len(b)-windowSize:]
|
||||
}
|
||||
// Add all to window.
|
||||
n := copy(d.window[d.windowEnd:], b)
|
||||
|
||||
// Calculate 256 hashes at the time (more L1 cache hits)
|
||||
loops := (n + 256 - minMatchLength) / 256
|
||||
for j := range loops {
|
||||
startindex := j * 256
|
||||
end := min(startindex+256+minMatchLength-1, n)
|
||||
tocheck := d.window[startindex:end]
|
||||
dstSize := len(tocheck) - minMatchLength + 1
|
||||
|
||||
if dstSize <= 0 {
|
||||
continue
|
||||
}
|
||||
|
||||
dst := s.hashMatch[:dstSize]
|
||||
bulkHash4(tocheck, dst)
|
||||
var newH uint32
|
||||
for i, val := range dst {
|
||||
di := i + startindex
|
||||
newH = val & hashMask
|
||||
// Get previous value with the same hash.
|
||||
// Our chain should point to the previous value.
|
||||
s.hashPrev[di&windowMask] = s.hashHead[newH]
|
||||
// Set the head of the hash chain to us.
|
||||
s.hashHead[newH] = uint32(di + s.hashOffset)
|
||||
}
|
||||
}
|
||||
// Update window information.
|
||||
d.windowEnd += n
|
||||
d.blockStart = d.windowEnd
|
||||
s.index = n
|
||||
}
|
||||
|
||||
// Try to find a match starting at index whose length is greater than prevSize.
|
||||
// We only look at chainCount possibilities before giving up.
|
||||
// pos = s.index, prevHead = s.chainHead-s.hashOffset, prevLength=minMatchLength-1, lookahead
|
||||
func (d *compressor) findMatch(pos int, prevHead int, lookahead int) (length, offset int, ok bool) {
|
||||
minMatchLook := min(lookahead, maxMatchLength)
|
||||
|
||||
win := d.window[0 : pos+minMatchLook]
|
||||
|
||||
// We quit when we get a match that's at least nice long
|
||||
nice := min(d.nice, len(win)-pos)
|
||||
|
||||
// If we've got a match that's good enough, only look in 1/4 the chain.
|
||||
tries := d.chain
|
||||
length = minMatchLength - 1
|
||||
|
||||
wEnd := win[pos+length]
|
||||
wPos := win[pos:]
|
||||
minIndex := max(pos-windowSize, 0)
|
||||
offset = 0
|
||||
|
||||
if d.chain < 100 {
|
||||
for i := prevHead; tries > 0; tries-- {
|
||||
if wEnd == win[i+length] {
|
||||
n := matchLen(win[i:i+minMatchLook], wPos)
|
||||
if n > length {
|
||||
length = n
|
||||
offset = pos - i
|
||||
ok = true
|
||||
if n >= nice {
|
||||
// The match is good enough that we don't try to find a better one.
|
||||
break
|
||||
}
|
||||
wEnd = win[pos+n]
|
||||
}
|
||||
}
|
||||
if i <= minIndex {
|
||||
// hashPrev[i & windowMask] has already been overwritten, so stop now.
|
||||
break
|
||||
}
|
||||
i = int(d.state.hashPrev[i&windowMask]) - d.state.hashOffset
|
||||
if i < minIndex {
|
||||
break
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// Minimum gain to accept a match.
|
||||
cGain := 4
|
||||
|
||||
// Some like it higher (CSV), some like it lower (JSON)
|
||||
const baseCost = 3
|
||||
// Base is 4 bytes at with an additional cost.
|
||||
// Matches must be better than this.
|
||||
|
||||
for i := prevHead; tries > 0; tries-- {
|
||||
if wEnd == win[i+length] {
|
||||
n := matchLen(win[i:i+minMatchLook], wPos)
|
||||
if n > length {
|
||||
// Calculate gain. Estimate
|
||||
newGain := d.h.bitLengthRaw(wPos[:n]) - int(offsetExtraBits[offsetCode(uint32(pos-i))]) - baseCost - int(lengthExtraBits[lengthCodes[(n-3)&255]])
|
||||
|
||||
//fmt.Println("gain:", newGain, "prev:", cGain, "raw:", d.h.bitLengthRaw(wPos[:n]), "this-len:", n, "prev-len:", length)
|
||||
if newGain > cGain {
|
||||
length = n
|
||||
offset = pos - i
|
||||
cGain = newGain
|
||||
ok = true
|
||||
if n >= nice {
|
||||
// The match is good enough that we don't try to find a better one.
|
||||
break
|
||||
}
|
||||
wEnd = win[pos+n]
|
||||
}
|
||||
}
|
||||
}
|
||||
if i <= minIndex {
|
||||
// hashPrev[i & windowMask] has already been overwritten, so stop now.
|
||||
break
|
||||
}
|
||||
i = int(d.state.hashPrev[i&windowMask]) - d.state.hashOffset
|
||||
if i < minIndex {
|
||||
break
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func (d *compressor) writeStoredBlock(buf []byte) error {
|
||||
if d.w.writeStoredHeader(len(buf), false); d.w.err != nil {
|
||||
return d.w.err
|
||||
}
|
||||
d.w.writeBytes(buf)
|
||||
return d.w.err
|
||||
}
|
||||
|
||||
// hash4 returns a hash representation of the first 4 bytes
|
||||
// of the supplied slice.
|
||||
// The caller must ensure that len(b) >= 4.
|
||||
func hash4(b []byte) uint32 {
|
||||
return hash4u(le.Load32(b, 0), hashBits)
|
||||
}
|
||||
|
||||
// hash4 returns the hash of u to fit in a hash table with h bits.
|
||||
// Preferably h should be a constant and should always be <32.
|
||||
func hash4u(u uint32, h uint8) uint32 {
|
||||
return (u * prime4bytes) >> (32 - h)
|
||||
}
|
||||
|
||||
// bulkHash4 will compute hashes using the same
|
||||
// algorithm as hash4
|
||||
func bulkHash4(b []byte, dst []uint32) {
|
||||
if len(b) < 4 {
|
||||
return
|
||||
}
|
||||
hb := le.Load32(b, 0)
|
||||
|
||||
dst[0] = hash4u(hb, hashBits)
|
||||
end := len(b) - 4 + 1
|
||||
for i := 1; i < end; i++ {
|
||||
hb = (hb >> 8) | uint32(b[i+3])<<24
|
||||
dst[i] = hash4u(hb, hashBits)
|
||||
}
|
||||
}
|
||||
|
||||
func (d *compressor) initDeflate() {
|
||||
d.window = make([]byte, 2*windowSize)
|
||||
d.byteAvailable = false
|
||||
d.err = nil
|
||||
if d.state == nil {
|
||||
return
|
||||
}
|
||||
s := d.state
|
||||
s.index = 0
|
||||
s.hashOffset = 1
|
||||
s.length = minMatchLength - 1
|
||||
s.offset = 0
|
||||
s.chainHead = -1
|
||||
}
|
||||
|
||||
// deflateLazy is the same as deflate, but with d.fastSkipHashing == skipNever,
|
||||
// meaning it always has lazy matching on.
|
||||
func (d *compressor) deflateLazy() {
|
||||
s := d.state
|
||||
// Sanity enables additional runtime tests.
|
||||
// It's intended to be used during development
|
||||
// to supplement the currently ad-hoc unit tests.
|
||||
const sanity = debugDeflate
|
||||
|
||||
if d.windowEnd-s.index < minMatchLength+maxMatchLength && !d.sync {
|
||||
return
|
||||
}
|
||||
if d.windowEnd != s.index && d.chain > 100 {
|
||||
// Get literal huffman coder.
|
||||
if d.h == nil {
|
||||
d.h = newHuffmanEncoder(maxFlateBlockTokens)
|
||||
}
|
||||
var tmp [256]uint16
|
||||
toIndex := d.window[s.index:d.windowEnd]
|
||||
toIndex = toIndex[:min(len(toIndex), maxFlateBlockTokens)]
|
||||
for _, v := range toIndex {
|
||||
tmp[v]++
|
||||
}
|
||||
d.h.generate(tmp[:], 15)
|
||||
}
|
||||
|
||||
s.maxInsertIndex = d.windowEnd - (minMatchLength - 1)
|
||||
|
||||
for {
|
||||
if sanity && s.index > d.windowEnd {
|
||||
panic("index > windowEnd")
|
||||
}
|
||||
lookahead := d.windowEnd - s.index
|
||||
if lookahead < minMatchLength+maxMatchLength {
|
||||
if !d.sync {
|
||||
return
|
||||
}
|
||||
if sanity && s.index > d.windowEnd {
|
||||
panic("index > windowEnd")
|
||||
}
|
||||
if lookahead == 0 {
|
||||
// Flush current output block if any.
|
||||
if d.byteAvailable {
|
||||
// There is still one pending token that needs to be flushed
|
||||
d.tokens.AddLiteral(d.window[s.index-1])
|
||||
d.byteAvailable = false
|
||||
}
|
||||
if d.tokens.n > 0 {
|
||||
if d.err = d.writeBlock(&d.tokens, s.index, false); d.err != nil {
|
||||
return
|
||||
}
|
||||
d.tokens.Reset()
|
||||
}
|
||||
return
|
||||
}
|
||||
}
|
||||
if s.index < s.maxInsertIndex {
|
||||
// Update the hash
|
||||
hash := hash4(d.window[s.index:])
|
||||
ch := s.hashHead[hash]
|
||||
s.chainHead = int(ch)
|
||||
s.hashPrev[s.index&windowMask] = ch
|
||||
s.hashHead[hash] = uint32(s.index + s.hashOffset)
|
||||
}
|
||||
prevLength := s.length
|
||||
prevOffset := s.offset
|
||||
s.length = minMatchLength - 1
|
||||
s.offset = 0
|
||||
minIndex := max(s.index-windowSize, 0)
|
||||
|
||||
if s.chainHead-s.hashOffset >= minIndex && lookahead > prevLength && prevLength < d.lazy {
|
||||
if newLength, newOffset, ok := d.findMatch(s.index, s.chainHead-s.hashOffset, lookahead); ok {
|
||||
s.length = newLength
|
||||
s.offset = newOffset
|
||||
}
|
||||
}
|
||||
|
||||
if prevLength >= minMatchLength && s.length <= prevLength {
|
||||
// No better match, but check for better match at end...
|
||||
//
|
||||
// Skip forward a number of bytes.
|
||||
// Offset of 2 seems to yield best results. 3 is sometimes better.
|
||||
const checkOff = 2
|
||||
|
||||
// Check all, except full length
|
||||
if prevLength < maxMatchLength-checkOff {
|
||||
prevIndex := s.index - 1
|
||||
if prevIndex+prevLength < s.maxInsertIndex {
|
||||
end := min(lookahead, maxMatchLength+checkOff)
|
||||
end += prevIndex
|
||||
|
||||
// Hash at match end.
|
||||
h := hash4(d.window[prevIndex+prevLength:])
|
||||
ch2 := int(s.hashHead[h]) - s.hashOffset - prevLength
|
||||
if prevIndex-ch2 != prevOffset && ch2 > minIndex+checkOff {
|
||||
length := matchLen(d.window[prevIndex+checkOff:end], d.window[ch2+checkOff:])
|
||||
// It seems like a pure length metric is best.
|
||||
if length > prevLength {
|
||||
prevLength = length
|
||||
prevOffset = prevIndex - ch2
|
||||
|
||||
// Extend back...
|
||||
for i := checkOff - 1; i >= 0; i-- {
|
||||
if prevLength >= maxMatchLength || d.window[prevIndex+i] != d.window[ch2+i] {
|
||||
// Emit tokens we "owe"
|
||||
for j := 0; j <= i; j++ {
|
||||
d.tokens.AddLiteral(d.window[prevIndex+j])
|
||||
if d.tokens.n == maxFlateBlockTokens {
|
||||
// The block includes the current character
|
||||
if d.err = d.writeBlock(&d.tokens, s.index, false); d.err != nil {
|
||||
return
|
||||
}
|
||||
d.tokens.Reset()
|
||||
}
|
||||
s.index++
|
||||
if s.index < s.maxInsertIndex {
|
||||
h := hash4(d.window[s.index:])
|
||||
ch := s.hashHead[h]
|
||||
s.chainHead = int(ch)
|
||||
s.hashPrev[s.index&windowMask] = ch
|
||||
s.hashHead[h] = uint32(s.index + s.hashOffset)
|
||||
}
|
||||
}
|
||||
break
|
||||
} else {
|
||||
prevLength++
|
||||
}
|
||||
}
|
||||
} else if false {
|
||||
// Check one further ahead.
|
||||
// Only rarely better, disabled for now.
|
||||
prevIndex++
|
||||
h := hash4(d.window[prevIndex+prevLength:])
|
||||
ch2 := int(s.hashHead[h]) - s.hashOffset - prevLength
|
||||
if prevIndex-ch2 != prevOffset && ch2 > minIndex+checkOff {
|
||||
length := matchLen(d.window[prevIndex+checkOff:end], d.window[ch2+checkOff:])
|
||||
// It seems like a pure length metric is best.
|
||||
if length > prevLength+checkOff {
|
||||
prevLength = length
|
||||
prevOffset = prevIndex - ch2
|
||||
prevIndex--
|
||||
|
||||
// Extend back...
|
||||
for i := checkOff; i >= 0; i-- {
|
||||
if prevLength >= maxMatchLength || d.window[prevIndex+i] != d.window[ch2+i-1] {
|
||||
// Emit tokens we "owe"
|
||||
for j := 0; j <= i; j++ {
|
||||
d.tokens.AddLiteral(d.window[prevIndex+j])
|
||||
if d.tokens.n == maxFlateBlockTokens {
|
||||
// The block includes the current character
|
||||
if d.err = d.writeBlock(&d.tokens, s.index, false); d.err != nil {
|
||||
return
|
||||
}
|
||||
d.tokens.Reset()
|
||||
}
|
||||
s.index++
|
||||
if s.index < s.maxInsertIndex {
|
||||
h := hash4(d.window[s.index:])
|
||||
ch := s.hashHead[h]
|
||||
s.chainHead = int(ch)
|
||||
s.hashPrev[s.index&windowMask] = ch
|
||||
s.hashHead[h] = uint32(s.index + s.hashOffset)
|
||||
}
|
||||
}
|
||||
break
|
||||
} else {
|
||||
prevLength++
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
// There was a match at the previous step, and the current match is
|
||||
// not better. Output the previous match.
|
||||
d.tokens.AddMatch(uint32(prevLength-3), uint32(prevOffset-minOffsetSize))
|
||||
|
||||
// Insert in the hash table all strings up to the end of the match.
|
||||
// index and index-1 are already inserted. If there is not enough
|
||||
// lookahead, the last two strings are not inserted into the hash
|
||||
// table.
|
||||
newIndex := s.index + prevLength - 1
|
||||
// Calculate missing hashes
|
||||
end := min(newIndex, s.maxInsertIndex)
|
||||
end += minMatchLength - 1
|
||||
startindex := min(s.index+1, s.maxInsertIndex)
|
||||
tocheck := d.window[startindex:end]
|
||||
dstSize := len(tocheck) - minMatchLength + 1
|
||||
if dstSize > 0 {
|
||||
dst := s.hashMatch[:dstSize]
|
||||
bulkHash4(tocheck, dst)
|
||||
var newH uint32
|
||||
for i, val := range dst {
|
||||
di := i + startindex
|
||||
newH = val & hashMask
|
||||
// Get previous value with the same hash.
|
||||
// Our chain should point to the previous value.
|
||||
s.hashPrev[di&windowMask] = s.hashHead[newH]
|
||||
// Set the head of the hash chain to us.
|
||||
s.hashHead[newH] = uint32(di + s.hashOffset)
|
||||
}
|
||||
}
|
||||
|
||||
s.index = newIndex
|
||||
d.byteAvailable = false
|
||||
s.length = minMatchLength - 1
|
||||
if d.tokens.n == maxFlateBlockTokens {
|
||||
// The block includes the current character
|
||||
if d.err = d.writeBlock(&d.tokens, s.index, false); d.err != nil {
|
||||
return
|
||||
}
|
||||
d.tokens.Reset()
|
||||
}
|
||||
s.ii = 0
|
||||
} else {
|
||||
// Reset, if we got a match this run.
|
||||
if s.length >= minMatchLength {
|
||||
s.ii = 0
|
||||
}
|
||||
// We have a byte waiting. Emit it.
|
||||
if d.byteAvailable {
|
||||
s.ii++
|
||||
d.tokens.AddLiteral(d.window[s.index-1])
|
||||
if d.tokens.n == maxFlateBlockTokens {
|
||||
if d.err = d.writeBlock(&d.tokens, s.index, false); d.err != nil {
|
||||
return
|
||||
}
|
||||
d.tokens.Reset()
|
||||
}
|
||||
s.index++
|
||||
|
||||
// If we have a long run of no matches, skip additional bytes
|
||||
// Resets when s.ii overflows after 64KB.
|
||||
if n := int(s.ii) - d.chain; n > 0 {
|
||||
n = 1 + int(n>>6)
|
||||
for j := 0; j < n; j++ {
|
||||
if s.index >= d.windowEnd-1 {
|
||||
break
|
||||
}
|
||||
d.tokens.AddLiteral(d.window[s.index-1])
|
||||
if d.tokens.n == maxFlateBlockTokens {
|
||||
if d.err = d.writeBlock(&d.tokens, s.index, false); d.err != nil {
|
||||
return
|
||||
}
|
||||
d.tokens.Reset()
|
||||
}
|
||||
// Index...
|
||||
if s.index < s.maxInsertIndex {
|
||||
h := hash4(d.window[s.index:])
|
||||
ch := s.hashHead[h]
|
||||
s.chainHead = int(ch)
|
||||
s.hashPrev[s.index&windowMask] = ch
|
||||
s.hashHead[h] = uint32(s.index + s.hashOffset)
|
||||
}
|
||||
s.index++
|
||||
}
|
||||
// Flush last byte
|
||||
d.tokens.AddLiteral(d.window[s.index-1])
|
||||
d.byteAvailable = false
|
||||
// s.length = minMatchLength - 1 // not needed, since s.ii is reset above, so it should never be > minMatchLength
|
||||
if d.tokens.n == maxFlateBlockTokens {
|
||||
if d.err = d.writeBlock(&d.tokens, s.index, false); d.err != nil {
|
||||
return
|
||||
}
|
||||
d.tokens.Reset()
|
||||
}
|
||||
}
|
||||
} else {
|
||||
s.index++
|
||||
d.byteAvailable = true
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (d *compressor) store() {
|
||||
if d.windowEnd > 0 && (d.windowEnd == maxStoreBlockSize || d.sync) {
|
||||
d.err = d.writeStoredBlock(d.window[:d.windowEnd])
|
||||
d.windowEnd = 0
|
||||
}
|
||||
}
|
||||
|
||||
// fillWindow will fill the buffer with data for huffman-only compression.
|
||||
// The number of bytes copied is returned.
|
||||
func (d *compressor) fillBlock(b []byte) int {
|
||||
n := copy(d.window[d.windowEnd:], b)
|
||||
d.windowEnd += n
|
||||
return n
|
||||
}
|
||||
|
||||
// storeHuff will compress and store the currently added data,
|
||||
// if enough has been accumulated or we at the end of the stream.
|
||||
// Any error that occurred will be in d.err
|
||||
func (d *compressor) storeHuff() {
|
||||
if d.windowEnd < len(d.window) && !d.sync || d.windowEnd == 0 {
|
||||
return
|
||||
}
|
||||
d.w.writeBlockHuff(false, d.window[:d.windowEnd], d.sync)
|
||||
d.err = d.w.err
|
||||
d.windowEnd = 0
|
||||
}
|
||||
|
||||
// storeFast will compress and store the currently added data,
|
||||
// if enough has been accumulated or we at the end of the stream.
|
||||
// Any error that occurred will be in d.err
|
||||
func (d *compressor) storeFast() {
|
||||
// We only compress if we have maxStoreBlockSize.
|
||||
if d.windowEnd < len(d.window) {
|
||||
if !d.sync {
|
||||
return
|
||||
}
|
||||
// Handle extremely small sizes.
|
||||
if d.windowEnd < 128 {
|
||||
if d.windowEnd == 0 {
|
||||
return
|
||||
}
|
||||
if d.windowEnd <= 32 {
|
||||
d.err = d.writeStoredBlock(d.window[:d.windowEnd])
|
||||
} else {
|
||||
d.w.writeBlockHuff(false, d.window[:d.windowEnd], true)
|
||||
d.err = d.w.err
|
||||
}
|
||||
d.tokens.Reset()
|
||||
d.windowEnd = 0
|
||||
d.fast.Reset()
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
d.fast.Encode(&d.tokens, d.window[:d.windowEnd])
|
||||
// If we made zero matches, store the block as is.
|
||||
if d.tokens.n == 0 {
|
||||
d.err = d.writeStoredBlock(d.window[:d.windowEnd])
|
||||
// If we removed less than 1/16th, huffman compress the block.
|
||||
} else if int(d.tokens.n) > d.windowEnd-(d.windowEnd>>4) {
|
||||
d.w.writeBlockHuff(false, d.window[:d.windowEnd], d.sync)
|
||||
d.err = d.w.err
|
||||
} else {
|
||||
d.w.writeBlockDynamic(&d.tokens, false, d.window[:d.windowEnd], d.sync)
|
||||
d.err = d.w.err
|
||||
}
|
||||
d.tokens.Reset()
|
||||
d.windowEnd = 0
|
||||
}
|
||||
|
||||
// write will add input byte to the stream.
|
||||
// Unless an error occurs all bytes will be consumed.
|
||||
func (d *compressor) write(b []byte) (n int, err error) {
|
||||
if d.err != nil {
|
||||
return 0, d.err
|
||||
}
|
||||
n = len(b)
|
||||
for len(b) > 0 {
|
||||
if d.windowEnd == len(d.window) || d.sync {
|
||||
d.step(d)
|
||||
}
|
||||
b = b[d.fill(d, b):]
|
||||
if d.err != nil {
|
||||
return 0, d.err
|
||||
}
|
||||
}
|
||||
return n, d.err
|
||||
}
|
||||
|
||||
func (d *compressor) syncFlush() error {
|
||||
d.sync = true
|
||||
if d.err != nil {
|
||||
return d.err
|
||||
}
|
||||
d.step(d)
|
||||
if d.err == nil {
|
||||
d.w.writeStoredHeader(0, false)
|
||||
d.w.flush()
|
||||
d.err = d.w.err
|
||||
}
|
||||
d.sync = false
|
||||
return d.err
|
||||
}
|
||||
|
||||
func (d *compressor) init(w io.Writer, level int) (err error) {
|
||||
d.w = newHuffmanBitWriter(w)
|
||||
|
||||
switch {
|
||||
case level == NoCompression:
|
||||
d.window = make([]byte, maxStoreBlockSize)
|
||||
d.fill = (*compressor).fillBlock
|
||||
d.step = (*compressor).store
|
||||
case level == ConstantCompression:
|
||||
d.w.logNewTablePenalty = 10
|
||||
d.window = make([]byte, 32<<10)
|
||||
d.fill = (*compressor).fillBlock
|
||||
d.step = (*compressor).storeHuff
|
||||
case level == DefaultCompression:
|
||||
level = 5
|
||||
fallthrough
|
||||
case level >= 1 && level <= 6:
|
||||
d.w.logNewTablePenalty = 7
|
||||
d.fast = newFastEnc(level)
|
||||
d.window = make([]byte, maxStoreBlockSize)
|
||||
d.fill = (*compressor).fillBlock
|
||||
d.step = (*compressor).storeFast
|
||||
case 7 <= level && level <= 9:
|
||||
d.w.logNewTablePenalty = 8
|
||||
d.state = &advancedState{}
|
||||
d.compressionLevel = levels[level]
|
||||
d.initDeflate()
|
||||
d.fill = (*compressor).fillDeflate
|
||||
d.step = (*compressor).deflateLazy
|
||||
case -level >= MinCustomWindowSize && -level <= MaxCustomWindowSize:
|
||||
d.w.logNewTablePenalty = 7
|
||||
d.fast = &fastEncL5Window{maxOffset: int32(-level), cur: maxStoreBlockSize}
|
||||
d.window = make([]byte, maxStoreBlockSize)
|
||||
d.fill = (*compressor).fillBlock
|
||||
d.step = (*compressor).storeFast
|
||||
default:
|
||||
return fmt.Errorf("flate: invalid compression level %d: want value in range [-2, 9]", level)
|
||||
}
|
||||
d.level = level
|
||||
return nil
|
||||
}
|
||||
|
||||
// reset the state of the compressor.
|
||||
func (d *compressor) reset(w io.Writer) {
|
||||
d.w.reset(w)
|
||||
d.sync = false
|
||||
d.err = nil
|
||||
// We only need to reset a few things for Snappy.
|
||||
if d.fast != nil {
|
||||
d.fast.Reset()
|
||||
d.windowEnd = 0
|
||||
d.tokens.Reset()
|
||||
return
|
||||
}
|
||||
switch d.compressionLevel.chain {
|
||||
case 0:
|
||||
// level was NoCompression or ConstantCompression.
|
||||
d.windowEnd = 0
|
||||
default:
|
||||
s := d.state
|
||||
s.chainHead = -1
|
||||
for i := range s.hashHead {
|
||||
s.hashHead[i] = 0
|
||||
}
|
||||
for i := range s.hashPrev {
|
||||
s.hashPrev[i] = 0
|
||||
}
|
||||
s.hashOffset = 1
|
||||
s.index, d.windowEnd = 0, 0
|
||||
d.blockStart, d.byteAvailable = 0, false
|
||||
d.tokens.Reset()
|
||||
s.length = minMatchLength - 1
|
||||
s.offset = 0
|
||||
s.ii = 0
|
||||
s.maxInsertIndex = 0
|
||||
}
|
||||
}
|
||||
|
||||
func (d *compressor) close() error {
|
||||
if d.err != nil {
|
||||
return d.err
|
||||
}
|
||||
d.sync = true
|
||||
d.step(d)
|
||||
if d.err != nil {
|
||||
return d.err
|
||||
}
|
||||
if d.w.writeStoredHeader(0, true); d.w.err != nil {
|
||||
return d.w.err
|
||||
}
|
||||
d.w.flush()
|
||||
d.w.reset(nil)
|
||||
return d.w.err
|
||||
}
|
||||
|
||||
// NewWriter returns a new Writer compressing data at the given level.
|
||||
// Following zlib, levels range from 1 (BestSpeed) to 9 (BestCompression);
|
||||
// higher levels typically run slower but compress more.
|
||||
// Level 0 (NoCompression) does not attempt any compression; it only adds the
|
||||
// necessary DEFLATE framing.
|
||||
// Level -1 (DefaultCompression) uses the default compression level.
|
||||
// Level -2 (ConstantCompression) will use Huffman compression only, giving
|
||||
// a very fast compression for all types of input, but sacrificing considerable
|
||||
// compression efficiency.
|
||||
//
|
||||
// If level is in the range [-2, 9] then the error returned will be nil.
|
||||
// Otherwise the error returned will be non-nil.
|
||||
func NewWriter(w io.Writer, level int) (*Writer, error) {
|
||||
var dw Writer
|
||||
if err := dw.d.init(w, level); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &dw, nil
|
||||
}
|
||||
|
||||
// NewWriterDict is like NewWriter but initializes the new
|
||||
// Writer with a preset dictionary. The returned Writer behaves
|
||||
// as if the dictionary had been written to it without producing
|
||||
// any compressed output. The compressed data written to w
|
||||
// can only be decompressed by a Reader initialized with the
|
||||
// same dictionary.
|
||||
func NewWriterDict(w io.Writer, level int, dict []byte) (*Writer, error) {
|
||||
zw, err := NewWriter(w, level)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
zw.d.fillWindow(dict)
|
||||
zw.dict = append(zw.dict, dict...) // duplicate dictionary for Reset method.
|
||||
return zw, err
|
||||
}
|
||||
|
||||
// MinCustomWindowSize is the minimum window size that can be sent to NewWriterWindow.
|
||||
const MinCustomWindowSize = 32
|
||||
|
||||
// MaxCustomWindowSize is the maximum custom window that can be sent to NewWriterWindow.
|
||||
const MaxCustomWindowSize = windowSize
|
||||
|
||||
// NewWriterWindow returns a new Writer compressing data with a custom window size.
|
||||
// windowSize must be from MinCustomWindowSize to MaxCustomWindowSize.
|
||||
func NewWriterWindow(w io.Writer, windowSize int) (*Writer, error) {
|
||||
if windowSize < MinCustomWindowSize {
|
||||
return nil, errors.New("flate: requested window size less than MinWindowSize")
|
||||
}
|
||||
if windowSize > MaxCustomWindowSize {
|
||||
return nil, errors.New("flate: requested window size bigger than MaxCustomWindowSize")
|
||||
}
|
||||
var dw Writer
|
||||
if err := dw.d.init(w, -windowSize); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &dw, nil
|
||||
}
|
||||
|
||||
// A Writer takes data written to it and writes the compressed
|
||||
// form of that data to an underlying writer (see NewWriter).
|
||||
type Writer struct {
|
||||
d compressor
|
||||
dict []byte
|
||||
}
|
||||
|
||||
// Write writes data to w, which will eventually write the
|
||||
// compressed form of data to its underlying writer.
|
||||
func (w *Writer) Write(data []byte) (n int, err error) {
|
||||
return w.d.write(data)
|
||||
}
|
||||
|
||||
// Flush flushes any pending data to the underlying writer.
|
||||
// It is useful mainly in compressed network protocols, to ensure that
|
||||
// a remote reader has enough data to reconstruct a packet.
|
||||
// Flush does not return until the data has been written.
|
||||
// Calling Flush when there is no pending data still causes the Writer
|
||||
// to emit a sync marker of at least 4 bytes.
|
||||
// If the underlying writer returns an error, Flush returns that error.
|
||||
//
|
||||
// In the terminology of the zlib library, Flush is equivalent to Z_SYNC_FLUSH.
|
||||
func (w *Writer) Flush() error {
|
||||
// For more about flushing:
|
||||
// http://www.bolet.org/~pornin/deflate-flush.html
|
||||
return w.d.syncFlush()
|
||||
}
|
||||
|
||||
// Close flushes and closes the writer.
|
||||
func (w *Writer) Close() error {
|
||||
return w.d.close()
|
||||
}
|
||||
|
||||
// Reset discards the writer's state and makes it equivalent to
|
||||
// the result of NewWriter or NewWriterDict called with dst
|
||||
// and w's level and dictionary.
|
||||
func (w *Writer) Reset(dst io.Writer) {
|
||||
if len(w.dict) > 0 {
|
||||
// w was created with NewWriterDict
|
||||
w.d.reset(dst)
|
||||
if dst != nil {
|
||||
w.d.fillWindow(w.dict)
|
||||
}
|
||||
} else {
|
||||
// w was created with NewWriter
|
||||
w.d.reset(dst)
|
||||
}
|
||||
}
|
||||
|
||||
// ResetDict discards the writer's state and makes it equivalent to
|
||||
// the result of NewWriter or NewWriterDict called with dst
|
||||
// and w's level, but sets a specific dictionary.
|
||||
func (w *Writer) ResetDict(dst io.Writer, dict []byte) {
|
||||
w.dict = dict
|
||||
w.d.reset(dst)
|
||||
w.d.fillWindow(w.dict)
|
||||
}
|
||||
-199
@@ -1,199 +0,0 @@
|
||||
// Copyright 2016 The Go Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package flate
|
||||
|
||||
// dictDecoder implements the LZ77 sliding dictionary as used in decompression.
|
||||
// LZ77 decompresses data through sequences of two forms of commands:
|
||||
//
|
||||
// - Literal insertions: Runs of one or more symbols are inserted into the data
|
||||
// stream as is. This is accomplished through the writeByte method for a
|
||||
// single symbol, or combinations of writeSlice/writeMark for multiple symbols.
|
||||
// Any valid stream must start with a literal insertion if no preset dictionary
|
||||
// is used.
|
||||
//
|
||||
// - Backward copies: Runs of one or more symbols are copied from previously
|
||||
// emitted data. Backward copies come as the tuple (dist, length) where dist
|
||||
// determines how far back in the stream to copy from and length determines how
|
||||
// many bytes to copy. Note that it is valid for the length to be greater than
|
||||
// the distance. Since LZ77 uses forward copies, that situation is used to
|
||||
// perform a form of run-length encoding on repeated runs of symbols.
|
||||
// The writeCopy and tryWriteCopy are used to implement this command.
|
||||
//
|
||||
// For performance reasons, this implementation performs little to no sanity
|
||||
// checks about the arguments. As such, the invariants documented for each
|
||||
// method call must be respected.
|
||||
type dictDecoder struct {
|
||||
hist []byte // Sliding window history
|
||||
|
||||
// Invariant: 0 <= rdPos <= wrPos <= len(hist)
|
||||
wrPos int // Current output position in buffer
|
||||
rdPos int // Have emitted hist[:rdPos] already
|
||||
flushed int64 // Total bytes returned by readFlush since init
|
||||
full bool // Has a full window length been written yet?
|
||||
}
|
||||
|
||||
// init initializes dictDecoder to have a sliding window dictionary of the given
|
||||
// size. If a preset dict is provided, it will initialize the dictionary with
|
||||
// the contents of dict.
|
||||
func (dd *dictDecoder) init(size int, dict []byte) {
|
||||
*dd = dictDecoder{hist: dd.hist}
|
||||
|
||||
if cap(dd.hist) < size {
|
||||
dd.hist = make([]byte, size)
|
||||
}
|
||||
dd.hist = dd.hist[:size]
|
||||
|
||||
if len(dict) > len(dd.hist) {
|
||||
dict = dict[len(dict)-len(dd.hist):]
|
||||
}
|
||||
dd.wrPos = copy(dd.hist, dict)
|
||||
if dd.wrPos == len(dd.hist) {
|
||||
dd.wrPos = 0
|
||||
dd.full = true
|
||||
}
|
||||
dd.rdPos = dd.wrPos
|
||||
}
|
||||
|
||||
// histSize reports the total amount of historical data in the dictionary.
|
||||
func (dd *dictDecoder) histSize() int {
|
||||
if dd.full {
|
||||
return len(dd.hist)
|
||||
}
|
||||
return dd.wrPos
|
||||
}
|
||||
|
||||
// availRead reports the number of bytes that can be flushed by readFlush.
|
||||
func (dd *dictDecoder) availRead() int {
|
||||
return dd.wrPos - dd.rdPos
|
||||
}
|
||||
|
||||
// availWrite reports the available amount of output buffer space.
|
||||
func (dd *dictDecoder) availWrite() int {
|
||||
return len(dd.hist) - dd.wrPos
|
||||
}
|
||||
|
||||
// writeSlice returns a slice of the available buffer to write data to.
|
||||
//
|
||||
// This invariant will be kept: len(s) <= availWrite()
|
||||
func (dd *dictDecoder) writeSlice() []byte {
|
||||
return dd.hist[dd.wrPos:]
|
||||
}
|
||||
|
||||
// writeMark advances the writer pointer by cnt.
|
||||
//
|
||||
// This invariant must be kept: 0 <= cnt <= availWrite()
|
||||
func (dd *dictDecoder) writeMark(cnt int) {
|
||||
dd.wrPos += cnt
|
||||
}
|
||||
|
||||
// writeByte writes a single byte to the dictionary.
|
||||
//
|
||||
// This invariant must be kept: 0 < availWrite()
|
||||
func (dd *dictDecoder) writeByte(c byte) {
|
||||
dd.hist[dd.wrPos] = c
|
||||
dd.wrPos++
|
||||
}
|
||||
|
||||
// writeCopy copies a string at a given (dist, length) to the output.
|
||||
// This returns the number of bytes copied and may be less than the requested
|
||||
// length if the available space in the output buffer is too small.
|
||||
//
|
||||
// This invariant must be kept: 0 < dist <= histSize()
|
||||
func (dd *dictDecoder) writeCopy(dist, length int) int {
|
||||
dstBase := dd.wrPos
|
||||
dstPos := dstBase
|
||||
srcPos := dstPos - dist
|
||||
endPos := min(dstPos+length, len(dd.hist))
|
||||
|
||||
// Copy non-overlapping section after destination position.
|
||||
//
|
||||
// This section is non-overlapping in that the copy length for this section
|
||||
// is always less than or equal to the backwards distance. This can occur
|
||||
// if a distance refers to data that wraps-around in the buffer.
|
||||
// Thus, a backwards copy is performed here; that is, the exact bytes in
|
||||
// the source prior to the copy is placed in the destination.
|
||||
if srcPos < 0 {
|
||||
srcPos += len(dd.hist)
|
||||
dstPos += copy(dd.hist[dstPos:endPos], dd.hist[srcPos:])
|
||||
srcPos = 0
|
||||
}
|
||||
|
||||
// Copy possibly overlapping section before destination position.
|
||||
//
|
||||
// This section can overlap if the copy length for this section is larger
|
||||
// than the backwards distance. This is allowed by LZ77 so that repeated
|
||||
// strings can be succinctly represented using (dist, length) pairs.
|
||||
// Thus, a forwards copy is performed here; that is, the bytes copied is
|
||||
// possibly dependent on the resulting bytes in the destination as the copy
|
||||
// progresses along. This is functionally equivalent to the following:
|
||||
//
|
||||
// for i := 0; i < endPos-dstPos; i++ {
|
||||
// dd.hist[dstPos+i] = dd.hist[srcPos+i]
|
||||
// }
|
||||
// dstPos = endPos
|
||||
//
|
||||
for dstPos < endPos {
|
||||
dstPos += copy(dd.hist[dstPos:endPos], dd.hist[srcPos:dstPos])
|
||||
}
|
||||
|
||||
dd.wrPos = dstPos
|
||||
return dstPos - dstBase
|
||||
}
|
||||
|
||||
// tryWriteCopy tries to copy a string at a given (distance, length) to the
|
||||
// output. This specialized version is optimized for short distances.
|
||||
//
|
||||
// This method is designed to be inlined for performance reasons.
|
||||
//
|
||||
// This invariant must be kept: 0 < dist <= histSize()
|
||||
func (dd *dictDecoder) tryWriteCopy(dist, length int) int {
|
||||
dstPos := dd.wrPos
|
||||
endPos := dstPos + length
|
||||
if dstPos < dist || endPos > len(dd.hist) {
|
||||
return 0
|
||||
}
|
||||
dstBase := dstPos
|
||||
srcPos := dstPos - dist
|
||||
|
||||
// Copy possibly overlapping section before destination position.
|
||||
loop:
|
||||
dstPos += copy(dd.hist[dstPos:endPos], dd.hist[srcPos:dstPos])
|
||||
if dstPos < endPos {
|
||||
goto loop // Avoid for-loop so that this function can be inlined
|
||||
}
|
||||
|
||||
dd.wrPos = dstPos
|
||||
return dstPos - dstBase
|
||||
}
|
||||
|
||||
// appendWindow appends the current sliding window (up to len(hist) most recent
|
||||
// bytes, oldest first) to dst.
|
||||
func (dd *dictDecoder) appendWindow(dst []byte) []byte {
|
||||
if dd.full {
|
||||
dst = append(dst, dd.hist[dd.wrPos:]...)
|
||||
return append(dst, dd.hist[:dd.wrPos]...)
|
||||
}
|
||||
return append(dst, dd.hist[:dd.wrPos]...)
|
||||
}
|
||||
|
||||
// readFlush returns a slice of the historical buffer that is ready to be
|
||||
// emitted to the user. The data returned by readFlush must be fully consumed
|
||||
// before calling any other dictDecoder methods.
|
||||
func (dd *dictDecoder) readFlush() []byte {
|
||||
toRead := dd.hist[dd.rdPos:dd.wrPos]
|
||||
dd.flushed += int64(len(toRead))
|
||||
dd.rdPos = dd.wrPos
|
||||
if dd.wrPos == len(dd.hist) {
|
||||
dd.wrPos, dd.rdPos = 0, 0
|
||||
dd.full = true
|
||||
}
|
||||
return toRead
|
||||
}
|
||||
|
||||
// decoded reports the total number of bytes written into the dictionary since
|
||||
// init (i.e. excluding any preset dict bytes).
|
||||
func (dd *dictDecoder) decoded() int64 {
|
||||
return dd.flushed + int64(dd.wrPos-dd.rdPos)
|
||||
}
|
||||
-190
@@ -1,190 +0,0 @@
|
||||
// Copyright 2011 The Snappy-Go Authors. All rights reserved.
|
||||
// Modified for deflate by Klaus Post (c) 2015.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package flate
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"github.com/klauspost/compress/internal/le"
|
||||
"github.com/klauspost/compress/internal/regmask"
|
||||
)
|
||||
|
||||
type fastEnc interface {
|
||||
Encode(dst *tokens, src []byte)
|
||||
Reset()
|
||||
}
|
||||
|
||||
func newFastEnc(level int) fastEnc {
|
||||
switch level {
|
||||
case 1:
|
||||
return &fastEncL1{fastGen: fastGen{cur: maxStoreBlockSize}}
|
||||
case 2:
|
||||
return &fastEncL2{fastGen: fastGen{cur: maxStoreBlockSize}}
|
||||
case 3:
|
||||
return &fastEncL3{fastGen: fastGen{cur: maxStoreBlockSize}}
|
||||
case 4:
|
||||
return &fastEncL4{fastGen: fastGen{cur: maxStoreBlockSize}}
|
||||
case 5:
|
||||
return &fastEncL5{fastGen: fastGen{cur: maxStoreBlockSize}}
|
||||
case 6:
|
||||
return &fastEncL6{fastGen: fastGen{cur: maxStoreBlockSize}}
|
||||
default:
|
||||
panic("invalid level specified")
|
||||
}
|
||||
}
|
||||
|
||||
const (
|
||||
tableBits = 15 // Bits used in the table
|
||||
tableSize = 1 << tableBits // Size of the table
|
||||
tableShift = 32 - tableBits // Right-shift to get the tableBits most significant bits of a uint32.
|
||||
baseMatchOffset = 1 // The smallest match offset
|
||||
baseMatchLength = 3 // The smallest match length per the RFC section 3.2.5
|
||||
maxMatchOffset = 1 << 15 // The largest match offset
|
||||
|
||||
bTableBits = 17 // Bits used in the big tables
|
||||
bTableSize = 1 << bTableBits // Size of the table
|
||||
allocHistory = maxStoreBlockSize * 5 // Size to preallocate for history.
|
||||
bufferReset = (1 << 31) - allocHistory - maxStoreBlockSize - 1 // Reset the buffer offset when reaching this.
|
||||
)
|
||||
|
||||
const (
|
||||
prime3bytes = 506832829
|
||||
prime4bytes = 2654435761
|
||||
prime5bytes = 889523592379
|
||||
prime6bytes = 227718039650203
|
||||
prime7bytes = 58295818150454627
|
||||
prime8bytes = 0xcf1bbcdcb7a56463
|
||||
)
|
||||
|
||||
func load3232(b []byte, i int32) uint32 {
|
||||
return le.Load32(b, i)
|
||||
}
|
||||
|
||||
func load6432(b []byte, i int32) uint64 {
|
||||
return le.Load64(b, i)
|
||||
}
|
||||
|
||||
type tableEntry struct {
|
||||
offset int32
|
||||
}
|
||||
|
||||
// fastGen maintains the table for matches,
|
||||
// and the previous byte block for level 2.
|
||||
// This is the generic implementation.
|
||||
type fastGen struct {
|
||||
hist []byte
|
||||
cur int32
|
||||
}
|
||||
|
||||
func (e *fastGen) addBlock(src []byte) int32 {
|
||||
// check if we have space already
|
||||
if len(e.hist)+len(src) > cap(e.hist) {
|
||||
if cap(e.hist) == 0 {
|
||||
e.hist = make([]byte, 0, allocHistory)
|
||||
} else {
|
||||
if cap(e.hist) < maxMatchOffset*2 {
|
||||
panic("unexpected buffer size")
|
||||
}
|
||||
// Move down
|
||||
offset := int32(len(e.hist)) - maxMatchOffset
|
||||
// copy(e.hist[0:maxMatchOffset], e.hist[offset:])
|
||||
*(*[maxMatchOffset]byte)(e.hist) = *(*[maxMatchOffset]byte)(e.hist[offset:])
|
||||
e.cur += offset
|
||||
e.hist = e.hist[:maxMatchOffset]
|
||||
}
|
||||
}
|
||||
s := int32(len(e.hist))
|
||||
e.hist = append(e.hist, src...)
|
||||
return s
|
||||
}
|
||||
|
||||
type tableEntryPrev struct {
|
||||
Cur tableEntry
|
||||
Prev tableEntry
|
||||
}
|
||||
|
||||
// hash7 returns the hash of the lowest 7 bytes of u to fit in a hash table with h bits.
|
||||
// Preferably h should be a constant and should always be <64.
|
||||
func hash7(u uint64, h uint8) uint32 {
|
||||
return uint32(((u << (64 - 56)) * prime7bytes) >> ((64 - h) & regmask.Shift64ByUint8))
|
||||
}
|
||||
|
||||
// hashLen returns a hash of the lowest mls bytes of with length output bits.
|
||||
// mls must be >=3 and <=8. Any other value will return hash for 4 bytes.
|
||||
// length should always be < 32.
|
||||
// Preferably length and mls should be a constant for inlining.
|
||||
func hashLen(u uint64, length, mls uint8) uint32 {
|
||||
switch mls {
|
||||
case 3:
|
||||
return (uint32(u<<8) * prime3bytes) >> (32 - length)
|
||||
case 5:
|
||||
return uint32(((u << (64 - 40)) * prime5bytes) >> (64 - length))
|
||||
case 6:
|
||||
return uint32(((u << (64 - 48)) * prime6bytes) >> (64 - length))
|
||||
case 7:
|
||||
return uint32(((u << (64 - 56)) * prime7bytes) >> (64 - length))
|
||||
case 8:
|
||||
return uint32((u * prime8bytes) >> (64 - length))
|
||||
default:
|
||||
return (uint32(u) * prime4bytes) >> (32 - length)
|
||||
}
|
||||
}
|
||||
|
||||
// matchlen will return the match length between offsets and t in src.
|
||||
// The maximum length returned is maxMatchLength - 4.
|
||||
// It is assumed that s > t, that t >=0 and s < len(src).
|
||||
func (e *fastGen) matchlen(s, t int, src []byte) int32 {
|
||||
if debugDeflate {
|
||||
if t >= s {
|
||||
panic(fmt.Sprint("t >=s:", t, s))
|
||||
}
|
||||
if int(s) >= len(src) {
|
||||
panic(fmt.Sprint("s >= len(src):", s, len(src)))
|
||||
}
|
||||
if t < 0 {
|
||||
panic(fmt.Sprint("t < 0:", t))
|
||||
}
|
||||
if s-t > maxMatchOffset {
|
||||
panic(fmt.Sprint(s, "-", t, "(", s-t, ") > maxMatchLength (", maxMatchOffset, ")"))
|
||||
}
|
||||
}
|
||||
a := src[s:min(s+maxMatchLength-4, len(src))]
|
||||
b := src[t:]
|
||||
return int32(matchLen(a, b))
|
||||
}
|
||||
|
||||
// matchlenLong will return the match length between offsets and t in src.
|
||||
// It is assumed that s > t, that t >=0 and s < len(src).
|
||||
func (e *fastGen) matchlenLong(s, t int, src []byte) int32 {
|
||||
if debugDeflate {
|
||||
if t >= s {
|
||||
panic(fmt.Sprint("t >=s:", t, s))
|
||||
}
|
||||
if int(s) >= len(src) {
|
||||
panic(fmt.Sprint("s >= len(src):", s, len(src)))
|
||||
}
|
||||
if t < 0 {
|
||||
panic(fmt.Sprint("t < 0:", t))
|
||||
}
|
||||
if s-t > maxMatchOffset {
|
||||
panic(fmt.Sprint(s, "-", t, "(", s-t, ") > maxMatchLength (", maxMatchOffset, ")"))
|
||||
}
|
||||
}
|
||||
return int32(matchLen(src[s:], src[t:]))
|
||||
}
|
||||
|
||||
// Reset the encoding table.
|
||||
func (e *fastGen) Reset() {
|
||||
if cap(e.hist) < allocHistory {
|
||||
e.hist = make([]byte, 0, allocHistory)
|
||||
}
|
||||
// We offset current position so everything will be out of reach.
|
||||
// If we are above the buffer reset it will be cleared anyway since len(hist) == 0.
|
||||
if e.cur <= bufferReset {
|
||||
e.cur += maxMatchOffset + int32(len(e.hist))
|
||||
}
|
||||
e.hist = e.hist[:0]
|
||||
}
|
||||
-1174
File diff suppressed because it is too large
Load Diff
-417
@@ -1,417 +0,0 @@
|
||||
// Copyright 2009 The Go Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package flate
|
||||
|
||||
import (
|
||||
"math"
|
||||
"math/bits"
|
||||
)
|
||||
|
||||
const (
|
||||
maxBitsLimit = 16
|
||||
// number of valid literals
|
||||
literalCount = 286
|
||||
)
|
||||
|
||||
// hcode is a huffman code with a bit code and bit length.
|
||||
type hcode uint32
|
||||
|
||||
func (h hcode) len() uint8 {
|
||||
return uint8(h)
|
||||
}
|
||||
|
||||
func (h hcode) code64() uint64 {
|
||||
return uint64(h >> 8)
|
||||
}
|
||||
|
||||
func (h hcode) zero() bool {
|
||||
return h == 0
|
||||
}
|
||||
|
||||
type huffmanEncoder struct {
|
||||
codes []hcode
|
||||
bitCount [17]int32
|
||||
|
||||
// Allocate a reusable buffer with the longest possible frequency table.
|
||||
// Possible lengths are codegenCodeCount, offsetCodeCount and literalCount.
|
||||
// The largest of these is literalCount, so we allocate for that case.
|
||||
freqcache [literalCount + 1]literalNode
|
||||
}
|
||||
|
||||
type literalNode struct {
|
||||
literal uint16
|
||||
freq uint16
|
||||
}
|
||||
|
||||
// A levelInfo describes the state of the constructed tree for a given depth.
|
||||
type levelInfo struct {
|
||||
// Our level. for better printing
|
||||
level int32
|
||||
|
||||
// The frequency of the last node at this level
|
||||
lastFreq int32
|
||||
|
||||
// The frequency of the next character to add to this level
|
||||
nextCharFreq int32
|
||||
|
||||
// The frequency of the next pair (from level below) to add to this level.
|
||||
// Only valid if the "needed" value of the next lower level is 0.
|
||||
nextPairFreq int32
|
||||
|
||||
// The number of chains remaining to generate for this level before moving
|
||||
// up to the next level
|
||||
needed int32
|
||||
}
|
||||
|
||||
// set sets the code and length of an hcode.
|
||||
func (h *hcode) set(code uint16, length uint8) {
|
||||
*h = hcode(length) | (hcode(code) << 8)
|
||||
}
|
||||
|
||||
func newhcode(code uint16, length uint8) hcode {
|
||||
return hcode(length) | (hcode(code) << 8)
|
||||
}
|
||||
|
||||
func reverseBits(number uint16, bitLength byte) uint16 {
|
||||
return bits.Reverse16(number << ((16 - bitLength) & 15))
|
||||
}
|
||||
|
||||
func maxNode() literalNode { return literalNode{math.MaxUint16, math.MaxUint16} }
|
||||
|
||||
func newHuffmanEncoder(size int) *huffmanEncoder {
|
||||
// Make capacity to next power of two.
|
||||
c := uint(bits.Len32(uint32(size - 1)))
|
||||
return &huffmanEncoder{codes: make([]hcode, size, 1<<c)}
|
||||
}
|
||||
|
||||
// Generates a HuffmanCode corresponding to the fixed literal table
|
||||
func generateFixedLiteralEncoding() *huffmanEncoder {
|
||||
h := newHuffmanEncoder(literalCount)
|
||||
codes := h.codes
|
||||
var ch uint16
|
||||
for ch = range uint16(literalCount) {
|
||||
var bits uint16
|
||||
var size uint8
|
||||
switch {
|
||||
case ch < 144:
|
||||
// size 8, 000110000 .. 10111111
|
||||
bits = ch + 48
|
||||
size = 8
|
||||
case ch < 256:
|
||||
// size 9, 110010000 .. 111111111
|
||||
bits = ch + 400 - 144
|
||||
size = 9
|
||||
case ch < 280:
|
||||
// size 7, 0000000 .. 0010111
|
||||
bits = ch - 256
|
||||
size = 7
|
||||
default:
|
||||
// size 8, 11000000 .. 11000111
|
||||
bits = ch + 192 - 280
|
||||
size = 8
|
||||
}
|
||||
codes[ch] = newhcode(reverseBits(bits, size), size)
|
||||
}
|
||||
return h
|
||||
}
|
||||
|
||||
func generateFixedOffsetEncoding() *huffmanEncoder {
|
||||
h := newHuffmanEncoder(30)
|
||||
codes := h.codes
|
||||
for ch := range codes {
|
||||
codes[ch] = newhcode(reverseBits(uint16(ch), 5), 5)
|
||||
}
|
||||
return h
|
||||
}
|
||||
|
||||
var fixedLiteralEncoding = generateFixedLiteralEncoding()
|
||||
var fixedOffsetEncoding = generateFixedOffsetEncoding()
|
||||
|
||||
func (h *huffmanEncoder) bitLength(freq []uint16) int {
|
||||
var total int
|
||||
for i, f := range freq {
|
||||
if f != 0 {
|
||||
total += int(f) * int(h.codes[i].len())
|
||||
}
|
||||
}
|
||||
return total
|
||||
}
|
||||
|
||||
func (h *huffmanEncoder) bitLengthRaw(b []byte) int {
|
||||
var total int
|
||||
for _, f := range b {
|
||||
total += int(h.codes[f].len())
|
||||
}
|
||||
return total
|
||||
}
|
||||
|
||||
// canReuseBits returns the number of bits or math.MaxInt32 if the encoder cannot be reused.
|
||||
func (h *huffmanEncoder) canReuseBits(freq []uint16) int {
|
||||
var total int
|
||||
for i, f := range freq {
|
||||
if f != 0 {
|
||||
code := h.codes[i]
|
||||
if code.zero() {
|
||||
return math.MaxInt32
|
||||
}
|
||||
total += int(f) * int(code.len())
|
||||
}
|
||||
}
|
||||
return total
|
||||
}
|
||||
|
||||
// Return the number of literals assigned to each bit size in the Huffman encoding
|
||||
//
|
||||
// This method is only called when list.length >= 3
|
||||
// The cases of 0, 1, and 2 literals are handled by special case code.
|
||||
//
|
||||
// list An array of the literals with non-zero frequencies
|
||||
//
|
||||
// and their associated frequencies. The array is in order of increasing
|
||||
// frequency, and has as its last element a special element with frequency
|
||||
// MaxInt32
|
||||
//
|
||||
// maxBits The maximum number of bits that should be used to encode any literal.
|
||||
//
|
||||
// Must be less than 16.
|
||||
//
|
||||
// return An integer array in which array[i] indicates the number of literals
|
||||
//
|
||||
// that should be encoded in i bits.
|
||||
func (h *huffmanEncoder) bitCounts(list []literalNode, maxBits int32) []int32 {
|
||||
if maxBits >= maxBitsLimit {
|
||||
panic("flate: maxBits too large")
|
||||
}
|
||||
n := int32(len(list))
|
||||
list = list[0 : n+1]
|
||||
list[n] = maxNode()
|
||||
|
||||
// The tree can't have greater depth than n - 1, no matter what. This
|
||||
// saves a little bit of work in some small cases
|
||||
if maxBits > n-1 {
|
||||
maxBits = n - 1
|
||||
}
|
||||
|
||||
// Create information about each of the levels.
|
||||
// A bogus "Level 0" whose sole purpose is so that
|
||||
// level1.prev.needed==0. This makes level1.nextPairFreq
|
||||
// be a legitimate value that never gets chosen.
|
||||
var levels [maxBitsLimit]levelInfo
|
||||
// leafCounts[i] counts the number of literals at the left
|
||||
// of ancestors of the rightmost node at level i.
|
||||
// leafCounts[i][j] is the number of literals at the left
|
||||
// of the level j ancestor.
|
||||
var leafCounts [maxBitsLimit][maxBitsLimit]int32
|
||||
|
||||
// Descending to only have 1 bounds check.
|
||||
l2f := int32(list[2].freq)
|
||||
l1f := int32(list[1].freq)
|
||||
l0f := int32(list[0].freq) + int32(list[1].freq)
|
||||
|
||||
for level := int32(1); level <= maxBits; level++ {
|
||||
// For every level, the first two items are the first two characters.
|
||||
// We initialize the levels as if we had already figured this out.
|
||||
levels[level] = levelInfo{
|
||||
level: level,
|
||||
lastFreq: l1f,
|
||||
nextCharFreq: l2f,
|
||||
nextPairFreq: l0f,
|
||||
}
|
||||
leafCounts[level][level] = 2
|
||||
if level == 1 {
|
||||
levels[level].nextPairFreq = math.MaxInt32
|
||||
}
|
||||
}
|
||||
|
||||
// We need a total of 2*n - 2 items at top level and have already generated 2.
|
||||
levels[maxBits].needed = 2*n - 4
|
||||
|
||||
level := uint32(maxBits)
|
||||
for level < 16 {
|
||||
l := &levels[level]
|
||||
if l.nextPairFreq == math.MaxInt32 && l.nextCharFreq == math.MaxInt32 {
|
||||
// We've run out of both leafs and pairs.
|
||||
// End all calculations for this level.
|
||||
// To make sure we never come back to this level or any lower level,
|
||||
// set nextPairFreq impossibly large.
|
||||
l.needed = 0
|
||||
levels[level+1].nextPairFreq = math.MaxInt32
|
||||
level++
|
||||
continue
|
||||
}
|
||||
|
||||
prevFreq := l.lastFreq
|
||||
if l.nextCharFreq < l.nextPairFreq {
|
||||
// The next item on this row is a leaf node.
|
||||
n := leafCounts[level][level] + 1
|
||||
l.lastFreq = l.nextCharFreq
|
||||
// Lower leafCounts are the same of the previous node.
|
||||
leafCounts[level][level] = n
|
||||
e := list[n]
|
||||
if e.literal < math.MaxUint16 {
|
||||
l.nextCharFreq = int32(e.freq)
|
||||
} else {
|
||||
l.nextCharFreq = math.MaxInt32
|
||||
}
|
||||
} else {
|
||||
// The next item on this row is a pair from the previous row.
|
||||
// nextPairFreq isn't valid until we generate two
|
||||
// more values in the level below
|
||||
l.lastFreq = l.nextPairFreq
|
||||
// Take leaf counts from the lower level, except counts[level] remains the same.
|
||||
if true {
|
||||
save := leafCounts[level][level]
|
||||
leafCounts[level] = leafCounts[level-1]
|
||||
leafCounts[level][level] = save
|
||||
} else {
|
||||
copy(leafCounts[level][:level], leafCounts[level-1][:level])
|
||||
}
|
||||
levels[l.level-1].needed = 2
|
||||
}
|
||||
|
||||
if l.needed--; l.needed == 0 {
|
||||
// We've done everything we need to do for this level.
|
||||
// Continue calculating one level up. Fill in nextPairFreq
|
||||
// of that level with the sum of the two nodes we've just calculated on
|
||||
// this level.
|
||||
if l.level == maxBits {
|
||||
// All done!
|
||||
break
|
||||
}
|
||||
levels[l.level+1].nextPairFreq = prevFreq + l.lastFreq
|
||||
level++
|
||||
} else {
|
||||
// If we stole from below, move down temporarily to replenish it.
|
||||
for levels[level-1].needed > 0 {
|
||||
level--
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Somethings is wrong if at the end, the top level is null or hasn't used
|
||||
// all of the leaves.
|
||||
if leafCounts[maxBits][maxBits] != n {
|
||||
panic("leafCounts[maxBits][maxBits] != n")
|
||||
}
|
||||
|
||||
bitCount := h.bitCount[:maxBits+1]
|
||||
bits := 1
|
||||
counts := &leafCounts[maxBits]
|
||||
for level := maxBits; level > 0; level-- {
|
||||
// chain.leafCount gives the number of literals requiring at least "bits"
|
||||
// bits to encode.
|
||||
bitCount[bits] = counts[level] - counts[level-1]
|
||||
bits++
|
||||
}
|
||||
return bitCount
|
||||
}
|
||||
|
||||
// Look at the leaves and assign them a bit count and an encoding as specified
|
||||
// in RFC 1951 3.2.2
|
||||
func (h *huffmanEncoder) assignEncodingAndSize(bitCount []int32, list []literalNode) {
|
||||
code := uint16(0)
|
||||
for n, bits := range bitCount {
|
||||
code <<= 1
|
||||
if n == 0 || bits == 0 {
|
||||
continue
|
||||
}
|
||||
// The literals list[len(list)-bits] .. list[len(list)-bits]
|
||||
// are encoded using "bits" bits, and get the values
|
||||
// code, code + 1, .... The code values are
|
||||
// assigned in literal order (not frequency order).
|
||||
chunk := list[len(list)-int(bits):]
|
||||
|
||||
sortByLiteral(chunk)
|
||||
for _, node := range chunk {
|
||||
h.codes[node.literal] = newhcode(reverseBits(code, uint8(n)), uint8(n))
|
||||
code++
|
||||
}
|
||||
list = list[0 : len(list)-int(bits)]
|
||||
}
|
||||
}
|
||||
|
||||
// Update this Huffman Code object to be the minimum code for the specified frequency count.
|
||||
//
|
||||
// freq An array of frequencies, in which frequency[i] gives the frequency of literal i.
|
||||
// maxBits The maximum number of bits to use for any literal.
|
||||
func (h *huffmanEncoder) generate(freq []uint16, maxBits int32) {
|
||||
list := h.freqcache[:len(freq)+1]
|
||||
codes := h.codes[:len(freq)]
|
||||
// Number of non-zero literals
|
||||
count := 0
|
||||
// Set list to be the set of all non-zero literals and their frequencies
|
||||
for i, f := range freq {
|
||||
if f != 0 {
|
||||
list[count] = literalNode{uint16(i), f}
|
||||
count++
|
||||
} else {
|
||||
codes[i] = 0
|
||||
}
|
||||
}
|
||||
list[count] = literalNode{}
|
||||
|
||||
list = list[:count]
|
||||
if count <= 2 {
|
||||
// Handle the small cases here, because they are awkward for the general case code. With
|
||||
// two or fewer literals, everything has bit length 1.
|
||||
for i, node := range list {
|
||||
// "list" is in order of increasing literal value.
|
||||
h.codes[node.literal].set(uint16(i), 1)
|
||||
}
|
||||
return
|
||||
}
|
||||
sortByFreq(list)
|
||||
|
||||
// Get the number of literals for each bit count
|
||||
bitCount := h.bitCounts(list, maxBits)
|
||||
// And do the assignment
|
||||
h.assignEncodingAndSize(bitCount, list)
|
||||
}
|
||||
|
||||
// atLeastOne clamps the result between 1 and 15.
|
||||
func atLeastOne(v float32) float32 {
|
||||
if v < 1 {
|
||||
return 1
|
||||
}
|
||||
if v > 15 {
|
||||
return 15
|
||||
}
|
||||
return v
|
||||
}
|
||||
|
||||
func histogram(b []byte, h []uint16) {
|
||||
if true && len(b) >= 8<<10 {
|
||||
// Split for bigger inputs
|
||||
histogramSplit(b, h)
|
||||
} else {
|
||||
h = h[:256]
|
||||
for _, t := range b {
|
||||
h[t]++
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func histogramSplit(b []byte, h []uint16) {
|
||||
// Tested, and slightly faster than 2-way.
|
||||
// Writing to separate arrays and combining is also slightly slower.
|
||||
h = h[:256]
|
||||
for len(b)&3 != 0 {
|
||||
h[b[0]]++
|
||||
b = b[1:]
|
||||
}
|
||||
n := len(b) / 4
|
||||
x, y, z, w := b[:n], b[n:], b[n+n:], b[n+n+n:]
|
||||
y, z, w = y[:len(x)], z[:len(x)], w[:len(x)]
|
||||
for i, t := range x {
|
||||
v0 := &h[t]
|
||||
v1 := &h[y[i]]
|
||||
v2 := &h[z[i]]
|
||||
v3 := &h[w[i]]
|
||||
*v0++
|
||||
*v1++
|
||||
*v2++
|
||||
*v3++
|
||||
}
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user