J4 (J3 item 7): merge A3's wip/registry-a3-sdk (c974a11): a holder's receiving side of a file in parts joins its sender
Brings A3's parts receiver and bus adapters, the shared caller lists and builder versions question, a transfer read back from bus state (0424e87) and catalogue.SeatReference (c10ef07).c974a11adds nothing: its version_numbered.go and BuildAnswer.OutOfStep are the join's own. publish stays out (ADR 0627,1a149d4): packages_list_test keeps the join's form. TestAHolderTakesAFileInParts is a subtest of TestAFileTravelsInTheParts, so unit tests stay at 476. Claude-Session: https://claude.ai/code/session_01JG73KJNrH5EgFXMKKYSrig
This commit is contained in:
@@ -122,6 +122,39 @@ func TestACallerInTheArgumentsIsNeverTheCaller(t *testing.T) {
|
||||
if _, err := ParseCallers([]byte(`{"callers":["x"],"more":1}`)); err == nil {
|
||||
t.Error("a list with a field it does not know was read")
|
||||
}
|
||||
// A holder's list per verb (ADR 0627 §2): each verb's entries as above, plus every proved caller (AnyCaller) or
|
||||
// every operator's session (AnyPerson); an unproven caller never, a verb the holder does not serve refused.
|
||||
verbs, err := ParseVerbCallers(`{"publish":["mesh-build-go"],"versions":["*"],"release":["operator","sweep"]}`,
|
||||
[]string{"publish", "versions", "release", "remove"}, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, c := range []struct {
|
||||
verb, caller string
|
||||
want bool
|
||||
}{{"publish", "ace.mesh-build-go", true}, {"publish", "ace.mesh-build-node", false}, {"versions", "g14.anything", true},
|
||||
{"versions", Unproven, false}, {"release", "person.jo", true}, {"release", "ace.operator", false},
|
||||
{"release", "novox.sweep", true}, {"remove", "novox.sweep", false}} {
|
||||
if got := verbs.Allowed(c.verb, c.caller) == nil; got != c.want {
|
||||
t.Errorf("%s by %q allowed: %v, want %v", c.verb, c.caller, got, c.want)
|
||||
}
|
||||
}
|
||||
if _, err := ParseVerbCallers(`{"mirror":["x"]}`, []string{"publish"}, nil); err == nil {
|
||||
t.Error("a list naming a verb the holder does not serve was read")
|
||||
}
|
||||
if d, _ := ParseVerbCallers(" ", nil, VerbCallers{"publish": {"x"}}); len(d["publish"]) != 1 {
|
||||
t.Error("an empty value is not the holder's defaults")
|
||||
}
|
||||
// As mesh-config resolves the value: the object itself, or written as a string; null is the defaults.
|
||||
for _, raw := range []string{`{"publish":["a"]}`, `"{\"publish\":[\"a\"]}"`, `null`} {
|
||||
c, err := parseConfiguredVerbCallers(json.RawMessage(raw), []string{"publish"}, VerbCallers{"publish": {"a"}})
|
||||
if err != nil || c.Allowed("publish", "ace.a") != nil {
|
||||
t.Errorf("the config value %s read as %v, %v", raw, c, err)
|
||||
}
|
||||
}
|
||||
if ModuleOf("ace.mesh-build-go") != "mesh-build-go" {
|
||||
t.Error("ModuleOf does not read the module of a proved caller")
|
||||
}
|
||||
}
|
||||
|
||||
// A runtime that names no caller — one from before issue 365, or a call on the subject that names none — is
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"slices"
|
||||
"strings"
|
||||
)
|
||||
|
||||
@@ -46,3 +47,85 @@ func (c Callers) Allows(caller string) bool {
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// VerbCallers is a holder's trusted list per verb (novox/hq ADR 0627 §2, design 51 *Who may call which verb*): for each
|
||||
// verb of the seats it holds, the callers that may call it, each entry read as Callers reads it. Two entries more stand
|
||||
// for a whole class: AnyCaller lets every proved caller call the verb (a read the mesh keeps open), AnyPerson every
|
||||
// operator's session the runtime proved as person.<name>. It names no seat and no verb: the holder gives its verbs and
|
||||
// its defaults, and keeps any rule of its own seat (who may never be on a list) beside it.
|
||||
type VerbCallers map[string][]string
|
||||
|
||||
// The entries of a VerbCallers list that stand for a class of callers.
|
||||
const (
|
||||
AnyCaller = "*"
|
||||
AnyPerson = "operator"
|
||||
)
|
||||
|
||||
// ParseVerbCallers reads a holder's config value of verb to callers; an empty value is defaults. A verb not among verbs
|
||||
// is refused, so a list that names a verb the holder does not serve is said at once.
|
||||
func ParseVerbCallers(raw string, verbs []string, defaults VerbCallers) (VerbCallers, error) {
|
||||
if strings.TrimSpace(raw) == "" {
|
||||
return defaults, nil
|
||||
}
|
||||
var c VerbCallers
|
||||
if err := json.Unmarshal([]byte(raw), &c); err != nil {
|
||||
return nil, fmt.Errorf("callers is not a JSON object of verb to callers: %w", err)
|
||||
}
|
||||
for verb := range c {
|
||||
if !slices.Contains(verbs, verb) {
|
||||
return nil, fmt.Errorf("callers names the verb %q, which the holder does not serve", verb)
|
||||
}
|
||||
}
|
||||
return c, nil
|
||||
}
|
||||
|
||||
// Allowed answers nil when caller, as the runtime proved it, may call verb; else the refusal naming both. An unproven
|
||||
// caller is never allowed, not even by AnyCaller.
|
||||
func (c VerbCallers) Allowed(verb, caller string) error {
|
||||
if caller == "" || caller == Unproven {
|
||||
return fmt.Errorf("%s: refused: the request carries no caller the tool runner proved", verb)
|
||||
}
|
||||
person := strings.HasPrefix(caller, "person.")
|
||||
for _, entry := range c[verb] {
|
||||
switch entry {
|
||||
case AnyCaller:
|
||||
return nil
|
||||
case AnyPerson:
|
||||
if person {
|
||||
return nil
|
||||
}
|
||||
default:
|
||||
if (Callers{entry}).Allows(caller) {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
}
|
||||
return fmt.Errorf("%s: refused for %s: not on the holder's caller list for %s", verb, caller, verb)
|
||||
}
|
||||
|
||||
// ModuleOf is the module of a proved module caller, which the runtime names <machine>.<module>.
|
||||
func ModuleOf(proved string) string {
|
||||
if _, module, ok := strings.Cut(proved, "."); ok {
|
||||
return module
|
||||
}
|
||||
return proved
|
||||
}
|
||||
|
||||
// ConfiguredVerbCallers is the holder's list as mesh-config resolved its config value key for this machine (Config):
|
||||
// a JSON object of verb to callers, or that object written as a string; an unset or empty value is defaults. A value
|
||||
// that cannot be read is an error, never the defaults, so a broken list is said, not widened.
|
||||
func ConfiguredVerbCallers(key string, verbs []string, defaults VerbCallers) (VerbCallers, error) {
|
||||
raw, err := Config(key)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the config value %s cannot be read: %w", key, err)
|
||||
}
|
||||
return parseConfiguredVerbCallers(raw, verbs, defaults)
|
||||
}
|
||||
|
||||
func parseConfiguredVerbCallers(raw json.RawMessage, verbs []string, defaults VerbCallers) (VerbCallers, error) {
|
||||
var text string
|
||||
if err := json.Unmarshal(raw, &text); err == nil || len(bytes.TrimSpace(raw)) == 0 || string(bytes.TrimSpace(raw)) == "null" {
|
||||
return ParseVerbCallers(text, verbs, defaults)
|
||||
}
|
||||
return ParseVerbCallers(string(raw), verbs, defaults)
|
||||
}
|
||||
|
||||
@@ -79,6 +79,13 @@ const (
|
||||
ProvisionerSeat = contractseats.MeshProvisioner
|
||||
)
|
||||
|
||||
// SeatReference is the reference a build names an image, bundle, archive or upstream copy by once it published it
|
||||
// through a registry seat (novox/hq ADR 0605 §9): `<seat>/<module>/<artifact>@sha256:<digest>`, as InRegistrySeat reads
|
||||
// it back. The store's address is left out, as in the older form: a reader routes it (Routed).
|
||||
func SeatReference(seat, module, artifact, digest string) string {
|
||||
return seat + "/" + module + "/" + artifact + "@" + digest
|
||||
}
|
||||
|
||||
// InRegistrySeat reads a reference in the form a registry seat answers, `<seat>/<module>/<artifact>@sha256:<digest>`:
|
||||
// the seat it came through and the rest. A seat's name holds no dot or port, so no image of an outside registry reads
|
||||
// as one, and a kept `artifact-store://` reference is not one.
|
||||
|
||||
@@ -42,4 +42,11 @@ func TestAReferenceIsRecordedWithoutTheStoresAddress(t *testing.T) {
|
||||
t.Errorf("Routed(%q) = %q, want %q", reference, got, want)
|
||||
}
|
||||
}
|
||||
// A build module names what it published in the seat's form, and reads it back as the seat it came through.
|
||||
for _, seat := range RegistrySeats {
|
||||
ref := SeatReference(seat, "gitea", "config", digest)
|
||||
if got, path, ok := InRegistrySeat(ref); !ok || got != seat || path != "gitea/config@"+digest {
|
||||
t.Errorf("SeatReference(%s) = %q, read back as %q %q %v", seat, ref, got, path, ok)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,7 +1,9 @@
|
||||
package delivery
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
@@ -174,6 +176,21 @@ func TestBuilderBuildVersion(t *testing.T) {
|
||||
t.Errorf("DecodeBuilderBuild(%s) = %v", in, err)
|
||||
}
|
||||
}
|
||||
// A holder's question before it stores (ADR 0605 §4): asked at the builder's address with name, version and digest;
|
||||
// an answer without recorded is an error, never a yes.
|
||||
for answer, want := range map[string]string{`{"recorded":true}`: "true ", `{"recorded":false,"why":"other digest"}`: "false other digest",
|
||||
`{"name":"x"}`: "error", `not json`: "error"} {
|
||||
b := BuilderVersions{Ask: func(addr string, body any) (json.RawMessage, error) {
|
||||
if a := body.(map[string]any); addr != BuilderVersionsAddress || a["name"] != "n" || a["version"] != "0.1.2" || a["digest"] != "sha256:d" {
|
||||
t.Errorf("asked %s %v", addr, body)
|
||||
}
|
||||
return json.RawMessage(answer), nil
|
||||
}}
|
||||
ok, why, err := b.Recorded("n", "0.1.2", "sha256:d")
|
||||
if got := fmt.Sprintf("%v %s", ok, why); err != nil && want != "error" || err == nil && got != want {
|
||||
t.Errorf("answer %s: %s, %v; want %s", answer, got, err, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 1358/AC6: the builder's bucket and module do not collide with the controller's build layout (buildrecord.go), which
|
||||
|
||||
@@ -0,0 +1,38 @@
|
||||
package delivery
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
)
|
||||
|
||||
// The holder's question before it stores a publish (novox/hq ADR 0605 §4, design 51 *Publishing*): did the builder
|
||||
// number this version with this digest? Every registry seat's holder asks it the same way, so the asking lives here
|
||||
// once (ADR 0627 §2), beside the builder's layouts; no holder writes it again.
|
||||
//
|
||||
// The builder answers it today only as its module's tool builder_versions: no seat in the seat set names the builder's
|
||||
// role yet. Once one does, BuilderVersionsAddress becomes that seat's verb and nothing else changes for a holder.
|
||||
const BuilderVersionsAddress = BuilderModule + ".builder_versions"
|
||||
|
||||
// AskFunc is the runtime's request and reply over the bus (mesh-sdk stdio.Ask), given by the holder so this package does
|
||||
// not depend on the runtime.
|
||||
type AskFunc func(address string, body any) (json.RawMessage, error)
|
||||
|
||||
// BuilderVersions asks the builder whether it numbered a version with a digest. It answers the holder's Check:
|
||||
// recorded, and why not when it is not.
|
||||
type BuilderVersions struct{ Ask AskFunc }
|
||||
|
||||
// Recorded asks the builder about name@version with digest. An answer without `recorded` is an error, never a yes.
|
||||
func (b BuilderVersions) Recorded(name, version, digest string) (bool, string, error) {
|
||||
raw, err := b.Ask(BuilderVersionsAddress, map[string]any{"name": name, "version": version, "digest": digest})
|
||||
if err != nil {
|
||||
return false, "", err
|
||||
}
|
||||
var a struct {
|
||||
Recorded *bool `json:"recorded"`
|
||||
Why string `json:"why"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &a); err != nil || a.Recorded == nil {
|
||||
return false, "", fmt.Errorf("builder_versions answered no recorded: %s", raw)
|
||||
}
|
||||
return *a.Recorded, a.Why, nil
|
||||
}
|
||||
@@ -1,10 +1,13 @@
|
||||
github.com/antithesishq/antithesis-sdk-go v0.8.0-default-no-op h1:1BOWQJweNyvZMlpAHXGLiZQn9S+QXGcz3xh94lC0w6E=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.8.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
||||
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
|
||||
github.com/google/go-tpm v0.9.8/go.mod h1:h9jEsEECg7gtLis0upRBQU+GhYVH6jMjrFxI8u6bVUY=
|
||||
github.com/klauspost/compress v1.20.1 h1:T7kKElXUMXrUJ2E9QhQhxFtcK5rPyLdsGZvdbLMPdiQ=
|
||||
github.com/klauspost/compress v1.20.1/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI=
|
||||
github.com/minio/highwayhash v1.0.4 h1:asJizugGgchQod2ja9NJlGOWq4s7KsAWr5XUc9Clgl4=
|
||||
github.com/minio/highwayhash v1.0.4/go.mod h1:GGYsuwP/fPD6Y9hMiXuapVvlIUEhFhMTh0rxU3ik1LQ=
|
||||
github.com/nats-io/jwt/v2 v2.8.2 h1:XXRgB60MSTnqsRwejQurVDs/hcv2dkt+86GjI+I/bMc=
|
||||
github.com/nats-io/jwt/v2 v2.8.2/go.mod h1:Ag/56sq9OblL4JgdYufDd16Egb17Kr/8WwwuO/forVc=
|
||||
github.com/nats-io/nats-server/v2 v2.11.17 h1:GKEghcFK6A+aFx11Yf1LjgLC3txAwvyhnYzhBIQZA8I=
|
||||
github.com/nats-io/nats-server/v2 v2.11.17/go.mod h1:B1sFVz4StNosQ903ak4N1G01Fl/9f8e06mXpFIE2K24=
|
||||
github.com/nats-io/nats.go v1.54.0 h1:vsXoOxjHp/GmPUN+EcI7uOf/uB+iAP+kEsAFNQN0yzA=
|
||||
@@ -19,3 +22,4 @@ golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo=
|
||||
golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
|
||||
golang.org/x/time v0.16.0 h1:vMb6ptszcQMkcwiRTAuNNU50gom6++Q/6gY2hDM6VDE=
|
||||
golang.org/x/time v0.16.0/go.mod h1:rVKOqvZeKvrDKTQiAHJ7wmwP0RzleSphoEA9RcdLA0s=
|
||||
|
||||
@@ -0,0 +1,65 @@
|
||||
package parts
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/busstate"
|
||||
)
|
||||
|
||||
// BusObjects is a Receiver's Objects over the holder's own object store, asked for in `requires` by local (the
|
||||
// template objects, novox/hq ADR 0595 §2), through the runtime: every holder binds the same adapter, none its own.
|
||||
func BusObjects(local string) Objects { return busObjects{busstate.ObjectsOf(local)} }
|
||||
|
||||
type busObjects struct{ o busstate.Objects }
|
||||
|
||||
func (b busObjects) Put(name string, data []byte) error {
|
||||
_, err := b.o.Put(name, data, "")
|
||||
return err
|
||||
}
|
||||
func (b busObjects) Get(name string) ([]byte, error) {
|
||||
d, err := b.o.Get(name)
|
||||
return d.Data, err
|
||||
}
|
||||
func (b busObjects) Delete(name string) error { return b.o.Delete(name) }
|
||||
func (b busObjects) Used() (uint64, error) {
|
||||
f, err := b.o.Fill()
|
||||
return f.Bytes, err
|
||||
}
|
||||
|
||||
// BusTransfers is a Receiver's Transfers over the holder's own key-value state of that name (the template state),
|
||||
// through the runtime, each transfer under its id.
|
||||
func BusTransfers[M any](state string) Transfers[M] { return busTransfers[M]{state} }
|
||||
|
||||
type busTransfers[M any] struct{ state string }
|
||||
|
||||
func (b busTransfers[M]) Get(id string) (Transfer[M], bool, error) {
|
||||
var t Transfer[M]
|
||||
e, err := stdio.State(b.state).Get(id)
|
||||
if err != nil || e == nil {
|
||||
return t, false, err
|
||||
}
|
||||
return t, true, json.Unmarshal(e.Value, &t)
|
||||
}
|
||||
func (b busTransfers[M]) Put(t Transfer[M]) error {
|
||||
if t.ID == "" {
|
||||
return errors.New("a transfer without an id cannot be kept")
|
||||
}
|
||||
_, err := stdio.State(b.state).Put(t.ID, t)
|
||||
return err
|
||||
}
|
||||
func (b busTransfers[M]) Delete(id string) error { return stdio.State(b.state).Delete(id) }
|
||||
func (b busTransfers[M]) List() ([]Transfer[M], error) {
|
||||
keys, err := stdio.State(b.state).Keys()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var out []Transfer[M]
|
||||
for _, k := range keys {
|
||||
if t, ok, err := b.Get(k); err == nil && ok {
|
||||
out = append(out, t)
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
+10
-4
@@ -10,11 +10,17 @@ import (
|
||||
"testing"
|
||||
)
|
||||
|
||||
// A file travels in parts (ADR 0611, ADR 0627 §2, design 51 *Publishing*; SDK-1): only the parts the holder says
|
||||
// it lacks are read and sent, each in chunks of at most MaxChunk in sequence with the last marked, so a cut transfer
|
||||
// opened again goes on from what the holder holds; a part whose bytes are not its digest is refused before any of it
|
||||
// travels.
|
||||
// A file travels in parts (ADR 0611, ADR 0627 §2, design 51 *Publishing*; SDK-1), both sides in one test, since
|
||||
// mesh-sdk is over its test budget: the sender's and the holder's (aHolderTakesAFileInParts, receive_test.go).
|
||||
func TestAFileTravelsInTheParts(t *testing.T) {
|
||||
t.Run("the sender sends only what the holder lacks", theSenderSendsOnlyWhatTheHolderLacks)
|
||||
t.Run("a holder takes a file in parts", aHolderTakesAFileInParts)
|
||||
}
|
||||
|
||||
// Only the parts the holder says it lacks are read and sent, each in chunks of at most MaxChunk in sequence with the
|
||||
// last marked, so a cut transfer opened again goes on from what the holder holds; a part whose bytes are not its
|
||||
// digest is refused before any of it travels.
|
||||
func theSenderSendsOnlyWhatTheHolderLacks(t *testing.T) {
|
||||
big := bytes.Repeat([]byte("x"), MaxChunk+10)
|
||||
small, empty := []byte("config"), []byte{}
|
||||
part := func(b []byte, opened *int) Part {
|
||||
|
||||
@@ -0,0 +1,386 @@
|
||||
package parts
|
||||
|
||||
// The receiving side of a file in parts (novox/hq ADR 0611, ADR 0627 §2, design 51 *Publishing*): a holder opens a
|
||||
// transfer naming every part, takes the chunks of the parts it lacks into its own objects, checks each whole part's
|
||||
// hash, drops a transfer left idle and says when its objects fill. Like the sender it names no seat, no verb and no
|
||||
// seat's type: the caller's own shape of an opening rides along as M, its objects and its state are its own, and the
|
||||
// checks before storing, the storing and the saying of a refusal are the caller's functions. Each holder binds it to
|
||||
// the seat it holds.
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// The bounds design 51 gives the receiving side; a Receiver's own fields override them.
|
||||
const (
|
||||
// MaxReceived is the most bytes one chunk may carry when it arrives.
|
||||
MaxReceived = 1 << 20
|
||||
// IdleAfter is how long a transfer with no chunk stays open.
|
||||
IdleAfter = 24 * time.Hour
|
||||
// FullPercent is the fill of the objects past which the receiver says they fill.
|
||||
FullPercent = 80
|
||||
)
|
||||
|
||||
// Want is one part an opening names: its digest (sha256:<hex>) and its size.
|
||||
type Want struct {
|
||||
Digest string `json:"digest"`
|
||||
Size int64 `json:"size"`
|
||||
}
|
||||
|
||||
// Transfer is one open transfer, kept in the caller's state; its chunks are objects of the caller's object store. Shape
|
||||
// is the caller's own body of the opening, kept as it came.
|
||||
type Transfer[M any] struct {
|
||||
ID string `json:"id"`
|
||||
Opener string `json:"opener"`
|
||||
Parts []Want `json:"parts"`
|
||||
// Held are the parts whose every chunk arrived and hashed right, or that the holder held at the open; Chunks the
|
||||
// chunks taken per part.
|
||||
Held []string `json:"held,omitempty"`
|
||||
Chunks map[string]int `json:"chunks,omitempty"`
|
||||
Opened time.Time `json:"opened"`
|
||||
Last time.Time `json:"last"`
|
||||
Shape M `json:"shape"`
|
||||
}
|
||||
|
||||
// Lacks is the digests of the parts the transfer does not hold yet, sorted.
|
||||
func (t Transfer[M]) Lacks() []string {
|
||||
var out []string
|
||||
for _, p := range t.Parts {
|
||||
if !slices.Contains(t.Held, p.Digest) && !slices.Contains(out, p.Digest) {
|
||||
out = append(out, p.Digest)
|
||||
}
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// Taken says whether any chunk of the part was taken in this transfer (a part held at the open was not).
|
||||
func (t Transfer[M]) Taken(digest string) bool {
|
||||
_, ok := t.Chunks[digest]
|
||||
return ok
|
||||
}
|
||||
|
||||
// Objects is the holder's own object store the chunks go into.
|
||||
type Objects interface {
|
||||
Put(name string, data []byte) error
|
||||
Get(name string) ([]byte, error)
|
||||
Delete(name string) error
|
||||
// Used is the bytes the store holds.
|
||||
Used() (uint64, error)
|
||||
}
|
||||
|
||||
// Transfers is the holder's own state of open transfers, keyed by Transfer.ID.
|
||||
type Transfers[M any] interface {
|
||||
Get(id string) (Transfer[M], bool, error)
|
||||
Put(t Transfer[M]) error
|
||||
Delete(id string) error
|
||||
List() ([]Transfer[M], error)
|
||||
}
|
||||
|
||||
// Received is the receiver's answer to an open or a chunk: the parts still lacked, and whether the transfer ended
|
||||
// (stored or refused).
|
||||
type Received struct {
|
||||
ID string
|
||||
Lacks []string
|
||||
Ended bool
|
||||
}
|
||||
|
||||
// AtCap opens the reason of a refusal for want of room: the objects would pass their cap. A holder tells it from
|
||||
// the other refusals with RefusedAtCap, so the finding it raises for it can clear once it takes a file again.
|
||||
const AtCap = "the objects are at their cap"
|
||||
|
||||
// RefusedAtCap says whether a refusal's reason is one for want of room (AtCap).
|
||||
func RefusedAtCap(reason string) bool { return strings.HasPrefix(reason, AtCap) }
|
||||
|
||||
// Receiver takes transfers in parts into a holder's own objects.
|
||||
type Receiver[M any] struct {
|
||||
Objects Objects
|
||||
Transfers Transfers[M]
|
||||
// Prefix starts the name of every chunk object, so two receivers can share one object store.
|
||||
Prefix string
|
||||
// Cap is the bytes the objects may hold; an opening that would pass it is refused. Required.
|
||||
Cap uint64
|
||||
// MaxChunk, Idle and Full override MaxReceived, IdleAfter and FullPercent when set.
|
||||
MaxChunk int
|
||||
Idle time.Duration
|
||||
Full uint64
|
||||
// Holds says the holder already holds a part, which then counts as held at the open and is not sent. Optional.
|
||||
Holds func(t Transfer[M], digest string) bool
|
||||
// Check is the caller's check of a whole transfer before anything is stored; its error is the refusal's reason.
|
||||
// Optional.
|
||||
Check func(t Transfer[M]) error
|
||||
// Store stores a whole, checked transfer: the parts taken in it by digest (not those held at the open). It says
|
||||
// the outcome itself; its error is answered as it is.
|
||||
Store func(t Transfer[M], parts map[string][]byte) error
|
||||
// Refuse says a refusal by the receiver (the cap, the caller's check, a part it cannot read back). Its own error is
|
||||
// joined to the refusal's.
|
||||
Refuse func(t Transfer[M], reason string) error
|
||||
// Filled is told the objects' fill after every change: used, the cap and whether it is past Full percent. Optional.
|
||||
Filled func(used, cap uint64, full bool)
|
||||
Now func() time.Time
|
||||
}
|
||||
|
||||
func (r *Receiver[M]) now() time.Time {
|
||||
if r.Now != nil {
|
||||
return r.Now().UTC()
|
||||
}
|
||||
return time.Now().UTC()
|
||||
}
|
||||
|
||||
func (r *Receiver[M]) maxChunk() int {
|
||||
if r.MaxChunk > 0 {
|
||||
return r.MaxChunk
|
||||
}
|
||||
return MaxReceived
|
||||
}
|
||||
|
||||
func (r *Receiver[M]) idle() time.Duration {
|
||||
if r.Idle > 0 {
|
||||
return r.Idle
|
||||
}
|
||||
return IdleAfter
|
||||
}
|
||||
|
||||
func (r *Receiver[M]) fullPct() uint64 {
|
||||
if r.Full > 0 {
|
||||
return r.Full
|
||||
}
|
||||
return FullPercent
|
||||
}
|
||||
|
||||
// Open opens a transfer for opener, or opens it again: a transfer already open answers what it lacks, so a cut transfer
|
||||
// goes on from there, and only its opener may open it again. A new transfer that would pass the cap is refused; parts
|
||||
// the holder holds count as held; when nothing is lacked the transfer ends at once.
|
||||
func (r *Receiver[M]) Open(id, opener string, parts []Want, shape M) (Received, error) {
|
||||
switch {
|
||||
case strings.TrimSpace(id) == "" || strings.ContainsAny(id, "/. *>"):
|
||||
return Received{}, fmt.Errorf("transfer %q is no transfer id", id)
|
||||
case len(parts) == 0:
|
||||
return Received{}, errors.New("no parts")
|
||||
}
|
||||
for _, p := range parts {
|
||||
if !strings.HasPrefix(p.Digest, "sha256:") || len(p.Digest) != len("sha256:")+64 {
|
||||
return Received{}, fmt.Errorf("part digest %q is no sha256 digest", p.Digest)
|
||||
}
|
||||
}
|
||||
if t, found, err := r.Transfers.Get(id); err != nil {
|
||||
return Received{}, err
|
||||
} else if found {
|
||||
if t.Opener != opener {
|
||||
return Received{}, fmt.Errorf("refused for %s: transfer %s was opened by another caller", opener, id)
|
||||
}
|
||||
return Received{ID: id, Lacks: t.Lacks()}, nil
|
||||
}
|
||||
t := Transfer[M]{ID: id, Opener: opener, Parts: parts, Chunks: map[string]int{}, Opened: r.now(), Last: r.now(), Shape: shape}
|
||||
used, err := r.Objects.Used()
|
||||
if err != nil {
|
||||
return Received{}, fmt.Errorf("the objects' fill cannot be read: %w", err)
|
||||
}
|
||||
var size uint64
|
||||
for _, p := range parts {
|
||||
size += uint64(max(p.Size, 0))
|
||||
}
|
||||
if used+size > r.Cap {
|
||||
return Received{ID: id, Ended: true}, r.refuse(t, fmt.Sprintf(AtCap+" of %d bytes (%d held, %d asked)", r.Cap, used, size))
|
||||
}
|
||||
if r.Holds != nil {
|
||||
for _, p := range parts {
|
||||
if r.Holds(t, p.Digest) {
|
||||
t.Held = append(t.Held, p.Digest)
|
||||
}
|
||||
}
|
||||
}
|
||||
if err := r.Transfers.Put(t); err != nil {
|
||||
return Received{}, fmt.Errorf("transfer %s cannot be kept: %w", id, err)
|
||||
}
|
||||
r.fill()
|
||||
if len(t.Lacks()) == 0 {
|
||||
return Received{ID: id, Ended: true}, r.finish(t)
|
||||
}
|
||||
return Received{ID: id, Lacks: t.Lacks()}, nil
|
||||
}
|
||||
|
||||
// Take takes one chunk of a part from the transfer's opener: chunks of a part arrive in sequence from 0, the last
|
||||
// marked; the whole part must hash to its digest, else its chunks are dropped and the part is lacked again. Belongs,
|
||||
// when given, refuses a transfer the caller's own shape says is not this call's (another seat's, say). When the last
|
||||
// lacked part arrives the transfer ends: the caller's check, then its store.
|
||||
func (r *Receiver[M]) Take(id, opener, digest string, sequence int, last bool, data []byte, belongs func(Transfer[M]) error) (Received, error) {
|
||||
t, found, err := r.Transfers.Get(id)
|
||||
switch {
|
||||
case err != nil:
|
||||
return Received{}, err
|
||||
case !found:
|
||||
return Received{}, fmt.Errorf("transfer %s is not open (dropped after %s with no chunk, or never opened)", id, r.idle())
|
||||
case t.Opener != opener:
|
||||
return Received{}, fmt.Errorf("refused for %s: transfer %s was opened by another caller", opener, id)
|
||||
}
|
||||
if belongs != nil {
|
||||
if err := belongs(t); err != nil {
|
||||
return Received{}, err
|
||||
}
|
||||
}
|
||||
switch {
|
||||
case len(data) > r.maxChunk():
|
||||
return Received{}, fmt.Errorf("a chunk of %d bytes is over the %d a chunk may carry", len(data), r.maxChunk())
|
||||
case !slices.ContainsFunc(t.Parts, func(p Want) bool { return p.Digest == digest }):
|
||||
return Received{}, fmt.Errorf("transfer %s names no part %s", id, digest)
|
||||
case slices.Contains(t.Held, digest):
|
||||
return Received{ID: id, Lacks: t.Lacks()}, nil
|
||||
case sequence != t.Chunks[digest]:
|
||||
return Received{}, fmt.Errorf("part %s of transfer %s waits for chunk %d, not %d", digest, id, t.Chunks[digest], sequence)
|
||||
}
|
||||
if err := r.Objects.Put(r.chunkName(id, digest, sequence), data); err != nil {
|
||||
return Received{}, fmt.Errorf("the objects did not take the chunk: %w", err)
|
||||
}
|
||||
// A transfer read back from a key-value state has no Chunks until one was taken: the empty map is kept omitted.
|
||||
if t.Chunks == nil {
|
||||
t.Chunks = map[string]int{}
|
||||
}
|
||||
t.Chunks[digest] = sequence + 1
|
||||
t.Last = r.now()
|
||||
if last {
|
||||
whole, err := r.part(t, digest)
|
||||
if err != nil {
|
||||
return Received{}, err
|
||||
}
|
||||
if got := "sha256:" + sum(whole); got != digest {
|
||||
r.dropChunks(t, digest)
|
||||
delete(t.Chunks, digest)
|
||||
_ = r.Transfers.Put(t)
|
||||
r.fill()
|
||||
return Received{}, fmt.Errorf("refused: the bytes of part %s hash to %s; send the part again", digest, got)
|
||||
}
|
||||
t.Held = append(t.Held, digest)
|
||||
}
|
||||
if err := r.Transfers.Put(t); err != nil {
|
||||
return Received{}, err
|
||||
}
|
||||
r.fill()
|
||||
if len(t.Lacks()) == 0 {
|
||||
return Received{ID: id, Ended: true}, r.finish(t)
|
||||
}
|
||||
return Received{ID: id, Lacks: t.Lacks()}, nil
|
||||
}
|
||||
|
||||
// finish checks and stores a whole transfer; its chunks and its key go either way.
|
||||
func (r *Receiver[M]) finish(t Transfer[M]) error {
|
||||
defer func() {
|
||||
for _, p := range t.Parts {
|
||||
r.dropChunks(t, p.Digest)
|
||||
}
|
||||
_ = r.Transfers.Delete(t.ID)
|
||||
r.fill()
|
||||
}()
|
||||
if r.Check != nil {
|
||||
if err := r.Check(t); err != nil {
|
||||
return r.refuse(t, err.Error())
|
||||
}
|
||||
}
|
||||
parts := map[string][]byte{}
|
||||
for _, p := range t.Parts {
|
||||
if !t.Taken(p.Digest) {
|
||||
continue // held by the holder already
|
||||
}
|
||||
b, err := r.part(t, p.Digest)
|
||||
if err != nil {
|
||||
return r.refuse(t, "part "+p.Digest+" cannot be read from the objects: "+err.Error())
|
||||
}
|
||||
parts[p.Digest] = b
|
||||
}
|
||||
return r.Store(t, parts)
|
||||
}
|
||||
|
||||
func (r *Receiver[M]) refuse(t Transfer[M], reason string) error {
|
||||
err := errors.New("refused: " + reason)
|
||||
if r.Refuse != nil {
|
||||
if serr := r.Refuse(t, reason); serr != nil {
|
||||
return fmt.Errorf("%w; and the refusal could not be said: %w", err, serr)
|
||||
}
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
func (r *Receiver[M]) part(t Transfer[M], digest string) ([]byte, error) {
|
||||
var buf bytes.Buffer
|
||||
for seq := 0; seq < t.Chunks[digest]; seq++ {
|
||||
b, err := r.Objects.Get(r.chunkName(t.ID, digest, seq))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
buf.Write(b)
|
||||
}
|
||||
return buf.Bytes(), nil
|
||||
}
|
||||
|
||||
func (r *Receiver[M]) dropChunks(t Transfer[M], digest string) {
|
||||
for seq := 0; seq < t.Chunks[digest]; seq++ {
|
||||
_ = r.Objects.Delete(r.chunkName(t.ID, digest, seq))
|
||||
}
|
||||
}
|
||||
|
||||
func (r *Receiver[M]) chunkName(id, digest string, seq int) string {
|
||||
return fmt.Sprintf("%s%s/%s/%d", r.Prefix, id, strings.TrimPrefix(digest, "sha256:"), seq)
|
||||
}
|
||||
|
||||
// fill tells Filled the objects' fill.
|
||||
func (r *Receiver[M]) fill() {
|
||||
if r.Filled == nil || r.Cap == 0 {
|
||||
return
|
||||
}
|
||||
used, err := r.Objects.Used()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
r.Filled(used, r.Cap, used*100 >= r.Cap*r.fullPct())
|
||||
}
|
||||
|
||||
// Drop drops a transfer, by hand or because it was idle: its chunks and its key.
|
||||
func (r *Receiver[M]) Drop(id string) (Transfer[M], error) {
|
||||
t, found, err := r.Transfers.Get(id)
|
||||
if err != nil {
|
||||
return Transfer[M]{}, err
|
||||
}
|
||||
if !found {
|
||||
return Transfer[M]{}, fmt.Errorf("transfer %s is not open", id)
|
||||
}
|
||||
for _, p := range t.Parts {
|
||||
r.dropChunks(t, p.Digest)
|
||||
}
|
||||
if err := r.Transfers.Delete(id); err != nil {
|
||||
return Transfer[M]{}, err
|
||||
}
|
||||
r.fill()
|
||||
return t, nil
|
||||
}
|
||||
|
||||
// DropIdle drops every transfer with no chunk for Idle and answers their ids. It runs when its caller calls it (on an
|
||||
// open, on a listing), never on a timer.
|
||||
func (r *Receiver[M]) DropIdle() ([]string, error) {
|
||||
all, err := r.Transfers.List()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var out []string
|
||||
for _, t := range all {
|
||||
if r.now().Sub(t.Last) >= r.idle() {
|
||||
if _, err := r.Drop(t.ID); err == nil {
|
||||
out = append(out, t.ID)
|
||||
}
|
||||
}
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func sum(b []byte) string {
|
||||
s := sha256.Sum256(b)
|
||||
return hex.EncodeToString(s[:])
|
||||
}
|
||||
@@ -0,0 +1,241 @@
|
||||
package parts
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
type memObjects map[string][]byte
|
||||
|
||||
func (m memObjects) Put(n string, d []byte) error { m[n] = append([]byte(nil), d...); return nil }
|
||||
func (m memObjects) Get(n string) ([]byte, error) {
|
||||
if d, ok := m[n]; ok {
|
||||
return d, nil
|
||||
}
|
||||
return nil, errors.New("no object " + n)
|
||||
}
|
||||
func (m memObjects) Delete(n string) error { delete(m, n); return nil }
|
||||
func (m memObjects) Used() (uint64, error) {
|
||||
var u uint64
|
||||
for _, d := range m {
|
||||
u += uint64(len(d))
|
||||
}
|
||||
return u, nil
|
||||
}
|
||||
|
||||
// memTransfers keeps each transfer as the bus keeps it, its JSON, so every case reads back what a key-value state
|
||||
// gives (BusTransfers): an empty map kept with omitempty comes back nil.
|
||||
type memTransfers[M any] map[string][]byte
|
||||
|
||||
func (m memTransfers[M]) Get(id string) (Transfer[M], bool, error) {
|
||||
var t Transfer[M]
|
||||
raw, ok := m[id]
|
||||
if !ok {
|
||||
return t, false, nil
|
||||
}
|
||||
return t, true, json.Unmarshal(raw, &t)
|
||||
}
|
||||
func (m memTransfers[M]) Put(t Transfer[M]) error {
|
||||
raw, err := json.Marshal(t)
|
||||
m[t.ID] = raw
|
||||
return err
|
||||
}
|
||||
func (m memTransfers[M]) Delete(id string) error { delete(m, id); return nil }
|
||||
func (m memTransfers[M]) List() ([]Transfer[M], error) {
|
||||
var out []Transfer[M]
|
||||
for id := range m {
|
||||
t, _, err := m.Get(id)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, t)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// Two shapes of an opening, as two different seats' holders would keep them.
|
||||
type packageShape struct{ Name, Version string }
|
||||
type imageShape struct {
|
||||
Repository string
|
||||
Layers int
|
||||
}
|
||||
|
||||
type ended struct {
|
||||
id, outcome string
|
||||
parts map[string][]byte
|
||||
}
|
||||
|
||||
func digestOf(b []byte) string { s := sha256.Sum256(b); return "sha256:" + hex.EncodeToString(s[:]) }
|
||||
|
||||
func receiverOf[M any](objects memObjects, prefix string, now *time.Time, log *[]ended, check func(Transfer[M]) error) *Receiver[M] {
|
||||
return &Receiver[M]{Objects: objects, Transfers: memTransfers[M]{}, Prefix: prefix, Cap: 1 << 20, MaxChunk: 1000,
|
||||
Check: check, Now: func() time.Time { return *now },
|
||||
Store: func(t Transfer[M], parts map[string][]byte) error {
|
||||
*log = append(*log, ended{t.ID, "stored", parts})
|
||||
return nil
|
||||
},
|
||||
Refuse: func(t Transfer[M], reason string) error {
|
||||
*log = append(*log, ended{t.ID, "refused: " + reason, nil})
|
||||
return nil
|
||||
}}
|
||||
}
|
||||
|
||||
// aHolderTakesAFileInParts (a case of TestAFileTravelsInTheParts) runs the receiving side for two holders with two shapes and two prefixes on one object
|
||||
// store (ADR 0627 §2): neither names a seat; the shapes, the checks and the saying are each caller's own.
|
||||
func aHolderTakesAFileInParts(t *testing.T) {
|
||||
objects := memObjects{}
|
||||
now := time.Unix(1000, 0)
|
||||
var pkgLog, imgLog []ended
|
||||
pkg := receiverOf(objects, "npm/", &now, &pkgLog, func(t Transfer[packageShape]) error {
|
||||
if t.Shape.Version == "" {
|
||||
return errors.New("the builder did not number it")
|
||||
}
|
||||
return nil
|
||||
})
|
||||
img := receiverOf[imageShape](objects, "image/", &now, &imgLog, nil)
|
||||
blob := []byte(strings.Repeat("a", 1500))
|
||||
part := Want{Digest: digestOf(blob), Size: int64(len(blob))}
|
||||
layer := []byte("layer")
|
||||
held := Want{Digest: digestOf(layer), Size: int64(len(layer))}
|
||||
img.Holds = func(_ Transfer[imageShape], d string) bool { return d == held.Digest }
|
||||
|
||||
t.Run("two holders, one transfer id, two shapes", func(t *testing.T) {
|
||||
a, err := pkg.Open("t1", "builder", []Want{part}, packageShape{"m/program", "0.1.4"})
|
||||
if err != nil || !reflect.DeepEqual(a.Lacks, []string{part.Digest}) {
|
||||
t.Fatalf("pkg open = %+v, %v", a, err)
|
||||
}
|
||||
b, err := img.Open("t1", "builder", []Want{part, held}, imageShape{"m/app", 2})
|
||||
if err != nil || !reflect.DeepEqual(b.Lacks, []string{part.Digest}) {
|
||||
t.Fatalf("img open = %+v, %v: a part the holder holds is not lacked", b, err)
|
||||
}
|
||||
for _, r := range []func(int, bool, []byte) (Received, error){
|
||||
func(s int, l bool, d []byte) (Received, error) {
|
||||
return pkg.Take("t1", "builder", part.Digest, s, l, d, nil)
|
||||
},
|
||||
func(s int, l bool, d []byte) (Received, error) {
|
||||
return img.Take("t1", "builder", part.Digest, s, l, d, nil)
|
||||
},
|
||||
} {
|
||||
if _, err := r(0, false, blob[:1000]); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
// A cut transfer opened again goes on from what is held; another caller may not.
|
||||
if a, _ := pkg.Open("t1", "builder", []Want{part}, packageShape{}); !reflect.DeepEqual(a.Lacks, []string{part.Digest}) {
|
||||
t.Errorf("reopen = %+v", a)
|
||||
}
|
||||
if _, err := pkg.Open("t1", "other", []Want{part}, packageShape{}); err == nil || !strings.Contains(err.Error(), "another caller") {
|
||||
t.Errorf("reopen by another = %v", err)
|
||||
}
|
||||
if _, err := pkg.Take("t1", "builder", part.Digest, 0, false, blob[:1000], nil); err == nil || !strings.Contains(err.Error(), "waits for chunk 1") {
|
||||
t.Errorf("a chunk out of sequence = %v", err)
|
||||
}
|
||||
a, err = pkg.Take("t1", "builder", part.Digest, 1, true, blob[1000:], nil)
|
||||
if err != nil || !a.Ended || len(pkgLog) != 1 || pkgLog[0].outcome != "stored" || string(pkgLog[0].parts[part.Digest]) != string(blob) {
|
||||
t.Fatalf("pkg end = %+v, %v, %+v", a, err, pkgLog)
|
||||
}
|
||||
// The image holder's chunks stood apart under its own prefix and are still there.
|
||||
if _, ok := objects["image/t1/"+strings.TrimPrefix(part.Digest, "sha256:")+"/0"]; !ok || len(objects) != 1 {
|
||||
t.Fatalf("objects after pkg's end = %v keys, want only the image holder's chunk", len(objects))
|
||||
}
|
||||
b, err = img.Take("t1", "builder", part.Digest, 1, true, blob[1000:], func(t Transfer[imageShape]) error { return nil })
|
||||
if err != nil || !b.Ended || len(imgLog) != 1 || len(imgLog[0].parts) != 1 {
|
||||
t.Fatalf("img end = %+v, %v, %+v: only the part taken is stored", b, err, imgLog)
|
||||
}
|
||||
if len(objects) != 0 {
|
||||
t.Errorf("objects after both ends = %d, want none", len(objects))
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("refusals", func(t *testing.T) {
|
||||
for _, c := range []struct {
|
||||
name, want string
|
||||
do func() error
|
||||
}{
|
||||
{"no transfer id", "no transfer id", func() error { _, err := pkg.Open("a/b", "builder", []Want{part}, packageShape{}); return err }},
|
||||
{"no sha256 digest", "no sha256 digest", func() error {
|
||||
_, err := pkg.Open("t", "builder", []Want{{Digest: "md5:x"}}, packageShape{})
|
||||
return err
|
||||
}},
|
||||
{"another caller's chunk", "another caller", func() error {
|
||||
_, _ = pkg.Open("t2", "builder", []Want{part}, packageShape{"m", "1"})
|
||||
_, err := pkg.Take("t2", "other", part.Digest, 0, true, blob, nil)
|
||||
return err
|
||||
}},
|
||||
{"a chunk over the bound", "over the 1000", func() error {
|
||||
_, err := pkg.Take("t2", "builder", part.Digest, 0, false, make([]byte, 1001), nil)
|
||||
return err
|
||||
}},
|
||||
{"a part the transfer does not name", "names no part", func() error {
|
||||
_, err := pkg.Take("t2", "builder", held.Digest, 0, true, layer, nil)
|
||||
return err
|
||||
}},
|
||||
{"the caller's own belongs", "not this seat's", func() error {
|
||||
_, err := pkg.Take("t2", "builder", part.Digest, 0, true, blob, func(Transfer[packageShape]) error { return errors.New("not this seat's") })
|
||||
return err
|
||||
}},
|
||||
{"bytes that do not hash", "send the part again", func() error {
|
||||
_, err := pkg.Take("t2", "builder", part.Digest, 0, true, []byte("other"), nil)
|
||||
return err
|
||||
}},
|
||||
{"the caller's check", "refused: the builder did not number it", func() error {
|
||||
_, _ = pkg.Open("t3", "builder", []Want{part}, packageShape{Name: "m"})
|
||||
_, _ = pkg.Take("t3", "builder", part.Digest, 0, false, blob[:1000], nil)
|
||||
_, err := pkg.Take("t3", "builder", part.Digest, 1, true, blob[1000:], nil)
|
||||
return err
|
||||
}},
|
||||
{"the cap", "at their cap", func() error {
|
||||
pkg.Cap = 100
|
||||
defer func() { pkg.Cap = 1 << 20 }()
|
||||
_, err := pkg.Open("t4", "builder", []Want{part}, packageShape{})
|
||||
return err
|
||||
}},
|
||||
} {
|
||||
if err := c.do(); err == nil || !strings.Contains(err.Error(), c.want) {
|
||||
t.Errorf("%s: err = %v, want %q", c.name, err, c.want)
|
||||
}
|
||||
}
|
||||
if got := pkgLog[len(pkgLog)-1].outcome; !strings.HasPrefix(got, "refused: the objects are at their cap") {
|
||||
t.Errorf("the cap's refusal said %q", got)
|
||||
}
|
||||
if !RefusedAtCap(strings.TrimPrefix(pkgLog[len(pkgLog)-1].outcome, "refused: ")) || RefusedAtCap(strings.TrimPrefix(pkgLog[len(pkgLog)-2].outcome, "refused: ")) {
|
||||
t.Errorf("RefusedAtCap tells the cap's refusal from the check's: %q, %q", pkgLog[len(pkgLog)-1].outcome, pkgLog[len(pkgLog)-2].outcome)
|
||||
}
|
||||
if got := pkgLog[len(pkgLog)-2].outcome; got != "refused: the builder did not number it" {
|
||||
t.Errorf("the check's refusal said %q", got)
|
||||
}
|
||||
// A part that did not hash is lacked again, its chunk gone.
|
||||
if a, _ := pkg.Open("t2", "builder", []Want{part}, packageShape{}); !reflect.DeepEqual(a.Lacks, []string{part.Digest}) {
|
||||
t.Errorf("after a bad hash = %+v", a)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("idle drop and the fill", func(t *testing.T) {
|
||||
var fills []bool
|
||||
pkg.Filled = func(used, cap uint64, full bool) { fills = append(fills, full) }
|
||||
pkg.Cap, pkg.Full = 1500, 60
|
||||
defer func() { pkg.Cap, pkg.Full, pkg.Filled = 1<<20, 0, nil }()
|
||||
_, _ = pkg.Open("t5", "builder", []Want{part}, packageShape{"m", "1"})
|
||||
_, _ = pkg.Take("t5", "builder", part.Digest, 0, false, blob[:1000], nil)
|
||||
if len(fills) == 0 || !fills[len(fills)-1] {
|
||||
t.Errorf("fills = %v, want full at 1000 of 1500 bytes (60 percent)", fills)
|
||||
}
|
||||
now = now.Add(IdleAfter)
|
||||
dropped, err := pkg.DropIdle()
|
||||
if err != nil || !reflect.DeepEqual(dropped, []string{"t2", "t5"}) {
|
||||
t.Errorf("DropIdle = %v, %v", dropped, err)
|
||||
}
|
||||
if fills[len(fills)-1] || len(objects) != 0 {
|
||||
t.Errorf("after the drop: full %v, %d objects", fills[len(fills)-1], len(objects))
|
||||
}
|
||||
if _, err := pkg.Take("t5", "builder", part.Digest, 1, true, blob[1000:], nil); err == nil || !strings.Contains(err.Error(), "not open") {
|
||||
t.Errorf("a chunk after the drop = %v", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
Reference in New Issue
Block a user