The checker keeps a composition request in pieces and packs it in the compose ask (issue 1966, recovery; group with mesh-catalog) #189
@@ -485,7 +485,7 @@ type StepAskCompose struct {
|
||||
Fetch Fetch `json:"fetch"`
|
||||
Package string `json:"package"`
|
||||
Reader string `json:"reader,omitempty"`
|
||||
// the compose-candidate request
|
||||
// the compose-candidate request, in one of two forms: the request as a JSON object, or, for one near the bus's message limit, a JSON string "gzip+base64:" followed by the base64 of the request object gzip-compressed; a holder that does not know the packed form refuses it, naming that it holds no request object
|
||||
Request any `json:"request"`
|
||||
}
|
||||
|
||||
|
||||
@@ -485,7 +485,7 @@ type StepAskCompose struct {
|
||||
Fetch Fetch `json:"fetch"`
|
||||
Package string `json:"package"`
|
||||
Reader string `json:"reader,omitempty"`
|
||||
// the compose-candidate request
|
||||
// the compose-candidate request, in one of two forms: the request as a JSON object, or, for one near the bus's message limit, a JSON string "gzip+base64:" followed by the base64 of the request object gzip-compressed; a holder that does not know the packed form refuses it, naming that it holds no request object
|
||||
Request any `json:"request"`
|
||||
}
|
||||
|
||||
|
||||
@@ -485,7 +485,7 @@ type StepAskCompose struct {
|
||||
Fetch Fetch `json:"fetch"`
|
||||
Package string `json:"package"`
|
||||
Reader string `json:"reader,omitempty"`
|
||||
// the compose-candidate request
|
||||
// the compose-candidate request, in one of two forms: the request as a JSON object, or, for one near the bus's message limit, a JSON string "gzip+base64:" followed by the base64 of the request object gzip-compressed; a holder that does not know the packed form refuses it, naming that it holds no request object
|
||||
Request any `json:"request"`
|
||||
}
|
||||
|
||||
|
||||
@@ -161,7 +161,7 @@
|
||||
"type": "string"
|
||||
},
|
||||
"request": {
|
||||
"description": "the compose-candidate request"
|
||||
"description": "the compose-candidate request, in one of two forms: the request as a JSON object, or, for one near the bus's message limit, a JSON string \"gzip+base64:\" followed by the base64 of the request object gzip-compressed; a holder that does not know the packed form refuses it, naming that it holds no request object"
|
||||
},
|
||||
"reader": {
|
||||
"type": "string"
|
||||
|
||||
@@ -293,7 +293,7 @@ export interface StepAskCompose {
|
||||
"fetch": Fetch;
|
||||
"package": string;
|
||||
"reader"?: string;
|
||||
/** the compose-candidate request */
|
||||
/** the compose-candidate request, in one of two forms: the request as a JSON object, or, for one near the bus's message limit, a JSON string "gzip+base64:" followed by the base64 of the request object gzip-compressed; a holder that does not know the packed form refuses it, naming that it holds no request object */
|
||||
"request": unknown;
|
||||
}
|
||||
|
||||
|
||||
@@ -293,7 +293,7 @@ export interface StepAskCompose {
|
||||
"fetch": Fetch;
|
||||
"package": string;
|
||||
"reader"?: string;
|
||||
/** the compose-candidate request */
|
||||
/** the compose-candidate request, in one of two forms: the request as a JSON object, or, for one near the bus's message limit, a JSON string "gzip+base64:" followed by the base64 of the request object gzip-compressed; a holder that does not know the packed form refuses it, naming that it holds no request object */
|
||||
"request": unknown;
|
||||
}
|
||||
|
||||
|
||||
@@ -293,7 +293,7 @@ export interface StepAskCompose {
|
||||
"fetch": Fetch;
|
||||
"package": string;
|
||||
"reader"?: string;
|
||||
/** the compose-candidate request */
|
||||
/** the compose-candidate request, in one of two forms: the request as a JSON object, or, for one near the bus's message limit, a JSON string "gzip+base64:" followed by the base64 of the request object gzip-compressed; a holder that does not know the packed form refuses it, naming that it holds no request object */
|
||||
"request": unknown;
|
||||
}
|
||||
|
||||
|
||||
@@ -378,7 +378,7 @@ func RequestTools(r check.StateRead) []stdio.Tool {
|
||||
Description: "The composition request a check sent to a node: the composer's whole input for that node (the node-engine's composition input, the case's " +
|
||||
"manifests and verdicts, the seat list), kept exactly as sent so a session reproduces the check's composition (the composer run by hand over the same " +
|
||||
"request gives the check's answer). Kept in the checker's state compositions, per check key (group.heads, the 12-digit heads hash) and node, " +
|
||||
"about 430 KB each (kept compressed, so a value fits the state's 256 KiB), for 3 days (the state's ttl-seconds). With a check and a node, that request whole; with a check, " +
|
||||
"0.7 to 0.8 MB each (kept compressed, in pieces that each fit the state's 256 KiB value), for 3 days (the state's ttl-seconds). With a check and a node, that request whole; with a check, " +
|
||||
"the nodes it kept one for; with neither, every kept request, each with its size, digest and when it was kept. Read-only.",
|
||||
Input: map[string]any{"type": "object", "properties": map[string]any{
|
||||
"check": map[string]any{"type": "string", "description": "the check's key, <group>.<heads hash>, as the verdict and composition name it"},
|
||||
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"math/rand"
|
||||
"os"
|
||||
"regexp"
|
||||
"slices"
|
||||
@@ -48,7 +49,36 @@ func TestTheToolsServedAreTheFiveReadTools(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
// Issue 1966: a request whose compressed bytes pass the state's value limit is kept in pieces and read back whole; the
|
||||
// pieces are no requests in the list.
|
||||
rnd := make([]byte, 600<<10)
|
||||
_, _ = rand.New(rand.NewSource(1)).Read(rnd)
|
||||
big := []byte(`{"input":"` + base64.StdEncoding.EncodeToString(rnd) + `"}`)
|
||||
if err := keeper.Keep(context.Background(), "big.0123456789ab", "anchor", big); err != nil {
|
||||
t.Fatalf("a request of %d bytes: %v", len(big), err)
|
||||
}
|
||||
rt := RequestTools(check.StateRead{Keys: store.Keys, Get: store.Get})
|
||||
if one, err := rt[0].Run(map[string]any{"check": "big.0123456789ab", "node": "anchor"}); err != nil || !strings.Contains(func() string { b, _ := json.Marshal(one); return string(b) }(), base64.StdEncoding.EncodeToString(rnd)[:64]) {
|
||||
t.Errorf("the big request read back: %v", err)
|
||||
}
|
||||
// Listing reads no piece, and a request of another version is refused by name.
|
||||
store.vals["v1.0123456789ab/anchor"] = json.RawMessage(`{"v":1,"check":"v1.0123456789ab","node":"anchor","gzip":"x"}`)
|
||||
noPieces := check.StateRead{Keys: store.Keys, Get: func(k string) (json.RawMessage, error) {
|
||||
if strings.HasPrefix(k, "pieces/") {
|
||||
return nil, errors.New("a piece was read")
|
||||
}
|
||||
return store.Get(k)
|
||||
}}
|
||||
if _, err := RequestTools(noPieces)[0].Run(map[string]any{}); err != nil {
|
||||
t.Errorf("listing: %v", err)
|
||||
}
|
||||
if _, err := rt[0].Run(map[string]any{"check": "v1.0123456789ab", "node": "anchor"}); err == nil || !strings.Contains(err.Error(), "version 1") {
|
||||
t.Errorf("a version 1 request: %v", err)
|
||||
}
|
||||
delete(store.vals, "v1.0123456789ab/anchor")
|
||||
if all, _ := rt[0].Run(map[string]any{}); strings.Count(func() string { b, _ := json.Marshal(all); return string(b) }(), `"node"`) != 4 {
|
||||
t.Errorf("the list holds a piece as a request: %v", all)
|
||||
}
|
||||
if len(rt) != 1 || rt[0].Name != check.RequestsReadTool || !strings.Contains(rt[0].Description, "composition request a check sent to a node") {
|
||||
t.Fatalf("request tools = %+v", rt)
|
||||
}
|
||||
@@ -312,6 +342,9 @@ func TestTheTryToolsAskAndReadATry(t *testing.T) { // ADR 0501 §5, the agreed i
|
||||
type memPut struct{ m *memStore }
|
||||
|
||||
func (p memPut) Put(_ context.Context, key string, value []byte) (uint64, error) {
|
||||
if len(value) > 262144 { // the state's value limit, headers included (issue 1966)
|
||||
return 0, errors.New("the value is over the 262144 bytes the bucket takes")
|
||||
}
|
||||
p.m.vals[key] = value
|
||||
return 1, nil
|
||||
}
|
||||
|
||||
@@ -228,6 +228,10 @@ func (e EngineCompositions) inSandbox(ctx context.Context, c *Case, head, node s
|
||||
}
|
||||
job := buildstep.ComposeJob{Fetch: buildstep.Fetch{Repository: nodeEngineRepo, Commit: head, Packages: packagesFor(e.Published, c, c.Tagged, nodeEngineRepo)}, Dir: meshcatalogclient.RootPath,
|
||||
Package: composer.Package, Request: raw, Reader: "the node-engine head of " + c.Group}
|
||||
// The ask travels on the bus, whose message limit a whole request passes (1.2 MB for a large node): it goes packed.
|
||||
if err := job.PackRequest(raw); err != nil {
|
||||
return a, err
|
||||
}
|
||||
verdict, err := e.Sandbox.Compose(ctx, job)
|
||||
if err != nil {
|
||||
return a, fmt.Errorf("the composer at the node-engine's head %s did not answer for %s: %w", head, node, err)
|
||||
|
||||
@@ -59,7 +59,11 @@ type fakeSandbox struct {
|
||||
|
||||
func (f *fakeSandbox) Compose(_ context.Context, job buildstep.ComposeJob) (composer.Verdict, error) {
|
||||
var req composer.Request
|
||||
if err := json.Unmarshal(job.Request, &req); err != nil {
|
||||
body, err := job.RequestBytes()
|
||||
if err != nil {
|
||||
return composer.Verdict{}, err
|
||||
}
|
||||
if err := json.Unmarshal(body, &req); err != nil {
|
||||
return composer.Verdict{}, err
|
||||
}
|
||||
f.mu.Lock()
|
||||
@@ -206,7 +210,11 @@ func TestTheCompositionIsMadeFromTheEnginesAnswers(t *testing.T) {
|
||||
}
|
||||
job := sb.jobs[0]
|
||||
var req composer.Request
|
||||
_ = json.Unmarshal(job.Request, &req)
|
||||
if len(job.Request) == 0 || job.Request[0] != '"' {
|
||||
t.Errorf("the request went whole, not packed (issue 1966): %.60s", job.Request)
|
||||
}
|
||||
body, _ := job.RequestBytes()
|
||||
_ = json.Unmarshal(body, &req)
|
||||
if job.Fetch.Repository != nodeEngineRepo || job.Fetch.Commit != head2 || job.Package != composer.Package ||
|
||||
string(req.Input) != `{"nodes":[{"name":"anchor"}]}` || req.Cases[0].Manifests["web"].Digest == "" ||
|
||||
len(req.Seats) != 1 || string(req.Seats[0]) != string(host.Seats[0]) {
|
||||
@@ -220,8 +228,9 @@ func TestTheCompositionIsMadeFromTheEnginesAnswers(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
keyOfHost, _ := nodeengineclient.CompositionKey(host.Group, host.Heads)
|
||||
lastBody, _ := sb.jobs[len(sb.jobs)-1].RequestBytes()
|
||||
got, err := ReadRequests(store.read(), keyOfHost, "anchor")
|
||||
if b, _ := json.Marshal(got); err != nil || !strings.Contains(string(b), `"request":`+string(sb.jobs[len(sb.jobs)-1].Request)) {
|
||||
if b, _ := json.Marshal(got); err != nil || !strings.Contains(string(b), `"request":`+string(lastBody)) {
|
||||
t.Errorf("kept %s, %v: want the request of the last job under %s/anchor", b, err, keyOfHost)
|
||||
}
|
||||
store.err = errors.New("the state is full")
|
||||
@@ -311,7 +320,8 @@ func TestTheCompositionIsMadeFromTheEnginesAnswers(t *testing.T) {
|
||||
t.Fatalf("jobs %+v: want one job at the node-engine's trunk", sb.jobs)
|
||||
}
|
||||
var req composer.Request
|
||||
_ = json.Unmarshal(sb.jobs[0].Request, &req)
|
||||
b0, _ := sb.jobs[0].RequestBytes()
|
||||
_ = json.Unmarshal(b0, &req)
|
||||
if got := string(req.Cases[0].Verdicts["mesh-build-go"]); got != string(brings.Judged["mesh-build-go"]) || len(req.Cases[0].Verdicts) != 2 {
|
||||
t.Errorf("the request's verdicts %s: want the check's judge's for mesh-build-go and the indexer's for mail", req.Cases[0].Verdicts)
|
||||
}
|
||||
@@ -321,7 +331,8 @@ func TestTheCompositionIsMadeFromTheEnginesAnswers(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var none composer.Request
|
||||
_ = json.Unmarshal(sb.jobs[len(sb.jobs)-1].Request, &none)
|
||||
bn, _ := sb.jobs[len(sb.jobs)-1].RequestBytes()
|
||||
_ = json.Unmarshal(bn, &none)
|
||||
if _, kept := none.Cases[0].Verdicts["mesh-build-go"]; kept {
|
||||
t.Errorf("the running indexer's verdict on the change's manifest was kept: %s", none.Cases[0].Verdicts)
|
||||
}
|
||||
|
||||
@@ -20,20 +20,28 @@ import (
|
||||
// session reproduces a check's composition exactly. The checker is a foundation part (novox/hq ADR 0619 §1-3), so its
|
||||
// kept data is a section of its manifest's old form, here `state`, and not an object store asked in `requires`.
|
||||
//
|
||||
// **What bounds it.** A request is the node-engine's whole composition input of one node, about 430 KB as JSON. A
|
||||
// state's value holds at most 256 KiB (kvlimit.ModuleStateValueBytes), so it is kept gzip-compressed (JSON shrinks about
|
||||
// tenfold) in a document of its own; a request whose compressed bytes still do not fit is refused by the put, which
|
||||
// fails the node's composition naming the size (never dropped). The manifest's ttl-seconds is its lifetime (3 days):
|
||||
// the bus drops a value past it, so nothing waits on a timer here. A check kept again overwrites the key.
|
||||
// **What bounds it.** A request is the node-engine's whole composition input of one node, 0.7 to 0.8 MB as JSON and
|
||||
// growing with the mesh. A state's value holds at most 256 KiB (kvlimit.ModuleStateValueBytes, headers included), and
|
||||
// the compressed request already passed it (issue 1966: 300127 bytes), so it is kept gzip-compressed in pieces: the key
|
||||
// of the check and node holds a small document naming the request's digest, size and number of pieces, and each piece is
|
||||
// a value of its own under RequestChunkKey, of at most RequestChunkBytes. The kept data is the checker's, a foundation
|
||||
// part's, so it stays in the old section `state` (ADR 0619 §2) and is read through the tool runner's state read, which
|
||||
// serves no object store of the old sections. The manifest's ttl-seconds is the lifetime (3 days) of the pointer and of
|
||||
// every piece: the bus drops a value past it, so nothing waits on a timer here. A check kept again overwrites the key;
|
||||
// a request whose pieces cannot all be put is refused naming the size (never dropped, never half kept: the pointer is
|
||||
// put after its pieces).
|
||||
const (
|
||||
// RequestsStateName is the checker's state of kept requests and RequestsBucket its bucket.
|
||||
RequestsStateName = "compositions"
|
||||
RequestsBucket = CheckerModule + "_" + RequestsStateName
|
||||
// RequestsReadTool is the tool that reads a kept request.
|
||||
RequestsReadTool = "checker_composition_requests"
|
||||
// RequestChunkBytes is the most compressed bytes one piece holds: base64 of it, in its document, stays well under
|
||||
// the state's 256 KiB value.
|
||||
RequestChunkBytes = 128 << 10
|
||||
)
|
||||
|
||||
// KeptRequest is a request as the state holds it: whose it is, and the request's JSON, compressed.
|
||||
// KeptRequest is the pointer to a request as the state holds it: whose it is, and what its pieces make.
|
||||
type KeptRequest struct {
|
||||
V int `json:"v"`
|
||||
Check string `json:"check"`
|
||||
@@ -42,8 +50,32 @@ type KeptRequest struct {
|
||||
// Size and Digest are the request's, uncompressed: its bytes and their sha256.
|
||||
Size int `json:"size"`
|
||||
Digest string `json:"digest"`
|
||||
// Gzip is the request's bytes, gzip-compressed and base64-encoded.
|
||||
Gzip string `json:"gzip"`
|
||||
// Pieces is how many pieces (RequestChunkKey of the digest, 0 to Pieces-1) make the request's gzip bytes.
|
||||
Pieces int `json:"pieces"`
|
||||
}
|
||||
|
||||
// KeptPiece is one piece of a request's gzip bytes, base64-encoded.
|
||||
type KeptPiece struct {
|
||||
V int `json:"v"`
|
||||
Digest string `json:"digest"`
|
||||
Index int `json:"index"`
|
||||
Gzip string `json:"gzip"`
|
||||
}
|
||||
|
||||
// isPieceKey is whether a key is a piece's: under pieces/ and a sha256 digest, which no check's key (group.heads/node) is.
|
||||
func isPieceKey(k string) bool {
|
||||
rest, ok := strings.CutPrefix(k, "pieces/")
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
digest, _, _ := strings.Cut(rest, "/")
|
||||
return len(digest) == 64
|
||||
}
|
||||
|
||||
// RequestChunkKey is the key of a request's piece: under the request's digest, so a request kept again for the same
|
||||
// bytes writes the same pieces.
|
||||
func RequestChunkKey(digest string, index int) string {
|
||||
return fmt.Sprintf("pieces/%s/%d", digest, index)
|
||||
}
|
||||
|
||||
// RequestKey is the key a check's request to a node is kept under: the check's key (group.heads) and the node.
|
||||
@@ -75,14 +107,28 @@ func (k RequestKeeper) Keep(ctx context.Context, key, node string, body []byte)
|
||||
now = k.Now
|
||||
}
|
||||
sum := sha256.Sum256(body)
|
||||
doc, err := json.Marshal(KeptRequest{V: 1, Check: key, Node: node, KeptAt: now().UTC(), Size: len(body),
|
||||
Digest: hex.EncodeToString(sum[:]), Gzip: base64.StdEncoding.EncodeToString(packed.Bytes())})
|
||||
digest := hex.EncodeToString(sum[:])
|
||||
pieces := (packed.Len() + RequestChunkBytes - 1) / RequestChunkBytes
|
||||
fail := func(err error) error {
|
||||
return fmt.Errorf("the composition request of check %s to %s (%d bytes, %d compressed in %d pieces) could not be kept in the state %s: %w",
|
||||
key, node, len(body), packed.Len(), pieces, RequestsStateName, err)
|
||||
}
|
||||
for i := 0; i < pieces; i++ {
|
||||
end := min((i+1)*RequestChunkBytes, packed.Len())
|
||||
doc, err := json.Marshal(KeptPiece{V: 2, Digest: digest, Index: i, Gzip: base64.StdEncoding.EncodeToString(packed.Bytes()[i*RequestChunkBytes : end])})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := k.Store.Put(ctx, RequestChunkKey(digest, i), doc); err != nil {
|
||||
return fail(err)
|
||||
}
|
||||
}
|
||||
doc, err := json.Marshal(KeptRequest{V: 2, Check: key, Node: node, KeptAt: now().UTC(), Size: len(body), Digest: digest, Pieces: pieces})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := k.Store.Put(ctx, RequestKey(key, node), doc); err != nil {
|
||||
return fmt.Errorf("the composition request of check %s to %s (%d bytes, %d compressed) could not be kept in the state %s: %w",
|
||||
key, node, len(body), packed.Len(), RequestsStateName, err)
|
||||
return fail(err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -104,7 +150,7 @@ func ReadRequests(s StateRead, key, node string) (any, error) {
|
||||
if len(raw) == 0 {
|
||||
return nil, fmt.Errorf("no composition request of check %s to %s is kept (none was sent, or it is past its lifetime)", key, node)
|
||||
}
|
||||
kept, body, err := unpackRequest(raw)
|
||||
kept, body, err := unpackRequest(s, raw)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the kept request %s: %w", RequestKey(key, node), err)
|
||||
}
|
||||
@@ -120,6 +166,9 @@ func ReadRequests(s StateRead, key, node string) (any, error) {
|
||||
if key != "" && !strings.HasPrefix(k, key+"/") {
|
||||
continue
|
||||
}
|
||||
if isPieceKey(k) {
|
||||
continue // a piece of a request, not a request: never read here
|
||||
}
|
||||
raw, err := s.Get(k)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -134,15 +183,30 @@ func ReadRequests(s StateRead, key, node string) (any, error) {
|
||||
return map[string]any{"kept": listed}, nil
|
||||
}
|
||||
|
||||
// unpackRequest reads a kept document and its request's bytes, which must match the digest they were kept with.
|
||||
func unpackRequest(raw []byte) (KeptRequest, []byte, error) {
|
||||
// unpackRequest reads a kept pointer's pieces and the request's bytes, which must match the digest they were kept with.
|
||||
func unpackRequest(s StateRead, raw []byte) (KeptRequest, []byte, error) {
|
||||
var kept KeptRequest
|
||||
if err := json.Unmarshal(raw, &kept); err != nil {
|
||||
return kept, nil, fmt.Errorf("is no kept request: %w", err)
|
||||
}
|
||||
packed, err := base64.StdEncoding.DecodeString(kept.Gzip)
|
||||
if err != nil {
|
||||
return kept, nil, fmt.Errorf("holds no base64: %w", err)
|
||||
if kept.V != 2 {
|
||||
return kept, nil, fmt.Errorf("is a kept request of version %d, which this reader does not read (it reads version 2; an older one is past its lifetime within 3 days)", kept.V)
|
||||
}
|
||||
var packed []byte
|
||||
for i := 0; i < kept.Pieces; i++ {
|
||||
raw, err := s.Get(RequestChunkKey(kept.Digest, i))
|
||||
if err != nil {
|
||||
return kept, nil, err
|
||||
}
|
||||
var piece KeptPiece
|
||||
if err := json.Unmarshal(raw, &piece); err != nil || len(raw) == 0 {
|
||||
return kept, nil, fmt.Errorf("lacks its piece %d of %d (past its lifetime, or never kept)", i, kept.Pieces)
|
||||
}
|
||||
part, err := base64.StdEncoding.DecodeString(piece.Gzip)
|
||||
if err != nil {
|
||||
return kept, nil, fmt.Errorf("holds no base64 in its piece %d: %w", i, err)
|
||||
}
|
||||
packed = append(packed, part...)
|
||||
}
|
||||
zr, err := gzip.NewReader(bytes.NewReader(packed))
|
||||
if err != nil {
|
||||
|
||||
+54
-4
@@ -1,8 +1,12 @@
|
||||
package buildstep
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"path"
|
||||
"slices"
|
||||
"sort"
|
||||
@@ -281,13 +285,55 @@ type CheckJob struct {
|
||||
|
||||
// ComposeJob asks one compose-candidate build.
|
||||
type ComposeJob struct {
|
||||
Fetch Fetch `json:"fetch"`
|
||||
Dir string `json:"dir"`
|
||||
Package string `json:"package"`
|
||||
Fetch Fetch `json:"fetch"`
|
||||
Dir string `json:"dir"`
|
||||
Package string `json:"package"`
|
||||
// Request is the composer's request: a JSON object, or a JSON string PackedRequestPrefix + the base64 of the object
|
||||
// gzip-compressed (issue 1966; the checker packs every request it sends, and
|
||||
// the part's `request` describes both forms). The part types it `any`, so both fit it; a
|
||||
// build module that does not know the packed form refuses the ask naming that it holds no request object.
|
||||
Request json.RawMessage `json:"request"`
|
||||
Reader string `json:"reader,omitempty"`
|
||||
}
|
||||
|
||||
// PackedRequestPrefix opens a compose job's request that travels gzip-compressed.
|
||||
const PackedRequestPrefix = "gzip+base64:"
|
||||
|
||||
// PackRequest sets the job's request from the JSON object raw, packed.
|
||||
func (c *ComposeJob) PackRequest(raw []byte) error {
|
||||
var b bytes.Buffer
|
||||
zw := gzip.NewWriter(&b)
|
||||
if _, err := zw.Write(raw); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := zw.Close(); err != nil {
|
||||
return err
|
||||
}
|
||||
packed, err := json.Marshal(PackedRequestPrefix + base64.StdEncoding.EncodeToString(b.Bytes()))
|
||||
c.Request = packed
|
||||
return err
|
||||
}
|
||||
|
||||
// RequestBytes is the job's request as a JSON object, unpacked when it travels packed.
|
||||
func (c ComposeJob) RequestBytes() ([]byte, error) {
|
||||
var packed string
|
||||
if len(c.Request) == 0 || c.Request[0] != '"' {
|
||||
return c.Request, nil
|
||||
}
|
||||
if err := json.Unmarshal(c.Request, &packed); err != nil || !strings.HasPrefix(packed, PackedRequestPrefix) {
|
||||
return nil, fmt.Errorf("buildstep: a compose job's request is a string that is no packed request")
|
||||
}
|
||||
raw, err := base64.StdEncoding.DecodeString(strings.TrimPrefix(packed, PackedRequestPrefix))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("buildstep: the packed request of a compose job is no base64: %w", err)
|
||||
}
|
||||
zr, err := gzip.NewReader(bytes.NewReader(raw))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("buildstep: the packed request of a compose job is no gzip: %w", err)
|
||||
}
|
||||
return io.ReadAll(zr)
|
||||
}
|
||||
|
||||
// AskOf is any seat's ask (or this form) as the neutral StepAsk: the one conversion, since the three seats' types
|
||||
// are the part's one layout. A value that does not fit the layout is refused.
|
||||
func AskOf[T any](seatAsk T) (StepAsk, error) {
|
||||
@@ -378,7 +424,11 @@ func (c ComposeJob) check(id string) error {
|
||||
if c.Package == "" {
|
||||
return fmt.Errorf("buildstep: the compose ask %s names no package to build", id)
|
||||
}
|
||||
if !json.Valid(c.Request) || len(c.Request) == 0 || c.Request[0] != '{' {
|
||||
req, err := c.RequestBytes()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !json.Valid(req) || len(req) == 0 || req[0] != '{' {
|
||||
return fmt.Errorf("buildstep: the compose ask %s holds no request object", id)
|
||||
}
|
||||
return nil
|
||||
|
||||
@@ -144,6 +144,28 @@ func TestAskCheckAndRoundTrip(t *testing.T) {
|
||||
if _, err := DecodeStepAsk([]byte(`{"v":2}`)); !errors.Is(err, ErrUnknownVersion) {
|
||||
t.Errorf("another version: %v", err)
|
||||
}
|
||||
// Issue 1966: a compose job's request travels packed when it passes the bus's message limit, and reads back as sent.
|
||||
t.Run("1966: a compose request travels packed", func(t *testing.T) {
|
||||
obj := []byte(`{"v":1,"input":{"nodes":[]}}`)
|
||||
job := ComposeJob{Fetch: Fetch{Repository: "novox/mesh-host", Commit: strings.Repeat("a", 40)}, Dir: ".", Package: "p"}
|
||||
if err := job.PackRequest(obj); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got, err := job.RequestBytes(); err != nil || string(got) != string(obj) {
|
||||
t.Errorf("read back %s, %v", got, err)
|
||||
}
|
||||
if err := job.check("ask"); err != nil {
|
||||
t.Errorf("a packed request is refused: %v", err)
|
||||
}
|
||||
job.Request = json.RawMessage(`"not packed"`)
|
||||
if err := job.check("ask"); err == nil {
|
||||
t.Errorf("a string that is no packed request is accepted")
|
||||
}
|
||||
job.Request = obj
|
||||
if got, err := job.RequestBytes(); err != nil || string(got) != string(obj) {
|
||||
t.Errorf("a whole request read back %s, %v", got, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// The seat's value and the neutral form are the same JSON: AskOf takes any seat's type (here its shape, with a
|
||||
|
||||
Reference in New Issue
Block a user