From 3f1d9cecba1e145d96a9202bcea742ca6860562f Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 9 Nov 2026 13:27:14 +0100 Subject: [PATCH 1/3] Checker keeps a composition request in pieces, each under the state's value limit (issue 1966) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit mesh-host #219's check failed: the request kept for g14 was 300127 bytes compressed, over the 262144 the state takes. The key of the check and node now holds a pointer (digest, size, pieces) and each piece of the gzip bytes is a value of its own under pieces//, at most 128 KiB; checker_composition_requests follows the pointer and checks the digest. Quick fix: a larger cap or a better compression: still a limit, met again as the mesh grows. Thorough fix (chosen, and bounded only by the bucket): pieces. Not an object store: the checker is a foundation part, whose data stays in the old sections (ADR 0619 §2), and the tool runner serves no old-section object read to its tools. No manifest change. Claude-Session: https://claude.ai/code/session_01NcbgfFfQ4Z9UE5jK32PmE5 --- .../cmd/mesh-delivery-checker-tools/main.go | 2 +- .../mesh-delivery-checker-tools/main_test.go | 18 ++++ .../internal/check/requests.go | 87 +++++++++++++++---- 3 files changed, 88 insertions(+), 19 deletions(-) diff --git a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker-tools/main.go b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker-tools/main.go index 84a9c770..89fd485d 100644 --- a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker-tools/main.go +++ b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker-tools/main.go @@ -378,7 +378,7 @@ func RequestTools(r check.StateRead) []stdio.Tool { Description: "The composition request a check sent to a node: the composer's whole input for that node (the node-engine's composition input, the case's " + "manifests and verdicts, the seat list), kept exactly as sent so a session reproduces the check's composition (the composer run by hand over the same " + "request gives the check's answer). Kept in the checker's state compositions, per check key (group.heads, the 12-digit heads hash) and node, " + - "about 430 KB each (kept compressed, so a value fits the state's 256 KiB), for 3 days (the state's ttl-seconds). With a check and a node, that request whole; with a check, " + + "0.7 to 0.8 MB each (kept compressed, in pieces that each fit the state's 256 KiB value), for 3 days (the state's ttl-seconds). With a check and a node, that request whole; with a check, " + "the nodes it kept one for; with neither, every kept request, each with its size, digest and when it was kept. Read-only.", Input: map[string]any{"type": "object", "properties": map[string]any{ "check": map[string]any{"type": "string", "description": "the check's key, ., as the verdict and composition name it"}, diff --git a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker-tools/main_test.go b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker-tools/main_test.go index 96f73db1..9b51fdd3 100644 --- a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker-tools/main_test.go +++ b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker-tools/main_test.go @@ -5,6 +5,7 @@ import ( "encoding/base64" "encoding/json" "errors" + "math/rand" "os" "regexp" "slices" @@ -48,7 +49,21 @@ func TestTheToolsServedAreTheFiveReadTools(t *testing.T) { t.Fatal(err) } } + // Issue 1966: a request whose compressed bytes pass the state's value limit is kept in pieces and read back whole; the + // pieces are no requests in the list. + rnd := make([]byte, 600<<10) + _, _ = rand.New(rand.NewSource(1)).Read(rnd) + big := []byte(`{"input":"` + base64.StdEncoding.EncodeToString(rnd) + `"}`) + if err := keeper.Keep(context.Background(), "big.0123456789ab", "anchor", big); err != nil { + t.Fatalf("a request of %d bytes: %v", len(big), err) + } rt := RequestTools(check.StateRead{Keys: store.Keys, Get: store.Get}) + if one, err := rt[0].Run(map[string]any{"check": "big.0123456789ab", "node": "anchor"}); err != nil || !strings.Contains(func() string { b, _ := json.Marshal(one); return string(b) }(), base64.StdEncoding.EncodeToString(rnd)[:64]) { + t.Errorf("the big request read back: %v", err) + } + if all, _ := rt[0].Run(map[string]any{}); strings.Count(func() string { b, _ := json.Marshal(all); return string(b) }(), `"node"`) != 4 { + t.Errorf("the list holds a piece as a request: %v", all) + } if len(rt) != 1 || rt[0].Name != check.RequestsReadTool || !strings.Contains(rt[0].Description, "composition request a check sent to a node") { t.Fatalf("request tools = %+v", rt) } @@ -312,6 +327,9 @@ func TestTheTryToolsAskAndReadATry(t *testing.T) { // ADR 0501 §5, the agreed i type memPut struct{ m *memStore } func (p memPut) Put(_ context.Context, key string, value []byte) (uint64, error) { + if len(value) > 262144 { // the state's value limit, headers included (issue 1966) + return 0, errors.New("the value is over the 262144 bytes the bucket takes") + } p.m.vals[key] = value return 1, nil } diff --git a/modules/mesh-delivery-checker/internal/check/requests.go b/modules/mesh-delivery-checker/internal/check/requests.go index 66f57613..07e1b0a3 100644 --- a/modules/mesh-delivery-checker/internal/check/requests.go +++ b/modules/mesh-delivery-checker/internal/check/requests.go @@ -20,20 +20,28 @@ import ( // session reproduces a check's composition exactly. The checker is a foundation part (novox/hq ADR 0619 §1-3), so its // kept data is a section of its manifest's old form, here `state`, and not an object store asked in `requires`. // -// **What bounds it.** A request is the node-engine's whole composition input of one node, about 430 KB as JSON. A -// state's value holds at most 256 KiB (kvlimit.ModuleStateValueBytes), so it is kept gzip-compressed (JSON shrinks about -// tenfold) in a document of its own; a request whose compressed bytes still do not fit is refused by the put, which -// fails the node's composition naming the size (never dropped). The manifest's ttl-seconds is its lifetime (3 days): -// the bus drops a value past it, so nothing waits on a timer here. A check kept again overwrites the key. +// **What bounds it.** A request is the node-engine's whole composition input of one node, 0.7 to 0.8 MB as JSON and +// growing with the mesh. A state's value holds at most 256 KiB (kvlimit.ModuleStateValueBytes, headers included), and +// the compressed request already passed it (issue 1966: 300127 bytes), so it is kept gzip-compressed in pieces: the key +// of the check and node holds a small document naming the request's digest, size and number of pieces, and each piece is +// a value of its own under RequestChunkKey, of at most RequestChunkBytes. The kept data is the checker's, a foundation +// part's, so it stays in the old section `state` (ADR 0619 §2) and is read through the tool runner's state read, which +// serves no object store of the old sections. The manifest's ttl-seconds is the lifetime (3 days) of the pointer and of +// every piece: the bus drops a value past it, so nothing waits on a timer here. A check kept again overwrites the key; +// a request whose pieces cannot all be put is refused naming the size (never dropped, never half kept: the pointer is +// put after its pieces). const ( // RequestsStateName is the checker's state of kept requests and RequestsBucket its bucket. RequestsStateName = "compositions" RequestsBucket = CheckerModule + "_" + RequestsStateName // RequestsReadTool is the tool that reads a kept request. RequestsReadTool = "checker_composition_requests" + // RequestChunkBytes is the most compressed bytes one piece holds: base64 of it, in its document, stays well under + // the state's 256 KiB value. + RequestChunkBytes = 128 << 10 ) -// KeptRequest is a request as the state holds it: whose it is, and the request's JSON, compressed. +// KeptRequest is the pointer to a request as the state holds it: whose it is, and what its pieces make. type KeptRequest struct { V int `json:"v"` Check string `json:"check"` @@ -42,8 +50,22 @@ type KeptRequest struct { // Size and Digest are the request's, uncompressed: its bytes and their sha256. Size int `json:"size"` Digest string `json:"digest"` - // Gzip is the request's bytes, gzip-compressed and base64-encoded. - Gzip string `json:"gzip"` + // Pieces is how many pieces (RequestChunkKey of the digest, 0 to Pieces-1) make the request's gzip bytes. + Pieces int `json:"pieces"` +} + +// KeptPiece is one piece of a request's gzip bytes, base64-encoded. +type KeptPiece struct { + V int `json:"v"` + Digest string `json:"digest"` + Index int `json:"index"` + Gzip string `json:"gzip"` +} + +// RequestChunkKey is the key of a request's piece: under the request's digest, so a request kept again for the same +// bytes writes the same pieces. +func RequestChunkKey(digest string, index int) string { + return fmt.Sprintf("pieces/%s/%d", digest, index) } // RequestKey is the key a check's request to a node is kept under: the check's key (group.heads) and the node. @@ -75,14 +97,28 @@ func (k RequestKeeper) Keep(ctx context.Context, key, node string, body []byte) now = k.Now } sum := sha256.Sum256(body) - doc, err := json.Marshal(KeptRequest{V: 1, Check: key, Node: node, KeptAt: now().UTC(), Size: len(body), - Digest: hex.EncodeToString(sum[:]), Gzip: base64.StdEncoding.EncodeToString(packed.Bytes())}) + digest := hex.EncodeToString(sum[:]) + pieces := (packed.Len() + RequestChunkBytes - 1) / RequestChunkBytes + fail := func(err error) error { + return fmt.Errorf("the composition request of check %s to %s (%d bytes, %d compressed in %d pieces) could not be kept in the state %s: %w", + key, node, len(body), packed.Len(), pieces, RequestsStateName, err) + } + for i := 0; i < pieces; i++ { + end := min((i+1)*RequestChunkBytes, packed.Len()) + doc, err := json.Marshal(KeptPiece{V: 2, Digest: digest, Index: i, Gzip: base64.StdEncoding.EncodeToString(packed.Bytes()[i*RequestChunkBytes : end])}) + if err != nil { + return err + } + if _, err := k.Store.Put(ctx, RequestChunkKey(digest, i), doc); err != nil { + return fail(err) + } + } + doc, err := json.Marshal(KeptRequest{V: 2, Check: key, Node: node, KeptAt: now().UTC(), Size: len(body), Digest: digest, Pieces: pieces}) if err != nil { return err } if _, err := k.Store.Put(ctx, RequestKey(key, node), doc); err != nil { - return fmt.Errorf("the composition request of check %s to %s (%d bytes, %d compressed) could not be kept in the state %s: %w", - key, node, len(body), packed.Len(), RequestsStateName, err) + return fail(err) } return nil } @@ -104,7 +140,7 @@ func ReadRequests(s StateRead, key, node string) (any, error) { if len(raw) == 0 { return nil, fmt.Errorf("no composition request of check %s to %s is kept (none was sent, or it is past its lifetime)", key, node) } - kept, body, err := unpackRequest(raw) + kept, body, err := unpackRequest(s, raw) if err != nil { return nil, fmt.Errorf("the kept request %s: %w", RequestKey(key, node), err) } @@ -129,20 +165,35 @@ func ReadRequests(s StateRead, key, node string) (any, error) { listed = append(listed, map[string]any{"key": k, "unreadable": err.Error()}) continue } + if kept.Check == "" { + continue // a piece of a request, not a request + } listed = append(listed, map[string]any{"check": kept.Check, "node": kept.Node, "size": kept.Size, "digest": kept.Digest, "kept_at": kept.KeptAt}) } return map[string]any{"kept": listed}, nil } -// unpackRequest reads a kept document and its request's bytes, which must match the digest they were kept with. -func unpackRequest(raw []byte) (KeptRequest, []byte, error) { +// unpackRequest reads a kept pointer's pieces and the request's bytes, which must match the digest they were kept with. +func unpackRequest(s StateRead, raw []byte) (KeptRequest, []byte, error) { var kept KeptRequest if err := json.Unmarshal(raw, &kept); err != nil { return kept, nil, fmt.Errorf("is no kept request: %w", err) } - packed, err := base64.StdEncoding.DecodeString(kept.Gzip) - if err != nil { - return kept, nil, fmt.Errorf("holds no base64: %w", err) + var packed []byte + for i := 0; i < kept.Pieces; i++ { + raw, err := s.Get(RequestChunkKey(kept.Digest, i)) + if err != nil { + return kept, nil, err + } + var piece KeptPiece + if err := json.Unmarshal(raw, &piece); err != nil || len(raw) == 0 { + return kept, nil, fmt.Errorf("lacks its piece %d of %d (past its lifetime, or never kept)", i, kept.Pieces) + } + part, err := base64.StdEncoding.DecodeString(piece.Gzip) + if err != nil { + return kept, nil, fmt.Errorf("holds no base64 in its piece %d: %w", i, err) + } + packed = append(packed, part...) } zr, err := gzip.NewReader(bytes.NewReader(packed)) if err != nil { From 173c49fc87f12a86062f474989162709120465f8 Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 9 Nov 2026 13:36:38 +0100 Subject: [PATCH 2/3] The compose ask carries its request gzip-packed; requests of another version are refused by name (issue 1966) A large node's request is 1.2 MB, over the bus's 1 MiB message limit, so the compose ask failed once keeping no longer did. The checker packs the request (a JSON string "gzip+base64:..." in the part's `any` request, no contract change) and buildstep.ComposeJob.RequestBytes reads either form. A kept request of another version is refused naming it, and listing no longer reads pieces. An older mesh-build-go refuses a packed ask loudly (it holds no request object): the check fails naming it, so the catalog change is delivered with this one as a group. Quick fix (chosen as the recovery): pack the request, ~290 KB for the largest node. Thorough fix (milestone G): the request named by digest and served on a seat verb, as RunInput is. Claude-Session: https://claude.ai/code/session_01NcbgfFfQ4Z9UE5jK32PmE5 --- .../mesh-delivery-checker-tools/main_test.go | 15 +++++ .../internal/check/engines.go | 4 ++ .../internal/check/engines_test.go | 21 +++++-- .../internal/check/requests.go | 19 ++++++- sdk/buildstep/ask.go | 57 +++++++++++++++++-- sdk/buildstep/ask_test.go | 22 +++++++ 6 files changed, 126 insertions(+), 12 deletions(-) diff --git a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker-tools/main_test.go b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker-tools/main_test.go index 9b51fdd3..a765459f 100644 --- a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker-tools/main_test.go +++ b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker-tools/main_test.go @@ -61,6 +61,21 @@ func TestTheToolsServedAreTheFiveReadTools(t *testing.T) { if one, err := rt[0].Run(map[string]any{"check": "big.0123456789ab", "node": "anchor"}); err != nil || !strings.Contains(func() string { b, _ := json.Marshal(one); return string(b) }(), base64.StdEncoding.EncodeToString(rnd)[:64]) { t.Errorf("the big request read back: %v", err) } + // Listing reads no piece, and a request of another version is refused by name. + store.vals["v1.0123456789ab/anchor"] = json.RawMessage(`{"v":1,"check":"v1.0123456789ab","node":"anchor","gzip":"x"}`) + noPieces := check.StateRead{Keys: store.Keys, Get: func(k string) (json.RawMessage, error) { + if strings.HasPrefix(k, "pieces/") { + return nil, errors.New("a piece was read") + } + return store.Get(k) + }} + if _, err := RequestTools(noPieces)[0].Run(map[string]any{}); err != nil { + t.Errorf("listing: %v", err) + } + if _, err := rt[0].Run(map[string]any{"check": "v1.0123456789ab", "node": "anchor"}); err == nil || !strings.Contains(err.Error(), "version 1") { + t.Errorf("a version 1 request: %v", err) + } + delete(store.vals, "v1.0123456789ab/anchor") if all, _ := rt[0].Run(map[string]any{}); strings.Count(func() string { b, _ := json.Marshal(all); return string(b) }(), `"node"`) != 4 { t.Errorf("the list holds a piece as a request: %v", all) } diff --git a/modules/mesh-delivery-checker/internal/check/engines.go b/modules/mesh-delivery-checker/internal/check/engines.go index 1a4a9971..6cfe453d 100644 --- a/modules/mesh-delivery-checker/internal/check/engines.go +++ b/modules/mesh-delivery-checker/internal/check/engines.go @@ -228,6 +228,10 @@ func (e EngineCompositions) inSandbox(ctx context.Context, c *Case, head, node s } job := buildstep.ComposeJob{Fetch: buildstep.Fetch{Repository: nodeEngineRepo, Commit: head, Packages: packagesFor(e.Published, c, c.Tagged, nodeEngineRepo)}, Dir: meshcatalogclient.RootPath, Package: composer.Package, Request: raw, Reader: "the node-engine head of " + c.Group} + // The ask travels on the bus, whose message limit a whole request passes (1.2 MB for a large node): it goes packed. + if err := job.PackRequest(raw); err != nil { + return a, err + } verdict, err := e.Sandbox.Compose(ctx, job) if err != nil { return a, fmt.Errorf("the composer at the node-engine's head %s did not answer for %s: %w", head, node, err) diff --git a/modules/mesh-delivery-checker/internal/check/engines_test.go b/modules/mesh-delivery-checker/internal/check/engines_test.go index f214f337..1c76571e 100644 --- a/modules/mesh-delivery-checker/internal/check/engines_test.go +++ b/modules/mesh-delivery-checker/internal/check/engines_test.go @@ -59,7 +59,11 @@ type fakeSandbox struct { func (f *fakeSandbox) Compose(_ context.Context, job buildstep.ComposeJob) (composer.Verdict, error) { var req composer.Request - if err := json.Unmarshal(job.Request, &req); err != nil { + body, err := job.RequestBytes() + if err != nil { + return composer.Verdict{}, err + } + if err := json.Unmarshal(body, &req); err != nil { return composer.Verdict{}, err } f.mu.Lock() @@ -206,7 +210,11 @@ func TestTheCompositionIsMadeFromTheEnginesAnswers(t *testing.T) { } job := sb.jobs[0] var req composer.Request - _ = json.Unmarshal(job.Request, &req) + if len(job.Request) == 0 || job.Request[0] != '"' { + t.Errorf("the request went whole, not packed (issue 1966): %.60s", job.Request) + } + body, _ := job.RequestBytes() + _ = json.Unmarshal(body, &req) if job.Fetch.Repository != nodeEngineRepo || job.Fetch.Commit != head2 || job.Package != composer.Package || string(req.Input) != `{"nodes":[{"name":"anchor"}]}` || req.Cases[0].Manifests["web"].Digest == "" || len(req.Seats) != 1 || string(req.Seats[0]) != string(host.Seats[0]) { @@ -220,8 +228,9 @@ func TestTheCompositionIsMadeFromTheEnginesAnswers(t *testing.T) { t.Fatal(err) } keyOfHost, _ := nodeengineclient.CompositionKey(host.Group, host.Heads) + lastBody, _ := sb.jobs[len(sb.jobs)-1].RequestBytes() got, err := ReadRequests(store.read(), keyOfHost, "anchor") - if b, _ := json.Marshal(got); err != nil || !strings.Contains(string(b), `"request":`+string(sb.jobs[len(sb.jobs)-1].Request)) { + if b, _ := json.Marshal(got); err != nil || !strings.Contains(string(b), `"request":`+string(lastBody)) { t.Errorf("kept %s, %v: want the request of the last job under %s/anchor", b, err, keyOfHost) } store.err = errors.New("the state is full") @@ -311,7 +320,8 @@ func TestTheCompositionIsMadeFromTheEnginesAnswers(t *testing.T) { t.Fatalf("jobs %+v: want one job at the node-engine's trunk", sb.jobs) } var req composer.Request - _ = json.Unmarshal(sb.jobs[0].Request, &req) + b0, _ := sb.jobs[0].RequestBytes() + _ = json.Unmarshal(b0, &req) if got := string(req.Cases[0].Verdicts["mesh-build-go"]); got != string(brings.Judged["mesh-build-go"]) || len(req.Cases[0].Verdicts) != 2 { t.Errorf("the request's verdicts %s: want the check's judge's for mesh-build-go and the indexer's for mail", req.Cases[0].Verdicts) } @@ -321,7 +331,8 @@ func TestTheCompositionIsMadeFromTheEnginesAnswers(t *testing.T) { t.Fatal(err) } var none composer.Request - _ = json.Unmarshal(sb.jobs[len(sb.jobs)-1].Request, &none) + bn, _ := sb.jobs[len(sb.jobs)-1].RequestBytes() + _ = json.Unmarshal(bn, &none) if _, kept := none.Cases[0].Verdicts["mesh-build-go"]; kept { t.Errorf("the running indexer's verdict on the change's manifest was kept: %s", none.Cases[0].Verdicts) } diff --git a/modules/mesh-delivery-checker/internal/check/requests.go b/modules/mesh-delivery-checker/internal/check/requests.go index 07e1b0a3..46d4531d 100644 --- a/modules/mesh-delivery-checker/internal/check/requests.go +++ b/modules/mesh-delivery-checker/internal/check/requests.go @@ -62,6 +62,16 @@ type KeptPiece struct { Gzip string `json:"gzip"` } +// isPieceKey is whether a key is a piece's: under pieces/ and a sha256 digest, which no check's key (group.heads/node) is. +func isPieceKey(k string) bool { + rest, ok := strings.CutPrefix(k, "pieces/") + if !ok { + return false + } + digest, _, _ := strings.Cut(rest, "/") + return len(digest) == 64 +} + // RequestChunkKey is the key of a request's piece: under the request's digest, so a request kept again for the same // bytes writes the same pieces. func RequestChunkKey(digest string, index int) string { @@ -156,6 +166,9 @@ func ReadRequests(s StateRead, key, node string) (any, error) { if key != "" && !strings.HasPrefix(k, key+"/") { continue } + if isPieceKey(k) { + continue // a piece of a request, not a request: never read here + } raw, err := s.Get(k) if err != nil { return nil, err @@ -165,9 +178,6 @@ func ReadRequests(s StateRead, key, node string) (any, error) { listed = append(listed, map[string]any{"key": k, "unreadable": err.Error()}) continue } - if kept.Check == "" { - continue // a piece of a request, not a request - } listed = append(listed, map[string]any{"check": kept.Check, "node": kept.Node, "size": kept.Size, "digest": kept.Digest, "kept_at": kept.KeptAt}) } return map[string]any{"kept": listed}, nil @@ -179,6 +189,9 @@ func unpackRequest(s StateRead, raw []byte) (KeptRequest, []byte, error) { if err := json.Unmarshal(raw, &kept); err != nil { return kept, nil, fmt.Errorf("is no kept request: %w", err) } + if kept.V != 2 { + return kept, nil, fmt.Errorf("is a kept request of version %d, which this reader does not read (it reads version 2; an older one is past its lifetime within 3 days)", kept.V) + } var packed []byte for i := 0; i < kept.Pieces; i++ { raw, err := s.Get(RequestChunkKey(kept.Digest, i)) diff --git a/sdk/buildstep/ask.go b/sdk/buildstep/ask.go index 6184dad3..1955f475 100644 --- a/sdk/buildstep/ask.go +++ b/sdk/buildstep/ask.go @@ -1,8 +1,12 @@ package buildstep import ( + "bytes" + "compress/gzip" + "encoding/base64" "encoding/json" "fmt" + "io" "path" "slices" "sort" @@ -281,13 +285,54 @@ type CheckJob struct { // ComposeJob asks one compose-candidate build. type ComposeJob struct { - Fetch Fetch `json:"fetch"` - Dir string `json:"dir"` - Package string `json:"package"` + Fetch Fetch `json:"fetch"` + Dir string `json:"dir"` + Package string `json:"package"` + // Request is the composer's request: a JSON object, or, for one near the bus's message limit (issue 1966), a JSON + // string PackedRequestPrefix + the base64 of the object gzip-compressed. The part types it `any`, so both fit it; a + // build module that does not know the packed form refuses the ask naming that it holds no request object. Request json.RawMessage `json:"request"` Reader string `json:"reader,omitempty"` } +// PackedRequestPrefix opens a compose job's request that travels gzip-compressed. +const PackedRequestPrefix = "gzip+base64:" + +// PackRequest sets the job's request from the JSON object raw, packed. +func (c *ComposeJob) PackRequest(raw []byte) error { + var b bytes.Buffer + zw := gzip.NewWriter(&b) + if _, err := zw.Write(raw); err != nil { + return err + } + if err := zw.Close(); err != nil { + return err + } + packed, err := json.Marshal(PackedRequestPrefix + base64.StdEncoding.EncodeToString(b.Bytes())) + c.Request = packed + return err +} + +// RequestBytes is the job's request as a JSON object, unpacked when it travels packed. +func (c ComposeJob) RequestBytes() ([]byte, error) { + var packed string + if len(c.Request) == 0 || c.Request[0] != '"' { + return c.Request, nil + } + if err := json.Unmarshal(c.Request, &packed); err != nil || !strings.HasPrefix(packed, PackedRequestPrefix) { + return nil, fmt.Errorf("buildstep: a compose job's request is a string that is no packed request") + } + raw, err := base64.StdEncoding.DecodeString(strings.TrimPrefix(packed, PackedRequestPrefix)) + if err != nil { + return nil, fmt.Errorf("buildstep: the packed request of a compose job is no base64: %w", err) + } + zr, err := gzip.NewReader(bytes.NewReader(raw)) + if err != nil { + return nil, fmt.Errorf("buildstep: the packed request of a compose job is no gzip: %w", err) + } + return io.ReadAll(zr) +} + // AskOf is any seat's ask (or this form) as the neutral StepAsk: the one conversion, since the three seats' types // are the part's one layout. A value that does not fit the layout is refused. func AskOf[T any](seatAsk T) (StepAsk, error) { @@ -378,7 +423,11 @@ func (c ComposeJob) check(id string) error { if c.Package == "" { return fmt.Errorf("buildstep: the compose ask %s names no package to build", id) } - if !json.Valid(c.Request) || len(c.Request) == 0 || c.Request[0] != '{' { + req, err := c.RequestBytes() + if err != nil { + return err + } + if !json.Valid(req) || len(req) == 0 || req[0] != '{' { return fmt.Errorf("buildstep: the compose ask %s holds no request object", id) } return nil diff --git a/sdk/buildstep/ask_test.go b/sdk/buildstep/ask_test.go index e8272b88..ce960a1b 100644 --- a/sdk/buildstep/ask_test.go +++ b/sdk/buildstep/ask_test.go @@ -144,6 +144,28 @@ func TestAskCheckAndRoundTrip(t *testing.T) { if _, err := DecodeStepAsk([]byte(`{"v":2}`)); !errors.Is(err, ErrUnknownVersion) { t.Errorf("another version: %v", err) } + // Issue 1966: a compose job's request travels packed when it passes the bus's message limit, and reads back as sent. + t.Run("1966: a compose request travels packed", func(t *testing.T) { + obj := []byte(`{"v":1,"input":{"nodes":[]}}`) + job := ComposeJob{Fetch: Fetch{Repository: "novox/mesh-host", Commit: strings.Repeat("a", 40)}, Dir: ".", Package: "p"} + if err := job.PackRequest(obj); err != nil { + t.Fatal(err) + } + if got, err := job.RequestBytes(); err != nil || string(got) != string(obj) { + t.Errorf("read back %s, %v", got, err) + } + if err := job.check("ask"); err != nil { + t.Errorf("a packed request is refused: %v", err) + } + job.Request = json.RawMessage(`"not packed"`) + if err := job.check("ask"); err == nil { + t.Errorf("a string that is no packed request is accepted") + } + job.Request = obj + if got, err := job.RequestBytes(); err != nil || string(got) != string(obj) { + t.Errorf("a whole request read back %s, %v", got, err) + } + }) } // The seat's value and the neutral form are the same JSON: AskOf takes any seat's type (here its shape, with a From bf7540cd9ec2799c4a738b41bf5a1ba33f8c2896 Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 9 Nov 2026 13:41:53 +0100 Subject: [PATCH 3/3] The build step part's request names both forms; the three build seats' contracts regenerated (issue 1966) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The compose job's `request` description now says it is the request object or the JSON string "gzip+base64:" followed by the base64 of the gzip-compressed object, as ADR 0634 §1 puts a seat's messages in its contract. The field stays `any`; seatgen regenerated node-build-go, node-build-node and node-build-image (Go and TypeScript), and its -check is clean. ask.go's comment says the checker packs every request it sends. Delivery: an old holder of node-build-go refuses the packed ask loudly (it holds no request object) until it runs the new mesh-build-go; the checker is built by mesh-build-go after it (module graph edge), and every check composing ace or novox fails today anyway. Claude-Session: https://claude.ai/code/session_01NcbgfFfQ4Z9UE5jK32PmE5 --- contracts/go/seats/nodebuildgo/contract.go | 2 +- contracts/go/seats/nodebuildimage/contract.go | 2 +- contracts/go/seats/nodebuildnode/contract.go | 2 +- contracts/parts/build-step.json | 2 +- contracts/ts/seats/nodebuildgo/src/parts/build-step.ts | 2 +- contracts/ts/seats/nodebuildimage/src/parts/build-step.ts | 2 +- contracts/ts/seats/nodebuildnode/src/parts/build-step.ts | 2 +- sdk/buildstep/ask.go | 5 +++-- 8 files changed, 10 insertions(+), 9 deletions(-) diff --git a/contracts/go/seats/nodebuildgo/contract.go b/contracts/go/seats/nodebuildgo/contract.go index 5deeb05e..61e93a98 100644 --- a/contracts/go/seats/nodebuildgo/contract.go +++ b/contracts/go/seats/nodebuildgo/contract.go @@ -485,7 +485,7 @@ type StepAskCompose struct { Fetch Fetch `json:"fetch"` Package string `json:"package"` Reader string `json:"reader,omitempty"` - // the compose-candidate request + // the compose-candidate request, in one of two forms: the request as a JSON object, or, for one near the bus's message limit, a JSON string "gzip+base64:" followed by the base64 of the request object gzip-compressed; a holder that does not know the packed form refuses it, naming that it holds no request object Request any `json:"request"` } diff --git a/contracts/go/seats/nodebuildimage/contract.go b/contracts/go/seats/nodebuildimage/contract.go index dde300d1..8ea6e02f 100644 --- a/contracts/go/seats/nodebuildimage/contract.go +++ b/contracts/go/seats/nodebuildimage/contract.go @@ -485,7 +485,7 @@ type StepAskCompose struct { Fetch Fetch `json:"fetch"` Package string `json:"package"` Reader string `json:"reader,omitempty"` - // the compose-candidate request + // the compose-candidate request, in one of two forms: the request as a JSON object, or, for one near the bus's message limit, a JSON string "gzip+base64:" followed by the base64 of the request object gzip-compressed; a holder that does not know the packed form refuses it, naming that it holds no request object Request any `json:"request"` } diff --git a/contracts/go/seats/nodebuildnode/contract.go b/contracts/go/seats/nodebuildnode/contract.go index ba461926..b12f7595 100644 --- a/contracts/go/seats/nodebuildnode/contract.go +++ b/contracts/go/seats/nodebuildnode/contract.go @@ -485,7 +485,7 @@ type StepAskCompose struct { Fetch Fetch `json:"fetch"` Package string `json:"package"` Reader string `json:"reader,omitempty"` - // the compose-candidate request + // the compose-candidate request, in one of two forms: the request as a JSON object, or, for one near the bus's message limit, a JSON string "gzip+base64:" followed by the base64 of the request object gzip-compressed; a holder that does not know the packed form refuses it, naming that it holds no request object Request any `json:"request"` } diff --git a/contracts/parts/build-step.json b/contracts/parts/build-step.json index 37c20cb4..9dc9c2ed 100644 --- a/contracts/parts/build-step.json +++ b/contracts/parts/build-step.json @@ -161,7 +161,7 @@ "type": "string" }, "request": { - "description": "the compose-candidate request" + "description": "the compose-candidate request, in one of two forms: the request as a JSON object, or, for one near the bus's message limit, a JSON string \"gzip+base64:\" followed by the base64 of the request object gzip-compressed; a holder that does not know the packed form refuses it, naming that it holds no request object" }, "reader": { "type": "string" diff --git a/contracts/ts/seats/nodebuildgo/src/parts/build-step.ts b/contracts/ts/seats/nodebuildgo/src/parts/build-step.ts index c8843899..1aee13ff 100644 --- a/contracts/ts/seats/nodebuildgo/src/parts/build-step.ts +++ b/contracts/ts/seats/nodebuildgo/src/parts/build-step.ts @@ -293,7 +293,7 @@ export interface StepAskCompose { "fetch": Fetch; "package": string; "reader"?: string; - /** the compose-candidate request */ + /** the compose-candidate request, in one of two forms: the request as a JSON object, or, for one near the bus's message limit, a JSON string "gzip+base64:" followed by the base64 of the request object gzip-compressed; a holder that does not know the packed form refuses it, naming that it holds no request object */ "request": unknown; } diff --git a/contracts/ts/seats/nodebuildimage/src/parts/build-step.ts b/contracts/ts/seats/nodebuildimage/src/parts/build-step.ts index c8843899..1aee13ff 100644 --- a/contracts/ts/seats/nodebuildimage/src/parts/build-step.ts +++ b/contracts/ts/seats/nodebuildimage/src/parts/build-step.ts @@ -293,7 +293,7 @@ export interface StepAskCompose { "fetch": Fetch; "package": string; "reader"?: string; - /** the compose-candidate request */ + /** the compose-candidate request, in one of two forms: the request as a JSON object, or, for one near the bus's message limit, a JSON string "gzip+base64:" followed by the base64 of the request object gzip-compressed; a holder that does not know the packed form refuses it, naming that it holds no request object */ "request": unknown; } diff --git a/contracts/ts/seats/nodebuildnode/src/parts/build-step.ts b/contracts/ts/seats/nodebuildnode/src/parts/build-step.ts index c8843899..1aee13ff 100644 --- a/contracts/ts/seats/nodebuildnode/src/parts/build-step.ts +++ b/contracts/ts/seats/nodebuildnode/src/parts/build-step.ts @@ -293,7 +293,7 @@ export interface StepAskCompose { "fetch": Fetch; "package": string; "reader"?: string; - /** the compose-candidate request */ + /** the compose-candidate request, in one of two forms: the request as a JSON object, or, for one near the bus's message limit, a JSON string "gzip+base64:" followed by the base64 of the request object gzip-compressed; a holder that does not know the packed form refuses it, naming that it holds no request object */ "request": unknown; } diff --git a/sdk/buildstep/ask.go b/sdk/buildstep/ask.go index 1955f475..5bdf2647 100644 --- a/sdk/buildstep/ask.go +++ b/sdk/buildstep/ask.go @@ -288,8 +288,9 @@ type ComposeJob struct { Fetch Fetch `json:"fetch"` Dir string `json:"dir"` Package string `json:"package"` - // Request is the composer's request: a JSON object, or, for one near the bus's message limit (issue 1966), a JSON - // string PackedRequestPrefix + the base64 of the object gzip-compressed. The part types it `any`, so both fit it; a + // Request is the composer's request: a JSON object, or a JSON string PackedRequestPrefix + the base64 of the object + // gzip-compressed (issue 1966; the checker packs every request it sends, and + // the part's `request` describes both forms). The part types it `any`, so both fit it; a // build module that does not know the packed form refuses the ask naming that it holds no request object. Request json.RawMessage `json:"request"` Reader string `json:"reader,omitempty"`