diff --git a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker-tools/main.go b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker-tools/main.go index f7ce0dca..72953a5e 100644 --- a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker-tools/main.go +++ b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker-tools/main.go @@ -325,6 +325,24 @@ func Tools(r check.Reader) []stdio.Tool { } } +// RequestTools is the tool that reads the composition requests the checker sent to nodes (issue 1921). +func RequestTools(r check.StateRead) []stdio.Tool { + return []stdio.Tool{{Name: check.RequestsReadTool, + Description: "The composition request a check sent to a node: the composer's whole input for that node (the node-engine's composition input, the case's " + + "manifests and verdicts, the seat list), kept exactly as sent so a session reproduces the check's composition (the composer run by hand over the same " + + "request gives the check's answer). Kept in the checker's state compositions, per check key (group.heads, the 12-digit heads hash) and node, " + + "about 430 KB each (kept compressed, so a value fits the state's 256 KiB), for 3 days (the state's ttl-seconds). With a check and a node, that request whole; with a check, " + + "the nodes it kept one for; with neither, every kept request, each with its size, digest and when it was kept. Read-only.", + Input: map[string]any{"type": "object", "properties": map[string]any{ + "check": map[string]any{"type": "string", "description": "the check's key, ., as the verdict and composition name it"}, + "node": map[string]any{"type": "string", "description": "the node the request was sent for, with a check"}}}, + Run: func(args map[string]any) (any, error) { + key, _ := args["check"].(string) + node, _ := args["node"].(string) + return check.ReadRequests(r, key, node) + }}} +} + // RecoveryTools are the tools that steer a check by hand (tools-are-how-a-module-is-spoken-to): ask a check again whose // wait judged nothing (the lost check of 2026-10-28). They write the checker's verdicts through w. func RecoveryTools(r check.Reader, w check.VerdictEntry) []stdio.Tool { @@ -403,6 +421,7 @@ func main() { // states stand, and asking one again by hand (ADR 0615, DLV-M2). tools = append(tools, views.StatusTools(check.CheckerModule, "checker_first_views", "checker_first_view_again", stdio.State(views.StatusState), meshhandactsclient.Recorder{Ask: stdio.Ask}, time.Now)...) tools = append(tools, SeatInputTools(r)...) + tools = append(tools, RequestTools(read(stdio.State(check.RequestsStateName)))...) if err := stdio.Serve("", logged(tools)); err != nil { fmt.Fprintf(os.Stderr, "[mesh-delivery-checker-tools] %v\n", err) os.Exit(1) diff --git a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker-tools/main_test.go b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker-tools/main_test.go index 10a4bb90..01b9bef8 100644 --- a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker-tools/main_test.go +++ b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker-tools/main_test.go @@ -1,6 +1,7 @@ package main import ( + "context" "encoding/base64" "encoding/json" "errors" @@ -37,6 +38,35 @@ func TestTheToolsServedAreTheFiveReadTools(t *testing.T) { if _, err := recovery[0].Run(map[string]any{"group": "feat/x", "why": "lost"}); err == nil { t.Errorf("%s answered with no state to write", check.AskAgainTool) } + // Issue 1921: the composition requests kept by the checker are read by check and node; the tool refuses a node with no check + // and a state it cannot read, and lists a check's nodes without their bodies. + store := &memStore{vals: map[string]json.RawMessage{}} + keeper := check.RequestKeeper{Store: memPut{store}} + for _, k := range [][2]string{{"g.0123456789ab", "anchor"}, {"g.0123456789ab", "ace"}, {"h.0123456789ab", "ace"}} { + if err := keeper.Keep(context.Background(), k[0], k[1], []byte(`{"input":{"nodes":[]}}`)); err != nil { + t.Fatal(err) + } + } + rt := RequestTools(check.StateRead{Keys: store.Keys, Get: store.Get}) + if len(rt) != 1 || rt[0].Name != check.RequestsReadTool || !strings.Contains(rt[0].Description, "composition request a check sent to a node") { + t.Fatalf("request tools = %+v", rt) + } + one, err := rt[0].Run(map[string]any{"check": "g.0123456789ab", "node": "anchor"}) + if b, _ := json.Marshal(one); err != nil || !strings.Contains(string(b), `"request":{"input":{"nodes":[]}}`) { + t.Errorf("one request = %s, %v", b, err) + } + list, _ := rt[0].Run(map[string]any{"check": "g.0123456789ab"}) + if b, _ := json.Marshal(list); strings.Count(string(b), `"node"`) != 2 || strings.Contains(string(b), "h.0123") || strings.Contains(string(b), `"request"`) { + t.Errorf("list of a check = %s", b) + } + for _, args := range []map[string]any{{"node": "anchor"}, {"check": "g.0123456789ab", "node": "gone"}} { + if _, err := rt[0].Run(args); err == nil { + t.Errorf("%v answered", args) + } + } + if _, err := RequestTools(check.StateRead{})[0].Run(map[string]any{}); err == nil { + t.Errorf("answered with no state to read") + } // With nothing to read from a tool says so and changes nothing. for _, tl := range Tools(check.Reader{}) { if tl.Name == check.StatusReadTool { @@ -209,3 +239,11 @@ func TestTheTryToolsAskAndReadATry(t *testing.T) { // ADR 0501 §5, the agreed i t.Errorf("hq: %v", err) } } + +// memPut writes a memStore as the checker's kept-request state is written. +type memPut struct{ m *memStore } + +func (p memPut) Put(_ context.Context, key string, value []byte) (uint64, error) { + p.m.vals[key] = value + return 1, nil +} diff --git a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/main.go b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/main.go index 7a149d19..b178e634 100644 --- a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/main.go +++ b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/main.go @@ -105,6 +105,8 @@ var keptStates = []announce.State{ // which serve one on the seat's verb input to the Go build module that runs the program (check.InputVerb; ADR 0596 §2, // issue 1921). {Name: check.InputsStateName}, + // The composition requests it sent the composer, per check and node (check.RequestsStateName): read by its tools alone. + {Name: check.RequestsStateName}, {Name: "kept", Readable: true}, {Name: "owed", Readable: true}, {Name: "previews", Readable: true}, @@ -225,6 +227,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro waiter := buildrun.LostAskWaiter{Inner: heard, Dead: dead, Hearing: hearing} asksKV := process.AwaitBucket(ctx, js, conn, check.AsksBucket, "checker_ask_again refuses every check, as it cannot tell whether its ask waits in a stream", logger.Printf) inputsKV := process.AwaitBucket(ctx, js, conn, check.InputsBucket, "no checking program can be asked, so every check fails naming it", logger.Printf) + requestsKV := process.AwaitBucket(ctx, js, conn, check.RequestsBucket, "no composition request is kept, so every composition made with the node-engine's head fails naming it", logger.Printf) ch := &check.Checker{ Asks: awaitedKV{asksKV}, Verdicts: kvStore{buckets[check.CheckerVerdictsBucket]}, @@ -255,6 +258,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro holdCallsFor(answerBound) verdicts := &manifestVerdicts{} compositions := check.EngineCompositions{ + Requests: check.RequestKeeper{Store: requestsKV}, Asker: engineAsks{call: caller.CallOn}, Placed: placedNodes{kv: func(context.Context) (jetstream.KeyValue, error) { return assignments, nil }}.Placed, Bound: answerBound, diff --git a/modules/mesh-delivery-checker/internal/check/engines.go b/modules/mesh-delivery-checker/internal/check/engines.go index f4fe5ea9..1a4a9971 100644 --- a/modules/mesh-delivery-checker/internal/check/engines.go +++ b/modules/mesh-delivery-checker/internal/check/engines.go @@ -67,6 +67,11 @@ type EngineCompositions struct { LatestPackages() buildstep.PackageVersions } Verdicts func() map[string]meshcatalog.VerdictsByDigestValue + // Requests keeps each request sent to the composer, per check and node (issue 1921); nil keeps none. A request that + // cannot be kept fails the node's composition, naming why: never dropped. + Requests interface { + Keep(ctx context.Context, key, node string, body []byte) error + } } // composerHead is the node-engine member's head of a case, or empty. @@ -212,6 +217,15 @@ func (e EngineCompositions) inSandbox(ctx context.Context, c *Case, head, node s if err != nil { return a, err } + if e.Requests != nil { + key, err := nodeengineclient.CompositionKey(c.Group, c.Heads) + if err == nil { + err = e.Requests.Keep(ctx, key, node, raw) + } + if err != nil { + return a, err + } + } job := buildstep.ComposeJob{Fetch: buildstep.Fetch{Repository: nodeEngineRepo, Commit: head, Packages: packagesFor(e.Published, c, c.Tagged, nodeEngineRepo)}, Dir: meshcatalogclient.RootPath, Package: composer.Package, Request: raw, Reader: "the node-engine head of " + c.Group} verdict, err := e.Sandbox.Compose(ctx, job) diff --git a/modules/mesh-delivery-checker/internal/check/engines_test.go b/modules/mesh-delivery-checker/internal/check/engines_test.go index 76eb06dc..f214f337 100644 --- a/modules/mesh-delivery-checker/internal/check/engines_test.go +++ b/modules/mesh-delivery-checker/internal/check/engines_test.go @@ -71,6 +71,36 @@ func (f *fakeSandbox) Compose(_ context.Context, job buildstep.ComposeJob) (comp return f.v(req.Machines[0]), nil } +// keptRequests is the state of composition requests in memory; a put fails with err when it is set. +type keptRequests struct { + mu sync.Mutex + vals map[string][]byte + err error +} + +func (k *keptRequests) Put(_ context.Context, key string, value []byte) (uint64, error) { + k.mu.Lock() + defer k.mu.Unlock() + if k.err != nil { + return 0, k.err + } + if k.vals == nil { + k.vals = map[string][]byte{} + } + k.vals[key] = value + return 1, nil +} + +func (k *keptRequests) read() StateRead { + return StateRead{Keys: func() ([]string, error) { + var out []string + for n := range k.vals { + out = append(out, n) + } + return out, nil + }, Get: func(key string) (json.RawMessage, error) { return k.vals[key], nil }} +} + type digests map[string]string func (d digests) ManifestDigests() map[string]string { return d } @@ -182,6 +212,24 @@ func TestTheCompositionIsMadeFromTheEnginesAnswers(t *testing.T) { len(req.Seats) != 1 || string(req.Seats[0]) != string(host.Seats[0]) { t.Errorf("job = %+v, request %+v", job, req) } + // Issue 1921 (notes 66-67): the request sent to the composer is kept per check and node, exactly as sent (read back + // whole through the tool's reader); a request that cannot be kept fails the node's composition, naming why. + store := &keptRequests{} + ec.Requests = RequestKeeper{Store: store} + if _, err := ec.Compose(context.Background(), host); err != nil { + t.Fatal(err) + } + keyOfHost, _ := nodeengineclient.CompositionKey(host.Group, host.Heads) + got, err := ReadRequests(store.read(), keyOfHost, "anchor") + if b, _ := json.Marshal(got); err != nil || !strings.Contains(string(b), `"request":`+string(sb.jobs[len(sb.jobs)-1].Request)) { + t.Errorf("kept %s, %v: want the request of the last job under %s/anchor", b, err, keyOfHost) + } + store.err = errors.New("the state is full") + raw, _ = ec.Compose(context.Background(), host) + if v := rules.JudgeEngines(raw); v.State != meshdeliverycheckerclient.CompositionFail || !strings.Contains(v.Words, "could not be kept") || !strings.Contains(v.Words, "the state is full") { + t.Errorf("a request that cannot be kept gave %+v", v) + } + ec.Requests = nil // Issue 1921: the composer is built at the group's heads, also a member that publishes no package yet. if pkgs := job.Fetch.Packages; len(pkgs) != 1 || pkgs[trunk.ForgeModulePrefix+"mesh-delivery"] != head1 { t.Errorf("the composer is fetched with the packages %v, want the group's mesh-delivery member at its head %s alone", pkgs, head1) diff --git a/modules/mesh-delivery-checker/internal/check/requests.go b/modules/mesh-delivery-checker/internal/check/requests.go new file mode 100644 index 00000000..66f57613 --- /dev/null +++ b/modules/mesh-delivery-checker/internal/check/requests.go @@ -0,0 +1,159 @@ +package check + +import ( + "bytes" + "compress/gzip" + "context" + "crypto/sha256" + "encoding/base64" + "encoding/hex" + "encoding/json" + "fmt" + "io" + "sort" + "strings" + "time" +) + +// The composition requests a check sends (issue 1921, notes 66-67): the request the checker sends the composer for one +// node, the checking program's whole input of a composition, is kept in the checker's own state RequestsStateName, so a +// session reproduces a check's composition exactly. The checker is a foundation part (novox/hq ADR 0619 §1-3), so its +// kept data is a section of its manifest's old form, here `state`, and not an object store asked in `requires`. +// +// **What bounds it.** A request is the node-engine's whole composition input of one node, about 430 KB as JSON. A +// state's value holds at most 256 KiB (kvlimit.ModuleStateValueBytes), so it is kept gzip-compressed (JSON shrinks about +// tenfold) in a document of its own; a request whose compressed bytes still do not fit is refused by the put, which +// fails the node's composition naming the size (never dropped). The manifest's ttl-seconds is its lifetime (3 days): +// the bus drops a value past it, so nothing waits on a timer here. A check kept again overwrites the key. +const ( + // RequestsStateName is the checker's state of kept requests and RequestsBucket its bucket. + RequestsStateName = "compositions" + RequestsBucket = CheckerModule + "_" + RequestsStateName + // RequestsReadTool is the tool that reads a kept request. + RequestsReadTool = "checker_composition_requests" +) + +// KeptRequest is a request as the state holds it: whose it is, and the request's JSON, compressed. +type KeptRequest struct { + V int `json:"v"` + Check string `json:"check"` + Node string `json:"node"` + KeptAt time.Time `json:"kept_at"` + // Size and Digest are the request's, uncompressed: its bytes and their sha256. + Size int `json:"size"` + Digest string `json:"digest"` + // Gzip is the request's bytes, gzip-compressed and base64-encoded. + Gzip string `json:"gzip"` +} + +// RequestKey is the key a check's request to a node is kept under: the check's key (group.heads) and the node. +func RequestKey(key, node string) string { return key + "/" + node } + +// RequestPutter writes a value of the state, replacing one of that key. +type RequestPutter interface { + Put(ctx context.Context, key string, value []byte) (uint64, error) +} + +// RequestKeeper keeps the requests of a check. +type RequestKeeper struct { + Store RequestPutter + Now func() time.Time +} + +// Keep stores body under the check's key and the node. An error says why it could not, for the verdict's words. +func (k RequestKeeper) Keep(ctx context.Context, key, node string, body []byte) error { + var packed bytes.Buffer + zw := gzip.NewWriter(&packed) + if _, err := zw.Write(body); err != nil { + return fmt.Errorf("the composition request of check %s to %s could not be compressed: %w", key, node, err) + } + if err := zw.Close(); err != nil { + return fmt.Errorf("the composition request of check %s to %s could not be compressed: %w", key, node, err) + } + now := time.Now + if k.Now != nil { + now = k.Now + } + sum := sha256.Sum256(body) + doc, err := json.Marshal(KeptRequest{V: 1, Check: key, Node: node, KeptAt: now().UTC(), Size: len(body), + Digest: hex.EncodeToString(sum[:]), Gzip: base64.StdEncoding.EncodeToString(packed.Bytes())}) + if err != nil { + return err + } + if _, err := k.Store.Put(ctx, RequestKey(key, node), doc); err != nil { + return fmt.Errorf("the composition request of check %s to %s (%d bytes, %d compressed) could not be kept in the state %s: %w", + key, node, len(body), packed.Len(), RequestsStateName, err) + } + return nil +} + +// ReadRequests answers the kept requests: with a check and a node, that request whole; with a check, the nodes it kept +// one for; with neither, every kept request. A node without a check is refused. get is the state's value of a key. +func ReadRequests(s StateRead, key, node string) (any, error) { + if s.Keys == nil || s.Get == nil { + return nil, fmt.Errorf("the state %s cannot be read here", RequestsStateName) + } + if node != "" && key == "" { + return nil, fmt.Errorf("a node is asked for with the check's key (group.heads) of the request") + } + if node != "" { + raw, err := s.Get(RequestKey(key, node)) + if err != nil { + return nil, err + } + if len(raw) == 0 { + return nil, fmt.Errorf("no composition request of check %s to %s is kept (none was sent, or it is past its lifetime)", key, node) + } + kept, body, err := unpackRequest(raw) + if err != nil { + return nil, fmt.Errorf("the kept request %s: %w", RequestKey(key, node), err) + } + return map[string]any{"check": kept.Check, "node": kept.Node, "size": kept.Size, "digest": kept.Digest, "kept_at": kept.KeptAt, "request": json.RawMessage(body)}, nil + } + keys, err := s.Keys() + if err != nil { + return nil, err + } + sort.Strings(keys) + listed := []map[string]any{} + for _, k := range keys { + if key != "" && !strings.HasPrefix(k, key+"/") { + continue + } + raw, err := s.Get(k) + if err != nil { + return nil, err + } + var kept KeptRequest + if err := json.Unmarshal(raw, &kept); err != nil { + listed = append(listed, map[string]any{"key": k, "unreadable": err.Error()}) + continue + } + listed = append(listed, map[string]any{"check": kept.Check, "node": kept.Node, "size": kept.Size, "digest": kept.Digest, "kept_at": kept.KeptAt}) + } + return map[string]any{"kept": listed}, nil +} + +// unpackRequest reads a kept document and its request's bytes, which must match the digest they were kept with. +func unpackRequest(raw []byte) (KeptRequest, []byte, error) { + var kept KeptRequest + if err := json.Unmarshal(raw, &kept); err != nil { + return kept, nil, fmt.Errorf("is no kept request: %w", err) + } + packed, err := base64.StdEncoding.DecodeString(kept.Gzip) + if err != nil { + return kept, nil, fmt.Errorf("holds no base64: %w", err) + } + zr, err := gzip.NewReader(bytes.NewReader(packed)) + if err != nil { + return kept, nil, fmt.Errorf("holds no gzip: %w", err) + } + body, err := io.ReadAll(zr) + if err != nil { + return kept, nil, fmt.Errorf("holds no complete gzip: %w", err) + } + if sum := sha256.Sum256(body); hex.EncodeToString(sum[:]) != kept.Digest { + return kept, nil, fmt.Errorf("does not match the digest it was kept with") + } + return kept, body, nil +} diff --git a/modules/mesh-delivery-checker/manifest_test.go b/modules/mesh-delivery-checker/manifest_test.go index 8805134d..36416382 100644 --- a/modules/mesh-delivery-checker/manifest_test.go +++ b/modules/mesh-delivery-checker/manifest_test.go @@ -108,7 +108,7 @@ func TestManifestDeclarations(t *testing.T) { // 1678/AC1 t.Errorf("module = %q", m.Module) } // The histories of the checker's four seat states (checked against internal/check names below); the names follow the seat mesh-delivery-checker. - if want := []state{{Name: "asks", History: 1}, {Name: "hearing", History: 1}, {Name: "inputs", TTL: 86400}, {Name: "kept", History: 1}, {Name: "owed", History: 5}, {Name: "previews", History: 5}, {Name: "tries", History: 1}, {Name: "verdicts", History: 5}, {Name: "first-views", History: 1}, {Name: "findings", History: 1}}; !reflect.DeepEqual(m.State, want) { + if want := []state{{Name: "asks", History: 1}, {Name: "hearing", History: 1}, {Name: "inputs", TTL: 86400}, {Name: "compositions", TTL: 259200}, {Name: "kept", History: 1}, {Name: "owed", History: 5}, {Name: "previews", History: 5}, {Name: "tries", History: 1}, {Name: "verdicts", History: 5}, {Name: "first-views", History: 1}, {Name: "findings", History: 1}}; !reflect.DeepEqual(m.State, want) { t.Errorf("state = %+v, want %+v", m.State, want) } if want := []string{"mesh-build-go.steps", "mesh-build-go.toolchains", "mesh-build-node.steps", "mesh-build-node.toolchains", @@ -186,7 +186,7 @@ func TestManifestDeclarations(t *testing.T) { // 1678/AC1 // The read tools (issue 1686) are served by the tools bundle, not by the checker's process. // The try's tools (ADR 0501 §5; the end by hand, issue 734) beside them: mesh-controller's try calls them. A lost check // is asked again by hand (checker_ask_again). - if want := []string{"checker_ask_again", "checker_first_view_again", "checker_first_views", "checker_inputs", "checker_kept", "checker_previews", "checker_pull", "checker_status", "checker_tries", "checker_try", "checker_try_end", "checker_try_read", "checker_verdicts"}; !reflect.DeepEqual(sorted(m.Tools), want) { + if want := []string{"checker_ask_again", "checker_composition_requests", "checker_first_view_again", "checker_first_views", "checker_inputs", "checker_kept", "checker_previews", "checker_pull", "checker_status", "checker_tries", "checker_try", "checker_try_end", "checker_try_read", "checker_verdicts"}; !reflect.DeepEqual(sorted(m.Tools), want) { t.Errorf("tools = %v, want %v", m.Tools, want) } // The try is the checker's own seat's verbs (ADR 0547 §10, issue 774), served beside the tools by the same code; so is diff --git a/modules/mesh-delivery-checker/module.json b/modules/mesh-delivery-checker/module.json index 8f99ab6c..fcce74fa 100644 --- a/modules/mesh-delivery-checker/module.json +++ b/modules/mesh-delivery-checker/module.json @@ -37,6 +37,7 @@ { "name": "asks", "history": 1 }, { "name": "hearing", "history": 1 }, { "name": "inputs", "ttl-seconds": 86400 }, + { "name": "compositions", "ttl-seconds": 259200 }, { "name": "kept", "history": 1 }, { "name": "owed", "history": 5 }, { "name": "previews", "history": 5 }, @@ -62,6 +63,7 @@ ], "tools": [ "checker_ask_again", + "checker_composition_requests", "checker_inputs", "checker_kept", "checker_previews",