From a3306361ee7b5391628b391b34ed7160bfbe0dd5 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 12 Nov 2026 10:48:08 +0100 Subject: [PATCH] checker: the state compositions is bounded at 256 MiB and a day, and a tool drops a check's kept requests Builds: the lead's handover of 2026-11-12 (principle 7, principle 4; no design line) Claude-Session: https://claude.ai/code/session_01NcbgfGKaxw984XyAPfLm4h --- .../cmd/mesh-delivery-checker-tools/main.go | 26 +++++- .../internal/check/engines_test.go | 31 ++++++- .../internal/check/requests.go | 87 +++++++++++++++++-- .../mesh-delivery-checker/manifest_test.go | 5 +- modules/mesh-delivery-checker/module.json | 5 +- 5 files changed, 141 insertions(+), 13 deletions(-) diff --git a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker-tools/main.go b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker-tools/main.go index d32b387c..eebac0e6 100644 --- a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker-tools/main.go +++ b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker-tools/main.go @@ -378,8 +378,8 @@ func RequestTools(r check.StateRead) []stdio.Tool { Description: "The composition request a check sent to a node: the composer's whole input for that node (the node-engine's composition input, the case's " + "manifests and verdicts, the seat list), kept exactly as sent so a session reproduces the check's composition (the composer run by hand over the same " + "request gives the check's answer). Kept in the checker's state compositions, per check key (group.heads, the 12-digit heads hash) and node, " + - "0.7 to 0.8 MB each (kept compressed, in pieces that each fit the state's 256 KiB value), for 3 days (the state's ttl-seconds). With a check and a node, that request whole; with a check, " + - "the nodes it kept one for; with neither, every kept request, each with its size, digest and when it was kept. Read-only.", + "0.7 to 0.8 MB each (kept compressed, in pieces that each fit the state's 256 KiB value), for a day (the state's ttl-seconds), the state holding at most 256 MiB (max-mib). With a check and a node, that request whole; with a check, " + + "the nodes it kept one for; with neither, every kept request, each with its size, digest and when it was kept. Read-only; " + check.RequestsDropTool + " drops a check's.", Input: map[string]any{"type": "object", "properties": map[string]any{ "check": map[string]any{"type": "string", "description": "the check's key, ., as the verdict and composition name it"}, "node": map[string]any{"type": "string", "description": "the node the request was sent for, with a check"}}}, @@ -390,6 +390,26 @@ func RequestTools(r check.StateRead) []stdio.Tool { }}} } +// RequestDropTools is the tool that drops a check's kept composition requests by hand, which frees the state when it is +// full (tools-are-how-a-module-is-spoken-to): it records the hand act through the seat mesh-hand-acts first. +func RequestDropTools(s check.StateDrop, acts views.HandActs) []stdio.Tool { + return []stdio.Tool{{Name: check.RequestsDropTool, + Description: "Drop the composition requests the checker kept for one check (" + check.RequestsReadTool + " lists them): every node's, and the pieces " + + "no other kept request names. The hand act is recorded through the seat mesh-hand-acts first, and a drop whose act could not be recorded is not made. " + + "Answers the nodes and pieces dropped. Use it when a check's words say its request could not be kept in the state compositions because it is full; " + + "a request is dropped by the state itself a day after it was kept.", + Input: map[string]any{"type": "object", "required": []string{"check", "why"}, "properties": map[string]any{ + "check": map[string]any{"type": "string", "description": "the check's key, ., whose kept requests are dropped"}, + "why": map[string]any{"type": "string", "description": "why they are dropped, in words; recorded as the hand act"}}}, + Run: func(args map[string]any) (any, error) { + key, _ := args["check"].(string) + why, _ := args["why"].(string) + return check.DropRequests(s, key, why, func(what string) error { + return acts.RecordHandAct(views.HandAct{What: what, Why: why, Verb: check.RequestsDropTool}) + }) + }}} +} + // RecoveryTools are the tools that steer a check by hand (tools-are-how-a-module-is-spoken-to): ask a check again whose // wait judged nothing (the lost check of 2026-10-28). They write the checker's verdicts through w. func RecoveryTools(r check.Reader, w check.VerdictEntry) []stdio.Tool { @@ -469,6 +489,8 @@ func main() { tools = append(tools, views.StatusTools(check.CheckerModule, "checker_first_views", "checker_first_view_again", stdio.State(views.StatusState), meshhandactsclient.Recorder{Ask: stdio.Ask}, time.Now)...) tools = append(tools, SeatInputTools(r)...) tools = append(tools, RequestTools(read(stdio.State(check.RequestsStateName)))...) + requests := stdio.State(check.RequestsStateName) + tools = append(tools, RequestDropTools(check.StateDrop{StateRead: read(requests), Delete: requests.Delete}, meshhandactsclient.Recorder{Ask: stdio.Ask})...) if err := stdio.Serve("", logged(tools)); err != nil { fmt.Fprintf(os.Stderr, "[mesh-delivery-checker-tools] %v\n", err) os.Exit(1) diff --git a/modules/mesh-delivery-checker/internal/check/engines_test.go b/modules/mesh-delivery-checker/internal/check/engines_test.go index 1c76571e..36a96514 100644 --- a/modules/mesh-delivery-checker/internal/check/engines_test.go +++ b/modules/mesh-delivery-checker/internal/check/engines_test.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "errors" + "fmt" "mesh-delivery-checker/rules" "strings" "sync" @@ -235,8 +236,34 @@ func TestTheCompositionIsMadeFromTheEnginesAnswers(t *testing.T) { } store.err = errors.New("the state is full") raw, _ = ec.Compose(context.Background(), host) - if v := rules.JudgeEngines(raw); v.State != meshdeliverycheckerclient.CompositionFail || !strings.Contains(v.Words, "could not be kept") || !strings.Contains(v.Words, "the state is full") { - t.Errorf("a request that cannot be kept gave %+v", v) + if v := rules.JudgeEngines(raw); v.State != meshdeliverycheckerclient.CompositionFail || !strings.Contains(v.Words, "could not be kept") || !strings.Contains(v.Words, "the state is full") || !strings.Contains(v.Words, RequestsDropTool) { + t.Errorf("a request that cannot be kept gave %+v, which must name %s", v, RequestsDropTool) + } + // The drop tool: records the hand act first (a drop whose act is not recorded is not made), then drops that check's + // requests and the pieces no other kept request names; another check's stay. + store.err = nil + other := RequestKeeper{Store: store} + if err := other.Keep(context.Background(), "g.other", "anchor", []byte("another check's request")); err != nil { + t.Fatal(err) + } + drop := StateDrop{StateRead: store.read(), Delete: func(k string) error { store.mu.Lock(); delete(store.vals, k); store.mu.Unlock(); return nil }} + before := len(store.vals) + if _, err := DropRequests(drop, keyOfHost, "the state is full", func(string) error { return errors.New("no holder") }); err == nil || len(store.vals) != before { + t.Errorf("a drop whose hand act was not recorded was made: %v, %d of %d kept", err, len(store.vals), before) + } + var acts []string + answer, err := DropRequests(drop, keyOfHost, "the state is full", func(what string) error { acts = append(acts, what); return nil }) + if b, _ := json.Marshal(answer); err != nil || len(acts) != 1 || !strings.Contains(string(b), `"dropped_requests":["anchor"]`) { + t.Errorf("the drop answered %s, %v, acts %v", b, err, acts) + } + if _, still := store.vals[RequestKey(keyOfHost, "anchor")]; still { + t.Errorf("the dropped check's request is kept still") + } + if got, err := ReadRequests(store.read(), "g.other", "anchor"); err != nil || !strings.Contains(fmt.Sprint(got), "another check's request") { + t.Errorf("another check's request after the drop: %v, %v", got, err) + } + if _, err := DropRequests(drop, keyOfHost, "again", func(string) error { return nil }); err == nil { + t.Errorf("a check with no kept request was dropped") } ec.Requests = nil // Issue 1921: the composer is built at the group's heads, also a member that publishes no package yet. diff --git a/modules/mesh-delivery-checker/internal/check/requests.go b/modules/mesh-delivery-checker/internal/check/requests.go index 46d4531d..64838a3c 100644 --- a/modules/mesh-delivery-checker/internal/check/requests.go +++ b/modules/mesh-delivery-checker/internal/check/requests.go @@ -26,8 +26,9 @@ import ( // of the check and node holds a small document naming the request's digest, size and number of pieces, and each piece is // a value of its own under RequestChunkKey, of at most RequestChunkBytes. The kept data is the checker's, a foundation // part's, so it stays in the old section `state` (ADR 0619 §2) and is read through the tool runner's state read, which -// serves no object store of the old sections. The manifest's ttl-seconds is the lifetime (3 days) of the pointer and of -// every piece: the bus drops a value past it, so nothing waits on a timer here. A check kept again overwrites the key; +// serves no object store of the old sections. The manifest's ttl-seconds is the lifetime (1 day) of the pointer and of +// every piece and its max-mib the most the state holds (256): the bus drops a value past the one and refuses a put past the +// other, so nothing waits on a timer here; a state that is full says so in the check's words, which name RequestsDropTool. A check kept again overwrites the key; // a request whose pieces cannot all be put is refused naming the size (never dropped, never half kept: the pointer is // put after its pieces). const ( @@ -36,6 +37,8 @@ const ( RequestsBucket = CheckerModule + "_" + RequestsStateName // RequestsReadTool is the tool that reads a kept request. RequestsReadTool = "checker_composition_requests" + // RequestsDropTool is the tool that drops a check's kept requests by hand (a state that is full is freed with it). + RequestsDropTool = "checker_composition_requests_drop" // RequestChunkBytes is the most compressed bytes one piece holds: base64 of it, in its document, stays well under // the state's 256 KiB value. RequestChunkBytes = 128 << 10 @@ -110,8 +113,8 @@ func (k RequestKeeper) Keep(ctx context.Context, key, node string, body []byte) digest := hex.EncodeToString(sum[:]) pieces := (packed.Len() + RequestChunkBytes - 1) / RequestChunkBytes fail := func(err error) error { - return fmt.Errorf("the composition request of check %s to %s (%d bytes, %d compressed in %d pieces) could not be kept in the state %s: %w", - key, node, len(body), packed.Len(), pieces, RequestsStateName, err) + return fmt.Errorf("the composition request of check %s to %s (%d bytes, %d compressed in %d pieces) could not be kept in the state %s: %w (when the state is full, free it by dropping the kept requests of a check no longer read, with the tool %s)", + key, node, len(body), packed.Len(), pieces, RequestsStateName, err, RequestsDropTool) } for i := 0; i < pieces; i++ { end := min((i+1)*RequestChunkBytes, packed.Len()) @@ -190,7 +193,7 @@ func unpackRequest(s StateRead, raw []byte) (KeptRequest, []byte, error) { return kept, nil, fmt.Errorf("is no kept request: %w", err) } if kept.V != 2 { - return kept, nil, fmt.Errorf("is a kept request of version %d, which this reader does not read (it reads version 2; an older one is past its lifetime within 3 days)", kept.V) + return kept, nil, fmt.Errorf("is a kept request of version %d, which this reader does not read (it reads version 2; an older one is past its lifetime within a day)", kept.V) } var packed []byte for i := 0; i < kept.Pieces; i++ { @@ -221,3 +224,77 @@ func unpackRequest(s StateRead, raw []byte) (KeptRequest, []byte, error) { } return kept, body, nil } + +// StateDrop is the state of kept requests as the drop tool needs it: its keys, a value, and a key removed. +type StateDrop struct { + StateRead + Delete func(key string) error +} + +// DropRequests drops the requests kept for a check, after record (the hand act, recorded first: a drop that is not +// recorded is not made), and answers what it dropped: the nodes, and the pieces. A piece another kept request still names +// (the same request kept for another check has the same digest) stays. A check naming no kept request is refused. +func DropRequests(s StateDrop, key, why string, record func(what string) error) (any, error) { + if strings.TrimSpace(key) == "" || strings.TrimSpace(why) == "" { + return nil, fmt.Errorf("drop takes check (its key, group.heads) and why") + } + if s.Keys == nil || s.Get == nil || s.Delete == nil { + return nil, fmt.Errorf("the state %s cannot be changed here", RequestsStateName) + } + keys, err := s.Keys() + if err != nil { + return nil, err + } + sort.Strings(keys) + var drop []string + kept := map[string]bool{} // digests the requests that stay name + dropping := map[string]bool{} + for _, k := range keys { + if isPieceKey(k) { + continue + } + raw, err := s.Get(k) + if err != nil { + return nil, err + } + var r KeptRequest + if len(raw) == 0 || json.Unmarshal(raw, &r) != nil { + continue + } + if strings.HasPrefix(k, key+"/") { + drop = append(drop, k) + dropping[r.Digest] = true + continue + } + kept[r.Digest] = true + } + if len(drop) == 0 { + return nil, fmt.Errorf("no composition request of check %s is kept (none was sent, or it is past its lifetime): nothing was dropped", key) + } + if err := record(fmt.Sprintf("dropped the %d kept composition requests of check %s (%s)", len(drop), key, RequestsStateName)); err != nil { + return nil, fmt.Errorf("the hand act was not recorded, so nothing was dropped: %w", err) + } + var pieces int + for _, k := range drop { + if err := s.Delete(k); err != nil { + return nil, fmt.Errorf("dropping %s: %w", k, err) + } + } + for _, k := range keys { + if !isPieceKey(k) { + continue + } + digest := strings.Split(strings.TrimPrefix(k, "pieces/"), "/")[0] + if dropping[digest] && !kept[digest] { + if err := s.Delete(k); err != nil { + return nil, fmt.Errorf("dropping %s: %w", k, err) + } + pieces++ + } + } + nodes := make([]string, len(drop)) + for i, k := range drop { + nodes[i] = strings.TrimPrefix(k, key+"/") + } + return map[string]any{"check": key, "dropped_requests": nodes, "dropped_pieces": pieces, "why": why}, nil +} diff --git a/modules/mesh-delivery-checker/manifest_test.go b/modules/mesh-delivery-checker/manifest_test.go index ac47d009..5451f26f 100644 --- a/modules/mesh-delivery-checker/manifest_test.go +++ b/modules/mesh-delivery-checker/manifest_test.go @@ -23,6 +23,7 @@ type state struct { Name string `json:"name"` History int `json:"history,omitempty"` TTL int `json:"ttl-seconds,omitempty"` + MaxMiB int `json:"max-mib,omitempty"` } type artifact struct { @@ -108,7 +109,7 @@ func TestManifestDeclarations(t *testing.T) { // 1678/AC1 t.Errorf("module = %q", m.Module) } // The histories of the checker's four seat states (checked against internal/check names below); the names follow the seat mesh-delivery-checker. - if want := []state{{Name: "asks", History: 1}, {Name: "hearing", History: 1}, {Name: "inputs", TTL: 86400}, {Name: "compositions", TTL: 259200}, {Name: "kept", History: 1}, {Name: "owed", History: 5}, {Name: "previews", History: 5}, {Name: "tries", History: 1}, {Name: "verdicts", History: 5}, {Name: "first-views", History: 1}, {Name: "findings", History: 1}}; !reflect.DeepEqual(m.State, want) { + if want := []state{{Name: "asks", History: 1}, {Name: "hearing", History: 1}, {Name: "inputs", TTL: 86400}, {Name: "compositions", TTL: 86400, MaxMiB: 256}, {Name: "kept", History: 1}, {Name: "owed", History: 5}, {Name: "previews", History: 5}, {Name: "tries", History: 1}, {Name: "verdicts", History: 5}, {Name: "first-views", History: 1}, {Name: "findings", History: 1}}; !reflect.DeepEqual(m.State, want) { t.Errorf("state = %+v, want %+v", m.State, want) } if want := []string{"mesh-build-go.steps", "mesh-build-go.toolchains", "mesh-build-node.steps", "mesh-build-node.toolchains", @@ -186,7 +187,7 @@ func TestManifestDeclarations(t *testing.T) { // 1678/AC1 // The read tools (issue 1686) are served by the tools bundle, not by the checker's process. // The try's tools (ADR 0501 §5; the end by hand, issue 734) beside them: mesh-controller's try calls them. A lost check // is asked again by hand (checker_ask_again). - if want := []string{"checker_ask_again", "checker_composition_requests", "checker_first_view_again", "checker_first_views", "checker_inputs", "checker_kept", "checker_previews", "checker_pull", "checker_status", "checker_tries", "checker_try", "checker_try_end", "checker_try_read", "checker_verdicts"}; !reflect.DeepEqual(sorted(m.Tools), want) { + if want := []string{"checker_ask_again", "checker_composition_requests", "checker_composition_requests_drop", "checker_first_view_again", "checker_first_views", "checker_inputs", "checker_kept", "checker_previews", "checker_pull", "checker_status", "checker_tries", "checker_try", "checker_try_end", "checker_try_read", "checker_verdicts"}; !reflect.DeepEqual(sorted(m.Tools), want) { t.Errorf("tools = %v, want %v", m.Tools, want) } // The try is the checker's own seat's verbs (ADR 0547 §10, issue 774), served beside the tools by the same code; so is diff --git a/modules/mesh-delivery-checker/module.json b/modules/mesh-delivery-checker/module.json index 49efcfe4..07d368dd 100644 --- a/modules/mesh-delivery-checker/module.json +++ b/modules/mesh-delivery-checker/module.json @@ -1,6 +1,6 @@ { "module": "mesh-delivery-checker", - "version": "1", + "version": "2", "slug": "checker", "claims": [ { @@ -37,7 +37,7 @@ { "name": "asks", "history": 1 }, { "name": "hearing", "history": 1 }, { "name": "inputs", "ttl-seconds": 86400 }, - { "name": "compositions", "ttl-seconds": 259200 }, + { "name": "compositions", "ttl-seconds": 86400, "max-mib": 256 }, { "name": "kept", "history": 1 }, { "name": "owed", "history": 5 }, { "name": "previews", "history": 5 }, @@ -64,6 +64,7 @@ "tools": [ "checker_ask_again", "checker_composition_requests", + "checker_composition_requests_drop", "checker_inputs", "checker_kept", "checker_previews",