From 22bbb9c2c09f9416df1af32ded939892f806db0f Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 12 Nov 2026 16:32:48 +0100 Subject: [PATCH] Keep a check's answer within the steps state: leave out the taken versions A catalog check's answer of 264712 bytes was refused by the steps state (256 KiB per value) and its end written failed without it, so the checker lost the biggest member's answer and could not preview it. When a passed or failed end is refused as too large, finish writes it once more without the fetch's taken versions (fetch.packages, which no reader judges), the step's reason saying how many were left out, why, and that the module's journal names each; still too large, it ends failed as before. Quick fix for issue 1921 (note 91); the thorough fix is design 47 G A-290 (an answer over the bound kept as the object output.). Claude-Session: https://claude.ai/code/session_01NcbgfGKaxw984XyAPfLm4h --- .../buildstep/digestshape_test.go | 66 +++++++++++++++++++ modules/mesh-build-go/buildstep/step.go | 48 ++++++++++++++ 2 files changed, 114 insertions(+) diff --git a/modules/mesh-build-go/buildstep/digestshape_test.go b/modules/mesh-build-go/buildstep/digestshape_test.go index af9511d0d..9b2aedc12 100644 --- a/modules/mesh-build-go/buildstep/digestshape_test.go +++ b/modules/mesh-build-go/buildstep/digestshape_test.go @@ -12,6 +12,7 @@ import ( "time" sdkstep "git.novox.be/novox/mesh-delivery/sdk/buildstep" + "git.novox.be/novox/mesh-sdk/go/kvlimit" ) // credentialField is the node runtime's rule for a state value (mesh-tools node-tools/internal/bus/state.go, novox/hq @@ -181,6 +182,71 @@ func TestARefusedEndIsSaidAndEndsTheStepFailed(t *testing.T) { } } +// 1921 note 91: given a check's answer over what the steps state takes per value, the bulk of it the fetch's taken +// versions, when finish writes it, then the end is written passed without the taken versions, its levels and the rest of +// its fetch kept, the reason saying how many were left out and why; an over-large answer with no taken versions, and one +// still too large without them, end failed as before. +func TestAnAnswerOverTheStateBoundLeavesOutItsTakenVersions(t *testing.T) { + fastEnds(t) + answer := func(taken int) map[string]any { + var packages []map[string]any + for i := range taken { + packages = append(packages, map[string]any{"dir": fmt.Sprintf("modules/m%d", i), "package": "git.novox.be/novox/mesh-sdk/go", "said": "the ask names it"}) + } + return map[string]any{ + "fetch": map[string]any{"workspace": "/w", "changed": []string{"modules/m0/module.json"}, "packages": packages}, + "levels": []any{map[string]any{"level": "unit", "tests": []any{map[string]any{"passed": true}}}}, + } + } + for _, tc := range []struct { + name string + taken int + limit int + wantState string + }{ + {"its taken versions left out", 40, 1500, sdkstep.StepPassed}, + {"no taken versions to leave out", 0, 200, sdkstep.StepFailed}, + {"still too large without them", 40, 200, sdkstep.StepFailed}, + } { + t.Run(tc.name, func(t *testing.T) { + var stored []sdkstep.Step + put := func(key string, value any) (uint64, error) { + raw := value.(json.RawMessage) + if err := kvlimit.Check("the state steps", key, len(raw), int64(tc.limit)); err != nil && strings.Contains(string(raw), `"result"`) { + return 0, err + } + st, _ := sdkstep.DecodeStep(raw) + stored = append(stored, st) + return 1, nil + } + if err := finish(put, "shanks.a4", sdkstep.Step{ID: "a4", Tool: "check ask", Node: "shanks", State: sdkstep.StepRunning, Started: time.Now()}, answer(tc.taken), nil, t.Logf); err != nil { + t.Fatal(err) + } + if len(stored) != 1 || stored[0].State != tc.wantState { + t.Fatalf("stored %+v, want one end %s", stored, tc.wantState) + } + s := stored[0] + if tc.wantState == sdkstep.StepFailed { + if len(s.Result) != 0 || !strings.Contains(s.Reason, "could not be written") { + t.Errorf("the end %+v, want the small failed end", s) + } + return + } + var a sdkstep.CheckAnswer + if err := json.Unmarshal(s.Result, &a); err != nil { + t.Fatal(err) + } + if len(a.Fetch.Packages) != 0 || a.Fetch.Workspace != "/w" || len(a.Fetch.Changed) != 1 || len(a.Levels) != 1 || len(a.Levels[0].Tests) != 1 { + t.Errorf("the stored answer %s, want its fetch and levels without the taken versions", s.Result) + } + if !strings.Contains(s.Reason, "the fetch's 40 taken versions are left out") || !strings.Contains(s.Reason, "journal names each") || + !strings.Contains(s.Reason, "bytes the bucket takes") { + t.Errorf("the reason %q, want how many were left out, why and where each is said", s.Reason) + } + }) + } +} + // 1930/AC2: given an ask whose end the state refuses, when it runs, then the ask is acknowledged only once its small // failed end is written; when no end can be written at all, it is given back, never acknowledged as done. func TestAnAskWhoseEndCannotBeWrittenIsNotAcked(t *testing.T) { diff --git a/modules/mesh-build-go/buildstep/step.go b/modules/mesh-build-go/buildstep/step.go index 47403fdfb..2bdeadc9f 100644 --- a/modules/mesh-build-go/buildstep/step.go +++ b/modules/mesh-build-go/buildstep/step.go @@ -10,10 +10,12 @@ import ( "errors" "fmt" "log" + "strings" "time" buildrun "git.novox.be/novox/mesh-delivery/sdk/buildrun" sdkstep "git.novox.be/novox/mesh-delivery/sdk/buildstep" + "git.novox.be/novox/mesh-sdk/go/kvlimit" ) // Put writes one key of a state: the steps state.Put's shape (go/state.go). @@ -102,6 +104,24 @@ func finish(put Put, key string, step sdkstep.Step, result any, err error, logf if werr == nil { return nil } + if errors.Is(werr, kvlimit.ErrTooLarge) { + // A check's answer over what the state takes per value is written once more without the fetch's taken versions, + // which no reader judges, the reason saying so (issue 1921, note 91; the thorough fix is design 47 G A-290). + if smaller, left, ok := withoutTakenVersions(step.Result); ok { + less := step + less.Result = smaller + said := fmt.Sprintf("the fetch's %d taken versions are left out of the stored answer: the answer was over what the steps state takes per value (%v); the module's journal names each", left, werr) + less.Reason = strings.TrimPrefix(step.Reason+"; "+said, "; ") + if lerr := writeEnd(put, key, less); lerr == nil { + if logf != nil { + logf("the end of the step %s (%s) was written without its fetch's %d taken versions: %v", key, step.State, left, werr) + } + return nil + } else { + werr = lerr + } + } + } if logf != nil { logf("the end of the step %s (%s) could not be written in %d tries: %v", key, step.State, EndAttempts, werr) } @@ -149,6 +169,34 @@ func protect(ctx context.Context, run func(ctx context.Context) (any, error)) (r return run(ctx) } +// withoutTakenVersions is a check's answer raw without its fetch's taken versions (`fetch.packages`) and how many it +// held; ok is false when the answer holds none, so nothing would be smaller. +func withoutTakenVersions(raw json.RawMessage) (json.RawMessage, int, bool) { + var answer map[string]json.RawMessage + if len(raw) == 0 || json.Unmarshal(raw, &answer) != nil || answer["fetch"] == nil { + return nil, 0, false + } + var fetch map[string]json.RawMessage + if json.Unmarshal(answer["fetch"], &fetch) != nil { + return nil, 0, false + } + var taken []json.RawMessage + if json.Unmarshal(fetch["packages"], &taken) != nil || len(taken) == 0 { + return nil, 0, false + } + delete(fetch, "packages") + f, err := json.Marshal(fetch) + if err != nil { + return nil, 0, false + } + answer["fetch"] = f + out, err := json.Marshal(answer) + if err != nil { + return nil, 0, false + } + return out, len(taken), true +} + func write(put Put, key string, step sdkstep.Step) error { raw, err := sdkstep.EncodeStep(step) if err != nil {