diff --git a/modules/mesh-build-go/buildstep/digestshape_test.go b/modules/mesh-build-go/buildstep/digestshape_test.go index af9511d0d..9b2aedc12 100644 --- a/modules/mesh-build-go/buildstep/digestshape_test.go +++ b/modules/mesh-build-go/buildstep/digestshape_test.go @@ -12,6 +12,7 @@ import ( "time" sdkstep "git.novox.be/novox/mesh-delivery/sdk/buildstep" + "git.novox.be/novox/mesh-sdk/go/kvlimit" ) // credentialField is the node runtime's rule for a state value (mesh-tools node-tools/internal/bus/state.go, novox/hq @@ -181,6 +182,71 @@ func TestARefusedEndIsSaidAndEndsTheStepFailed(t *testing.T) { } } +// 1921 note 91: given a check's answer over what the steps state takes per value, the bulk of it the fetch's taken +// versions, when finish writes it, then the end is written passed without the taken versions, its levels and the rest of +// its fetch kept, the reason saying how many were left out and why; an over-large answer with no taken versions, and one +// still too large without them, end failed as before. +func TestAnAnswerOverTheStateBoundLeavesOutItsTakenVersions(t *testing.T) { + fastEnds(t) + answer := func(taken int) map[string]any { + var packages []map[string]any + for i := range taken { + packages = append(packages, map[string]any{"dir": fmt.Sprintf("modules/m%d", i), "package": "git.novox.be/novox/mesh-sdk/go", "said": "the ask names it"}) + } + return map[string]any{ + "fetch": map[string]any{"workspace": "/w", "changed": []string{"modules/m0/module.json"}, "packages": packages}, + "levels": []any{map[string]any{"level": "unit", "tests": []any{map[string]any{"passed": true}}}}, + } + } + for _, tc := range []struct { + name string + taken int + limit int + wantState string + }{ + {"its taken versions left out", 40, 1500, sdkstep.StepPassed}, + {"no taken versions to leave out", 0, 200, sdkstep.StepFailed}, + {"still too large without them", 40, 200, sdkstep.StepFailed}, + } { + t.Run(tc.name, func(t *testing.T) { + var stored []sdkstep.Step + put := func(key string, value any) (uint64, error) { + raw := value.(json.RawMessage) + if err := kvlimit.Check("the state steps", key, len(raw), int64(tc.limit)); err != nil && strings.Contains(string(raw), `"result"`) { + return 0, err + } + st, _ := sdkstep.DecodeStep(raw) + stored = append(stored, st) + return 1, nil + } + if err := finish(put, "shanks.a4", sdkstep.Step{ID: "a4", Tool: "check ask", Node: "shanks", State: sdkstep.StepRunning, Started: time.Now()}, answer(tc.taken), nil, t.Logf); err != nil { + t.Fatal(err) + } + if len(stored) != 1 || stored[0].State != tc.wantState { + t.Fatalf("stored %+v, want one end %s", stored, tc.wantState) + } + s := stored[0] + if tc.wantState == sdkstep.StepFailed { + if len(s.Result) != 0 || !strings.Contains(s.Reason, "could not be written") { + t.Errorf("the end %+v, want the small failed end", s) + } + return + } + var a sdkstep.CheckAnswer + if err := json.Unmarshal(s.Result, &a); err != nil { + t.Fatal(err) + } + if len(a.Fetch.Packages) != 0 || a.Fetch.Workspace != "/w" || len(a.Fetch.Changed) != 1 || len(a.Levels) != 1 || len(a.Levels[0].Tests) != 1 { + t.Errorf("the stored answer %s, want its fetch and levels without the taken versions", s.Result) + } + if !strings.Contains(s.Reason, "the fetch's 40 taken versions are left out") || !strings.Contains(s.Reason, "journal names each") || + !strings.Contains(s.Reason, "bytes the bucket takes") { + t.Errorf("the reason %q, want how many were left out, why and where each is said", s.Reason) + } + }) + } +} + // 1930/AC2: given an ask whose end the state refuses, when it runs, then the ask is acknowledged only once its small // failed end is written; when no end can be written at all, it is given back, never acknowledged as done. func TestAnAskWhoseEndCannotBeWrittenIsNotAcked(t *testing.T) { diff --git a/modules/mesh-build-go/buildstep/step.go b/modules/mesh-build-go/buildstep/step.go index 47403fdfb..2bdeadc9f 100644 --- a/modules/mesh-build-go/buildstep/step.go +++ b/modules/mesh-build-go/buildstep/step.go @@ -10,10 +10,12 @@ import ( "errors" "fmt" "log" + "strings" "time" buildrun "git.novox.be/novox/mesh-delivery/sdk/buildrun" sdkstep "git.novox.be/novox/mesh-delivery/sdk/buildstep" + "git.novox.be/novox/mesh-sdk/go/kvlimit" ) // Put writes one key of a state: the steps state.Put's shape (go/state.go). @@ -102,6 +104,24 @@ func finish(put Put, key string, step sdkstep.Step, result any, err error, logf if werr == nil { return nil } + if errors.Is(werr, kvlimit.ErrTooLarge) { + // A check's answer over what the state takes per value is written once more without the fetch's taken versions, + // which no reader judges, the reason saying so (issue 1921, note 91; the thorough fix is design 47 G A-290). + if smaller, left, ok := withoutTakenVersions(step.Result); ok { + less := step + less.Result = smaller + said := fmt.Sprintf("the fetch's %d taken versions are left out of the stored answer: the answer was over what the steps state takes per value (%v); the module's journal names each", left, werr) + less.Reason = strings.TrimPrefix(step.Reason+"; "+said, "; ") + if lerr := writeEnd(put, key, less); lerr == nil { + if logf != nil { + logf("the end of the step %s (%s) was written without its fetch's %d taken versions: %v", key, step.State, left, werr) + } + return nil + } else { + werr = lerr + } + } + } if logf != nil { logf("the end of the step %s (%s) could not be written in %d tries: %v", key, step.State, EndAttempts, werr) } @@ -149,6 +169,34 @@ func protect(ctx context.Context, run func(ctx context.Context) (any, error)) (r return run(ctx) } +// withoutTakenVersions is a check's answer raw without its fetch's taken versions (`fetch.packages`) and how many it +// held; ok is false when the answer holds none, so nothing would be smaller. +func withoutTakenVersions(raw json.RawMessage) (json.RawMessage, int, bool) { + var answer map[string]json.RawMessage + if len(raw) == 0 || json.Unmarshal(raw, &answer) != nil || answer["fetch"] == nil { + return nil, 0, false + } + var fetch map[string]json.RawMessage + if json.Unmarshal(answer["fetch"], &fetch) != nil { + return nil, 0, false + } + var taken []json.RawMessage + if json.Unmarshal(fetch["packages"], &taken) != nil || len(taken) == 0 { + return nil, 0, false + } + delete(fetch, "packages") + f, err := json.Marshal(fetch) + if err != nil { + return nil, 0, false + } + answer["fetch"] = f + out, err := json.Marshal(answer) + if err != nil { + return nil, 0, false + } + return out, len(taken), true +} + func write(put Put, key string, step sdkstep.Step) error { raw, err := sdkstep.EncodeStep(step) if err != nil {