Merge pull request 'A step answer over the steps state's bound is kept without its taken versions, said, not lost' (#667) from fix/step-answer-over-bound into main

modules/mesh-build-go/buildstep/digestshape_test.go

@@ -12,6 +12,7 @@ import (
 	"time"
 
 	sdkstep "git.novox.be/novox/mesh-delivery/sdk/buildstep"
+	"git.novox.be/novox/mesh-sdk/go/kvlimit"
 )
 
 // credentialField is the node runtime's rule for a state value (mesh-tools node-tools/internal/bus/state.go, novox/hq
@@ -181,6 +182,71 @@ func TestARefusedEndIsSaidAndEndsTheStepFailed(t *testing.T) {
 	}
 }
 
+// 1921 note 91: given a check's answer over what the steps state takes per value, the bulk of it the fetch's taken
+// versions, when finish writes it, then the end is written passed without the taken versions, its levels and the rest of
+// its fetch kept, the reason saying how many were left out and why; an over-large answer with no taken versions, and one
+// still too large without them, end failed as before.
+func TestAnAnswerOverTheStateBoundLeavesOutItsTakenVersions(t *testing.T) {
+	fastEnds(t)
+	answer := func(taken int) map[string]any {
+		var packages []map[string]any
+		for i := range taken {
+			packages = append(packages, map[string]any{"dir": fmt.Sprintf("modules/m%d", i), "package": "git.novox.be/novox/mesh-sdk/go", "said": "the ask names it"})
+		}
+		return map[string]any{
+			"fetch":  map[string]any{"workspace": "/w", "changed": []string{"modules/m0/module.json"}, "packages": packages},
+			"levels": []any{map[string]any{"level": "unit", "tests": []any{map[string]any{"passed": true}}}},
+		}
+	}
+	for _, tc := range []struct {
+		name      string
+		taken     int
+		limit     int
+		wantState string
+	}{
+		{"its taken versions left out", 40, 1500, sdkstep.StepPassed},
+		{"no taken versions to leave out", 0, 200, sdkstep.StepFailed},
+		{"still too large without them", 40, 200, sdkstep.StepFailed},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			var stored []sdkstep.Step
+			put := func(key string, value any) (uint64, error) {
+				raw := value.(json.RawMessage)
+				if err := kvlimit.Check("the state steps", key, len(raw), int64(tc.limit)); err != nil && strings.Contains(string(raw), `"result"`) {
+					return 0, err
+				}
+				st, _ := sdkstep.DecodeStep(raw)
+				stored = append(stored, st)
+				return 1, nil
+			}
+			if err := finish(put, "shanks.a4", sdkstep.Step{ID: "a4", Tool: "check ask", Node: "shanks", State: sdkstep.StepRunning, Started: time.Now()}, answer(tc.taken), nil, t.Logf); err != nil {
+				t.Fatal(err)
+			}
+			if len(stored) != 1 || stored[0].State != tc.wantState {
+				t.Fatalf("stored %+v, want one end %s", stored, tc.wantState)
+			}
+			s := stored[0]
+			if tc.wantState == sdkstep.StepFailed {
+				if len(s.Result) != 0 || !strings.Contains(s.Reason, "could not be written") {
+					t.Errorf("the end %+v, want the small failed end", s)
+				}
+				return
+			}
+			var a sdkstep.CheckAnswer
+			if err := json.Unmarshal(s.Result, &a); err != nil {
+				t.Fatal(err)
+			}
+			if len(a.Fetch.Packages) != 0 || a.Fetch.Workspace != "/w" || len(a.Fetch.Changed) != 1 || len(a.Levels) != 1 || len(a.Levels[0].Tests) != 1 {
+				t.Errorf("the stored answer %s, want its fetch and levels without the taken versions", s.Result)
+			}
+			if !strings.Contains(s.Reason, "the fetch's 40 taken versions are left out") || !strings.Contains(s.Reason, "journal names each") ||
+				!strings.Contains(s.Reason, "bytes the bucket takes") {
+				t.Errorf("the reason %q, want how many were left out, why and where each is said", s.Reason)
+			}
+		})
+	}
+}
+
 // 1930/AC2: given an ask whose end the state refuses, when it runs, then the ask is acknowledged only once its small
 // failed end is written; when no end can be written at all, it is given back, never acknowledged as done.
 func TestAnAskWhoseEndCannotBeWrittenIsNotAcked(t *testing.T) {

modules/mesh-build-go/buildstep/step.go

@@ -10,10 +10,12 @@ import (
 	"errors"
 	"fmt"
 	"log"
+	"strings"
 	"time"
 
 	buildrun "git.novox.be/novox/mesh-delivery/sdk/buildrun"
 	sdkstep "git.novox.be/novox/mesh-delivery/sdk/buildstep"
+	"git.novox.be/novox/mesh-sdk/go/kvlimit"
 )
 
 // Put writes one key of a state: the steps state.Put's shape (go/state.go).
@@ -102,6 +104,24 @@ func finish(put Put, key string, step sdkstep.Step, result any, err error, logf
 	if werr == nil {
 		return nil
 	}
+	if errors.Is(werr, kvlimit.ErrTooLarge) {
+		// A check's answer over what the state takes per value is written once more without the fetch's taken versions,
+		// which no reader judges, the reason saying so (issue 1921, note 91; the thorough fix is design 47 G A-290).
+		if smaller, left, ok := withoutTakenVersions(step.Result); ok {
+			less := step
+			less.Result = smaller
+			said := fmt.Sprintf("the fetch's %d taken versions are left out of the stored answer: the answer was over what the steps state takes per value (%v); the module's journal names each", left, werr)
+			less.Reason = strings.TrimPrefix(step.Reason+"; "+said, "; ")
+			if lerr := writeEnd(put, key, less); lerr == nil {
+				if logf != nil {
+					logf("the end of the step %s (%s) was written without its fetch's %d taken versions: %v", key, step.State, left, werr)
+				}
+				return nil
+			} else {
+				werr = lerr
+			}
+		}
+	}
 	if logf != nil {
 		logf("the end of the step %s (%s) could not be written in %d tries: %v", key, step.State, EndAttempts, werr)
 	}
@@ -149,6 +169,34 @@ func protect(ctx context.Context, run func(ctx context.Context) (any, error)) (r
 	return run(ctx)
 }
 
+// withoutTakenVersions is a check's answer raw without its fetch's taken versions (`fetch.packages`) and how many it
+// held; ok is false when the answer holds none, so nothing would be smaller.
+func withoutTakenVersions(raw json.RawMessage) (json.RawMessage, int, bool) {
+	var answer map[string]json.RawMessage
+	if len(raw) == 0 || json.Unmarshal(raw, &answer) != nil || answer["fetch"] == nil {
+		return nil, 0, false
+	}
+	var fetch map[string]json.RawMessage
+	if json.Unmarshal(answer["fetch"], &fetch) != nil {
+		return nil, 0, false
+	}
+	var taken []json.RawMessage
+	if json.Unmarshal(fetch["packages"], &taken) != nil || len(taken) == 0 {
+		return nil, 0, false
+	}
+	delete(fetch, "packages")
+	f, err := json.Marshal(fetch)
+	if err != nil {
+		return nil, 0, false
+	}
+	answer["fetch"] = f
+	out, err := json.Marshal(answer)
+	if err != nil {
+		return nil, 0, false
+	}
+	return out, len(taken), true
+}
+
 func write(put Put, key string, step sdkstep.Step) error {
 	raw, err := sdkstep.EncodeStep(step)
 	if err != nil {