mesh-catalog/modules/mesh-build-go/buildstep/digestshape_test.go

package buildstep

import (
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"sort"
	"strings"
	"sync"
	"testing"
	"time"

	sdkstep "git.novox.be/novox/mesh-delivery/sdk/buildstep"
	"git.novox.be/novox/mesh-sdk/go/kvlimit"
)

// credentialField is the node runtime's rule for a state value (mesh-tools node-tools/internal/bus/state.go, novox/hq
// ADR 0201), copied whole: the first field anywhere in the value whose name, lower-cased and without - _ and spaces, ends
// in a credential word, or "". A step's end is refused when it answers a field.
func credentialField(value json.RawMessage) string {
	endings := []string{"password", "passwd", "secret", "token", "credential", "credentials",
		"authorization", "apikey", "privatekey", "accesskey", "cookie"}
	var v any
	if json.Unmarshal(value, &v) != nil {
		return ""
	}
	var walk func(any) string
	walk = func(v any) string {
		switch t := v.(type) {
		case map[string]any:
			keys := make([]string, 0, len(t))
			for k := range t {
				keys = append(keys, k)
			}
			sort.Strings(keys)
			for _, k := range keys {
				norm := strings.NewReplacer("-", "", "_", "", " ", "").Replace(strings.ToLower(k))
				for _, end := range endings {
					if strings.HasSuffix(norm, end) {
						return k
					}
				}
				if found := walk(t[k]); found != "" {
					return found
				}
			}
		case []any:
			for _, x := range t {
				if found := walk(x); found != "" {
					return found
				}
			}
		}
		return ""
	}
	return walk(v)
}

// runtimeState is a steps state that refuses a value as the node runtime does, and keeps every value it took.
type runtimeState struct {
	mu      sync.Mutex
	took    map[string][]sdkstep.Step
	refused int
}

func newRuntimeState() *runtimeState { return &runtimeState{took: map[string][]sdkstep.Step{}} }

func (s *runtimeState) put(key string, value any) (uint64, error) {
	raw, _ := value.(json.RawMessage)
	if f := credentialField(raw); f != "" {
		s.mu.Lock()
		s.refused++
		s.mu.Unlock()
		return 0, fmt.Errorf("steps.%s carries a field %q, which names a credential", key, f)
	}
	st, err := sdkstep.DecodeStep(raw)
	if err != nil {
		return 0, err
	}
	s.mu.Lock()
	defer s.mu.Unlock()
	s.took[key] = append(s.took[key], st)
	return uint64(len(s.took[key])), nil
}

func (s *runtimeState) last(key string) (sdkstep.Step, bool) {
	s.mu.Lock()
	defer s.mu.Unlock()
	t := s.took[key]
	if len(t) == 0 {
		return sdkstep.Step{}, false
	}
	return t[len(t)-1], true
}

func fastEnds(t *testing.T) {
	attempts, retry := EndAttempts, EndRetry
	EndAttempts, EndRetry = 2, time.Millisecond
	t.Cleanup(func() { EndAttempts, EndRetry = attempts, retry })
}

// 1930/AC1: given a check of a repository with the package .../internal/token, when its ask's answer is stored as the
// step's end, then no field is keyed by a package path: the runtime's credential rule takes the value, and the package
// and its digest read back from the stored end.
func TestADigestOfAPackageNamedTokenIsStoredAsAList(t *testing.T) {
	fastEnds(t)
	r := &recorder{
		fetch:   map[string]any{"workspace": "/w"},
		digests: map[string]string{"x/internal/token": "d-token", "x/pkg/a": "d-a"},
		tests:   map[string]bool{"x/internal/token": true, "x/pkg/a": true},
	}
	out, err := RunAsk(context.Background(), r.steps(), ask(sdkstep.CheckJob{Levels: []string{sdkstep.LevelUnit},
		Fetch: sdkstep.Fetch{Repository: "novox/x", Commit: runCommit}}))
	if err != nil {
		t.Fatal(err)
	}
	old, _ := json.Marshal(map[string]any{"levels": []any{map[string]any{"level": "unit", "digest": map[string]any{"packages": r.digests}}}})
	if credentialField(old) != "x/internal/token" {
		t.Fatalf("the rule does not refuse the form before issue 1930 (%s): the test proves nothing", old)
	}
	st := newRuntimeState()
	if err := finish(st.put, "shanks.a1", sdkstep.Step{ID: "a1", Tool: "check ask", Node: "shanks", State: sdkstep.StepRunning, Started: time.Now()}, out, nil, t.Logf); err != nil {
		t.Fatal(err)
	}
	s, ok := st.last("shanks.a1")
	if !ok || s.State != sdkstep.StepPassed || st.refused != 0 {
		t.Fatalf("the end %+v (written %v, refused %d), want it passed and taken at once", s, ok, st.refused)
	}
	var a sdkstep.CheckAnswer
	if err := json.Unmarshal(s.Result, &a); err != nil {
		t.Fatal(err)
	}
	if len(a.Levels) != 1 || a.Levels[0].Digest == nil {
		t.Fatalf("levels %+v, want the unit level with its digest", a.Levels)
	}
	want := []sdkstep.PackageDigest{{Package: "x/internal/token", Digest: "d-token"}, {Package: "x/pkg/a", Digest: "d-a"}}
	if got := a.Levels[0].Digest.Packages; fmt.Sprint(got) != fmt.Sprint(want) {
		t.Errorf("packages %v, want %v", got, want)
	}
	if !strings.Contains(string(s.Result), `"package":"x/internal/token"`) {
		t.Errorf("the stored end does not name the package as a value: %s", s.Result)
	}
}

// 1930/AC2: given an end the state refuses every time, when finish writes it, then it logs the refusal and writes a small
// failed end without the result that names the write's error; and when no end can be written at all it answers an error.
func TestARefusedEndIsSaidAndEndsTheStepFailed(t *testing.T) {
	fastEnds(t)
	big := errors.New("the value carries a field \"x/internal/token\", which names a credential")
	var refused int
	var stored []sdkstep.Step
	put := func(key string, value any) (uint64, error) {
		st, _ := sdkstep.DecodeStep(value.(json.RawMessage))
		if len(st.Result) > 0 {
			refused++
			return 0, big
		}
		stored = append(stored, st)
		return 1, nil
	}
	var logged []string
	logf := func(format string, args ...any) { logged = append(logged, fmt.Sprintf(format, args...)) }
	err := finish(put, "shanks.a2", sdkstep.Step{ID: "a2", Tool: "check ask", Node: "shanks", State: sdkstep.StepRunning, Started: time.Now()}, map[string]any{"levels": 1}, nil, logf)
	if err != nil {
		t.Fatalf("finish answered %v, want the small end written", err)
	}
	if refused != EndAttempts || len(stored) != 1 {
		t.Fatalf("refused %d, stored %d, want %d refusals and one small end", refused, len(stored), EndAttempts)
	}
	s := stored[0]
	if s.State != sdkstep.StepFailed || len(s.Result) != 0 || !strings.Contains(s.Reason, "could not be written") ||
		!strings.Contains(s.Reason, big.Error()) || !strings.Contains(s.Reason, "passed") {
		t.Errorf("the small end %+v, want failed, no result, naming the write's error and that it passed", s)
	}
	if len(logged) == 0 || !strings.Contains(logged[0], big.Error()) {
		t.Errorf("logged %q, want the refusal said", logged)
	}
	none := func(string, any) (uint64, error) { return 0, errors.New("the bus is gone") }
	if err := finish(none, "shanks.a3", sdkstep.Step{ID: "a3", Tool: "check ask", Node: "shanks", Started: time.Now()}, nil, nil, logf); err == nil || !strings.Contains(err.Error(), "the bus is gone") {
		t.Errorf("finish with no write possible answered %v, want the error", err)
	}
}

// 1921 note 91: given a check's answer over what the steps state takes per value, the bulk of it the fetch's taken
// versions, when finish writes it, then the end is written passed without the taken versions, its levels and the rest of
// its fetch kept, the reason saying how many were left out and why; an over-large answer with no taken versions, and one
// still too large without them, end failed as before.
func TestAnAnswerOverTheStateBoundLeavesOutItsTakenVersions(t *testing.T) {
	fastEnds(t)
	answer := func(taken int) map[string]any {
		var packages []map[string]any
		for i := range taken {
			packages = append(packages, map[string]any{"dir": fmt.Sprintf("modules/m%d", i), "package": "git.novox.be/novox/mesh-sdk/go", "said": "the ask names it"})
		}
		return map[string]any{
			"fetch":  map[string]any{"workspace": "/w", "changed": []string{"modules/m0/module.json"}, "packages": packages},
			"levels": []any{map[string]any{"level": "unit", "tests": []any{map[string]any{"passed": true}}}},
		}
	}
	for _, tc := range []struct {
		name      string
		taken     int
		limit     int
		wantState string
	}{
		{"its taken versions left out", 40, 1500, sdkstep.StepPassed},
		{"no taken versions to leave out", 0, 200, sdkstep.StepFailed},
		{"still too large without them", 40, 200, sdkstep.StepFailed},
	} {
		t.Run(tc.name, func(t *testing.T) {
			var stored []sdkstep.Step
			put := func(key string, value any) (uint64, error) {
				raw := value.(json.RawMessage)
				if err := kvlimit.Check("the state steps", key, len(raw), int64(tc.limit)); err != nil && strings.Contains(string(raw), `"result"`) {
					return 0, err
				}
				st, _ := sdkstep.DecodeStep(raw)
				stored = append(stored, st)
				return 1, nil
			}
			if err := finish(put, "shanks.a4", sdkstep.Step{ID: "a4", Tool: "check ask", Node: "shanks", State: sdkstep.StepRunning, Started: time.Now()}, answer(tc.taken), nil, t.Logf); err != nil {
				t.Fatal(err)
			}
			if len(stored) != 1 || stored[0].State != tc.wantState {
				t.Fatalf("stored %+v, want one end %s", stored, tc.wantState)
			}
			s := stored[0]
			if tc.wantState == sdkstep.StepFailed {
				if len(s.Result) != 0 || !strings.Contains(s.Reason, "could not be written") {
					t.Errorf("the end %+v, want the small failed end", s)
				}
				return
			}
			var a sdkstep.CheckAnswer
			if err := json.Unmarshal(s.Result, &a); err != nil {
				t.Fatal(err)
			}
			if len(a.Fetch.Packages) != 0 || a.Fetch.Workspace != "/w" || len(a.Fetch.Changed) != 1 || len(a.Levels) != 1 || len(a.Levels[0].Tests) != 1 {
				t.Errorf("the stored answer %s, want its fetch and levels without the taken versions", s.Result)
			}
			if !strings.Contains(s.Reason, "the fetch's 40 taken versions are left out") || !strings.Contains(s.Reason, "journal names each") ||
				!strings.Contains(s.Reason, "bytes the bucket takes") {
				t.Errorf("the reason %q, want how many were left out, why and where each is said", s.Reason)
			}
		})
	}
}

// 1930/AC2: given an ask whose end the state refuses, when it runs, then the ask is acknowledged only once its small
// failed end is written; when no end can be written at all, it is given back, never acknowledged as done.
func TestAnAskWhoseEndCannotBeWrittenIsNotAcked(t *testing.T) {
	fastEnds(t)
	for _, tc := range []struct {
		name      string
		smallOK   bool
		wantAcked bool
	}{{"the small end written", true, true}, {"no end written", false, false}} {
		t.Run(tc.name, func(t *testing.T) {
			src := NewFakeAsks()
			a := testAsk(sdkstep.AskCheck)
			_ = src.PublishAsk(a)
			var mu sync.Mutex
			var ends []sdkstep.Step
			put := func(key string, value any) (uint64, error) {
				st, err := sdkstep.DecodeStep(value.(json.RawMessage))
				if err != nil {
					return 0, err
				}
				if st.State != sdkstep.StepRunning && (len(st.Result) > 0 || !tc.smallOK) {
					return 0, errors.New("refused")
				}
				mu.Lock()
				ends = append(ends, st)
				mu.Unlock()
				return 1, nil
			}
			p := &Puller{Source: src, Language: sdkstep.AskGo, Node: "shanks", Put: put, Capacity: func() int { return 1 },
				PullWait: 10 * time.Millisecond, Logf: t.Logf,
				Run: func(ctx context.Context, a sdkstep.StepAsk) (any, error) { return map[string]any{"ran": true}, nil }}
			ctx, cancel := context.WithCancel(context.Background())
			defer cancel()
			go p.Pull(ctx)
			waitFor(t, "the ask's fate", func() bool { ac, tm, nk := src.Fates(); return len(ac)+len(tm)+len(nk) > 0 })
			cancel()
			acked, termed, naked := src.Fates()
			if tc.wantAcked && (len(acked) != 1 || len(naked) != 0) || !tc.wantAcked && (len(acked) != 0 || len(naked) == 0) || len(termed) != 0 {
				t.Fatalf("acked %v naked %v termed %v", acked, naked, termed)
			}
			mu.Lock()
			defer mu.Unlock()
			if tc.smallOK {
				last := ends[len(ends)-1]
				if last.State != sdkstep.StepFailed || !strings.Contains(last.Reason, "refused") {
					t.Errorf("the end %+v, want the small failed end", last)
				}
			}
		})
	}
}