mesh-delivery/modules/mesh-delivery-checker/internal/check/try.go

package check

import (
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"regexp"
	"sort"
	"strconv"
	"strings"
	"sync"
	"time"

	"git.novox.be/novox/mesh-sdk/go/delivery"
)

// A try is a pushed head judged by the checker as a pull request's head is, before its pull request is opened (ADR 0501
// §5; issue 686's `mesh-controller.try`, which calls these tools from R4e): the head merged onto its trunk, the unit and the
// integration level through the build modules. It owes the forge nothing (no status, no preview) and writes no verdict; its
// state is the value of its key in the checker's state tries.

// The try's tools and state.
const (
	TryTool     = "checker_try"
	TryReadTool = "checker_try_read"
	// TriesState is the checker's state of tries: the bucket mesh-delivery-checker_tries, keys try-<unix nanoseconds>.
	TriesState  = "tries"
	TriesBucket = delivery.CheckerModule + "_" + TriesState
	// MaxTries is how many tries may wait or run at once, mesh-wide: each holds a build node for minutes.
	MaxTries = 4
	// TryKeep is how long an ended try's key is kept before the checker removes it.
	TryKeep = 7 * 24 * time.Hour
)

// The states of a try: asked and waiting, running, and its end.
const (
	TryQueued  = "queued"
	TryRunning = "running"
	TryPass    = "pass"
	TryFail    = "fail"
	TryError   = "error"
)

var (
	tryKeyOK   = regexp.MustCompile(`^try-[0-9]{1,20}$`)
	tryHeadOK  = regexp.MustCompile(`^([0-9a-f]{40}|[0-9a-f]{64})$`)
	tryTestsOK = regexp.MustCompile(`^[A-Za-z0-9_^$|().*+?/\[\]-]+$`)
	tryRepoOK  = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._-]*/[A-Za-z0-9][A-Za-z0-9._-]*$`)
	tryTrunkOK = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._/-]*$`)
)

// Try is the value of a tries key.
type Try struct {
	V          int    `json:"v"`
	ID         string `json:"id"`
	Repository string `json:"repository"`
	// Head is the commit tried; Trunk the trunk commit it is merged onto, and Branch the trunk's branch when it was asked by name.
	Head   string `json:"head"`
	Trunk  string `json:"trunk"`
	Branch string `json:"branch,omitempty"`
	// Tests is a go test -run pattern: only those tests run, and nothing is kept. Level is unit or integration: only that
	// level runs. Empty is every test of both levels.
	Tests string `json:"tests,omitempty"`
	Level string `json:"level,omitempty"`
	State string `json:"state"`
	// Said is the try's report in words: what failed first, the tests run, kept and skipped per level, the time of each part.
	Said string `json:"said,omitempty"`
	// Took is the time from its start to its end, as Go writes a duration.
	Took     string                          `json:"took,omitempty"`
	Failed   []string                        `json:"failed,omitempty"`
	Levels   map[string]delivery.LevelResult `json:"levels,omitempty"`
	Machines []string                        `json:"machines,omitempty"`
	Asked    time.Time                       `json:"asked"`
	Started  time.Time                       `json:"started,omitzero"`
	Ended    time.Time                       `json:"ended,omitzero"`
}

// Over says whether the try came to its end.
func (t Try) Over() bool { return t.State == TryPass || t.State == TryFail || t.State == TryError }

// DecodeTry reads a tries value.
func DecodeTry(raw []byte) (Try, error) {
	var t Try
	if err := json.Unmarshal(raw, &t); err != nil {
		return Try{}, fmt.Errorf("not a try: %w", err)
	}
	if !tryKeyOK.MatchString(t.ID) || t.Repository == "" || t.Head == "" || t.Trunk == "" {
		return Try{}, errors.New("not a try: it names no id, repository, head or trunk")
	}
	return t, nil
}

func encodeTry(t Try) ([]byte, error) {
	t.V = delivery.Version
	return json.Marshal(t)
}

// TryAsk is what checker_try is given.
type TryAsk struct {
	Repository, Head, Trunk, Tests, Level string
}

// TryStore is the tries state as the tools see it.
type TryStore interface {
	Keys() ([]string, error)
	Get(key string) (json.RawMessage, error)
	Create(key string, value []byte) error
}

// NotChecked is the error of a try of a repository the checker judges no level of: hq, judged by merge-check.sh (ADR 0484
// clause 8), and a repository whose trunk the gate's source does not name.
func NotChecked(repository string) error {
	return fmt.Errorf("no level of %s is checked by the checker", repository)
}

// AskTry writes a new try, queued, and answers its id: a try of the same head that waits or runs is the answer to a second
// ask, and at most MaxTries wait or run at once. trunks are the gate's trunk heads of every repository (source.main): a
// repository with none, and hq, are no repository the checker checks. A trunk asked by name is its trunk head now.
func AskTry(store TryStore, trunks map[string]string, a TryAsk, now time.Time) (string, error) {
	repo := strings.ToLower(strings.TrimSuffix(strings.TrimSpace(a.Repository), ".git"))
	head := strings.ToLower(strings.TrimSpace(a.Head))
	switch {
	case !tryRepoOK.MatchString(repo) || strings.Contains(repo, ".."):
		return "", fmt.Errorf("%q is not a repository, <owner>/<name>", a.Repository)
	case !tryHeadOK.MatchString(head):
		return "", fmt.Errorf("the head %q is not a full commit, 40 or 64 hex digits", a.Head)
	case a.Level != "" && a.Level != delivery.LevelUnit && a.Level != delivery.LevelIntegration:
		return "", fmt.Errorf("the level %q is neither unit nor integration", a.Level)
	case a.Tests != "" && !tryTestsOK.MatchString(a.Tests):
		return "", fmt.Errorf("the tests %q are not a go test -run pattern", a.Tests)
	}
	trunkHead := ""
	for r, h := range trunks {
		if strings.EqualFold(r, repo) {
			trunkHead = h
		}
	}
	if repo == HQRepository || trunkHead == "" {
		return "", NotChecked(repo)
	}
	t := Try{Repository: repo, Head: head, Tests: a.Tests, Level: a.Level, State: TryQueued, Asked: now.UTC()}
	switch trunk := strings.TrimSpace(a.Trunk); {
	case trunk == "":
		t.Trunk = trunkHead
	case tryHeadOK.MatchString(strings.ToLower(trunk)):
		t.Trunk = strings.ToLower(trunk)
	case tryTrunkOK.MatchString(trunk) && !strings.Contains(trunk, ".."):
		// The checker judges pull requests into the trunk alone (ADR 0484 clause 11): a branch named is the trunk's head now.
		t.Trunk, t.Branch = trunkHead, trunk
	default:
		return "", fmt.Errorf("the trunk %q is neither a full commit nor a branch", a.Trunk)
	}
	keys, err := store.Keys()
	if err != nil {
		return "", fmt.Errorf("the tries cannot be listed: %w", err)
	}
	sort.Strings(keys)
	var open []string
	for _, k := range keys {
		raw, err := store.Get(k)
		if err != nil || len(raw) == 0 {
			continue
		}
		cur, err := DecodeTry(raw)
		if err != nil || cur.Over() {
			continue
		}
		if cur.Repository == t.Repository && cur.Head == t.Head && cur.Trunk == t.Trunk && cur.Tests == t.Tests && cur.Level == t.Level {
			return cur.ID, nil
		}
		open = append(open, fmt.Sprintf("%s (%s@%.12s, %s)", cur.ID, cur.Repository, cur.Head, cur.State))
	}
	if len(open) >= MaxTries {
		return "", fmt.Errorf("%d tries wait or run, the most at once: %s; ask again when one ends", len(open), strings.Join(open, ", "))
	}
	t.ID = "try-" + strconv.FormatInt(now.UnixNano(), 10)
	data, err := encodeTry(t)
	if err != nil {
		return "", err
	}
	if err := store.Create(t.ID, data); err != nil {
		return "", fmt.Errorf("the try %s cannot be written: %w", t.ID, err)
	}
	return t.ID, nil
}

// ReadTry is a try's value, as stored.
func ReadTry(store TryStore, id string) (json.RawMessage, error) {
	if !tryKeyOK.MatchString(id) {
		return nil, fmt.Errorf("%q is not a try's id, try-<digits>", id)
	}
	raw, err := store.Get(id)
	if err != nil {
		return nil, err
	}
	if len(raw) == 0 {
		return nil, fmt.Errorf("no try %s: it was never asked, or it ended more than %s ago and was removed", id, TryKeep)
	}
	return raw, nil
}

// TryOptions narrow what a try runs; a pull request's check has none.
type TryOptions struct {
	Level string
	Run   string
}

// Tryer runs the tries: it takes a queued try, judges it through the same Judge as a pull request's check, and writes its
// end. It holds no timer: a try starts when its key is written, or when a delivery that held it ends (Reconsider).
type Tryer struct {
	Tries KV
	Judge Judge
	// Hold says whether a delivery builds, so no try starts (ADR 0484 clause 12); nil never holds.
	Hold  func() bool
	Bound time.Duration
	Now   func() time.Time
	Logf  func(format string, args ...any)

	mu       sync.Mutex
	wg       sync.WaitGroup
	inflight map[string]bool
	seen     map[string]bool // looked at since this process started: a try left running is taken over the first time
}

func (tr *Tryer) now() time.Time {
	if tr.Now != nil {
		return tr.Now()
	}
	return time.Now()
}

func (tr *Tryer) logf(format string, args ...any) {
	if tr.Logf != nil {
		tr.Logf(format, args...)
	}
}

// Wait returns when every try started so far has ended.
func (tr *Tryer) Wait() { tr.wg.Wait() }

// Put takes a value of the tries state in: a queued try is started, and a try left running is taken over the first time
// this process sees it (the checker that ran it stopped; there is one checker).
func (tr *Tryer) Put(ctx context.Context, key string, value []byte) {
	t, err := DecodeTry(value)
	if err != nil {
		tr.logf("tries.%s holds what is not a try, left: %v", key, err)
		return
	}
	if t.Over() {
		return
	}
	held := tr.Hold != nil && tr.Hold()
	tr.mu.Lock()
	if tr.inflight == nil {
		tr.inflight, tr.seen = map[string]bool{}, map[string]bool{}
	}
	if tr.inflight[key] || (t.State == TryRunning && tr.seen[key]) {
		tr.mu.Unlock()
		return
	}
	if held {
		tr.mu.Unlock() // not seen yet: a try left running is still taken over when the hold ends
		tr.logf("%s: not started, a delivery builds", key)
		return
	}
	tr.seen[key] = true
	tr.inflight[key] = true
	tr.mu.Unlock()
	tr.wg.Add(1)
	go func() {
		defer tr.wg.Done()
		defer func() { tr.mu.Lock(); delete(tr.inflight, key); tr.mu.Unlock() }()
		tr.run(ctx, key)
	}()
}

// Reconsider looks at every try again (a delivery that held them ended), and removes the ended tries older than TryKeep.
func (tr *Tryer) Reconsider(ctx context.Context) {
	keys, err := tr.Tries.Keys(ctx)
	if err != nil {
		tr.logf("the tries cannot be listed: %v", err)
		return
	}
	for _, k := range keys {
		raw, _, found, err := tr.Tries.Get(ctx, k)
		if err != nil || !found {
			continue
		}
		if t, err := DecodeTry(raw); err == nil && t.Over() && tr.now().Sub(t.Ended) > TryKeep {
			if d, ok := tr.Tries.(interface {
				Delete(ctx context.Context, key string) error
			}); ok {
				if err := d.Delete(ctx, k); err != nil {
					tr.logf("%s ended %s ago and could not be removed: %v", k, tr.now().Sub(t.Ended).Round(time.Hour), err)
				}
			}
			continue
		}
		tr.Put(ctx, k, raw)
	}
}

// run claims the try by compare-and-set, judges it within its bound and writes its end.
func (tr *Tryer) run(ctx context.Context, key string) {
	raw, rev, found, err := tr.Tries.Get(ctx, key)
	if err != nil || !found {
		return
	}
	t, err := DecodeTry(raw)
	if err != nil || t.Over() {
		return
	}
	t.State, t.Started = TryRunning, tr.now().UTC()
	data, err := encodeTry(t)
	if err != nil {
		return
	}
	if rev, err = tr.Tries.Update(ctx, key, data, rev); err != nil {
		if !errors.Is(err, ErrConflict) {
			tr.logf("%s: %v", key, err)
		}
		return
	}
	cs := &Case{Key: key, Group: key,
		Pulls:    []Pull{{Repository: t.Repository, Head: t.Head, Trunk: t.Trunk}},
		Trunk:    map[string]string{t.Repository: t.Trunk},
		Queued:   t.Asked,
		Progress: &Progress{},
		Try:      &TryOptions{Level: t.Level, Run: t.Tests}}
	out, judged := judgeWithin(ctx, tr.Judge, cs, tr.Bound)
	if ctx.Err() != nil {
		return // left running: the checker takes it over when it starts
	}
	v := out.Verdict
	t.Ended = tr.now().UTC()
	t.Took = t.Ended.Sub(t.Started).Round(time.Second).String()
	t.Levels, t.Machines = v.Levels, v.Machines
	t.Said = Report(v, cs, cs.Pulls[0], out.Builds, out.Slowest)
	switch {
	case !judged || IsNodesFailure(v):
		t.State = TryError
	case v.State == delivery.VerdictPass:
		t.State = TryPass
	default:
		t.State = TryFail
		if f := firstFailed(v); f != "" {
			t.Failed = []string{f}
		}
	}
	if t.Tests != "" {
		t.Said += " | only the tests " + t.Tests + " (Go); nothing kept"
	}
	if data, err = encodeTry(t); err == nil {
		_, err = tr.Tries.Update(ctx, key, data, rev)
	}
	if err != nil {
		tr.logf("%s: its end could not be written: %v", key, err)
		return
	}
	tr.logf("%s: %s", key, t.State)
}

// judgeWithin runs a case within the bound (DefaultBound when 0). A judge that fails to judge, and a check that ran past its
// bound, are verdicts of failure naming the machine and the step, and judged says false for them.
func judgeWithin(ctx context.Context, j Judge, cs *Case, bound time.Duration) (Outcome, bool) {
	if bound <= 0 {
		bound = DefaultBound
	}
	run, cancel := context.WithTimeoutCause(ctx, bound, errBound)
	defer cancel()
	out, err := j.Judge(run, cs)
	stage, node := cs.Progress.Now()
	if node == "" {
		node = "checker"
	}
	switch {
	case ctx.Err() != nil:
		return Outcome{}, false
	case context.Cause(run) == errBound:
		if stage == "" {
			stage = "starting"
		}
		return Outcome{Verdict: failure(node, fmt.Sprintf("the check ran past its bound of %s, stuck at: %s", bound, stage))}, false
	case err != nil:
		return Outcome{Verdict: failure(node, err.Error())}, false
	}
	return out, true
}