package check import ( "context" "encoding/json" "errors" "fmt" "regexp" "sort" "strconv" "strings" "sync" "time" "git.novox.be/novox/mesh-sdk/go/delivery" ) // A try is a pushed head judged by the checker as a pull request's head is, before its pull request is opened (ADR 0501 // §5; issue 686's `mesh-controller.try`, which calls these tools from R4e): the head merged onto its trunk, the unit and the // integration level through the build modules. It owes the forge nothing (no status, no preview) and writes no verdict; its // state is the value of its key in the checker's state tries. // The try's tools and state. const ( TryTool = "checker_try" TryReadTool = "checker_try_read" // TriesState is the checker's state of tries: the bucket mesh-delivery-checker_tries, keys try-. TriesState = "tries" TriesBucket = delivery.CheckerModule + "_" + TriesState // MaxTries is how many tries may wait or run at once, mesh-wide: each holds a build node for minutes. MaxTries = 4 // TryKeep is how long an ended try's key is kept before the checker removes it. TryKeep = 7 * 24 * time.Hour ) // The states of a try: asked and waiting, running, and its end. const ( TryQueued = "queued" TryRunning = "running" TryPass = "pass" TryFail = "fail" TryError = "error" ) var ( tryKeyOK = regexp.MustCompile(`^try-[0-9]{1,20}$`) tryHeadOK = regexp.MustCompile(`^([0-9a-f]{40}|[0-9a-f]{64})$`) tryTestsOK = regexp.MustCompile(`^[A-Za-z0-9_^$|().*+?/\[\]-]+$`) tryRepoOK = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._-]*/[A-Za-z0-9][A-Za-z0-9._-]*$`) tryTrunkOK = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._/-]*$`) ) // Try is the value of a tries key. type Try struct { V int `json:"v"` ID string `json:"id"` Repository string `json:"repository"` // Head is the commit tried; Trunk the trunk commit it is merged onto, and Branch the trunk's branch when it was asked by name. Head string `json:"head"` Trunk string `json:"trunk"` Branch string `json:"branch,omitempty"` // Tests is a go test -run pattern: only those tests run, and nothing is kept. Level is unit or integration: only that // level runs. Empty is every test of both levels. Tests string `json:"tests,omitempty"` Level string `json:"level,omitempty"` State string `json:"state"` // Said is the try's report in words: what failed first, the tests run, kept and skipped per level, the time of each part. Said string `json:"said,omitempty"` // Took is the time from its start to its end, as Go writes a duration. Took string `json:"took,omitempty"` Failed []string `json:"failed,omitempty"` Levels map[string]delivery.LevelResult `json:"levels,omitempty"` Machines []string `json:"machines,omitempty"` Asked time.Time `json:"asked"` Started time.Time `json:"started,omitzero"` Ended time.Time `json:"ended,omitzero"` } // Over says whether the try came to its end. func (t Try) Over() bool { return t.State == TryPass || t.State == TryFail || t.State == TryError } // DecodeTry reads a tries value. func DecodeTry(raw []byte) (Try, error) { var t Try if err := json.Unmarshal(raw, &t); err != nil { return Try{}, fmt.Errorf("not a try: %w", err) } if !tryKeyOK.MatchString(t.ID) || t.Repository == "" || t.Head == "" || t.Trunk == "" { return Try{}, errors.New("not a try: it names no id, repository, head or trunk") } return t, nil } func encodeTry(t Try) ([]byte, error) { t.V = delivery.Version return json.Marshal(t) } // TryAsk is what checker_try is given. type TryAsk struct { Repository, Head, Trunk, Tests, Level string } // TryStore is the tries state as the tools see it. type TryStore interface { Keys() ([]string, error) Get(key string) (json.RawMessage, error) Create(key string, value []byte) error } // NotChecked is the error of a try of a repository the checker judges no level of: one whose trunk the gate's source does // not name. func NotChecked(repository string) error { return fmt.Errorf("no level of %s is checked by the checker", repository) } // AskTry writes a new try, queued, and answers its id: a try of the same head that waits or runs is the answer to a second // ask, and at most MaxTries wait or run at once. trunks are the gate's trunk heads of every repository (source.main): a // repository with none is no repository the checker checks. A trunk asked by name is its trunk head now. func AskTry(store TryStore, trunks map[string]string, a TryAsk, now time.Time) (string, error) { repo := strings.ToLower(strings.TrimSuffix(strings.TrimSpace(a.Repository), ".git")) head := strings.ToLower(strings.TrimSpace(a.Head)) switch { case !tryRepoOK.MatchString(repo) || strings.Contains(repo, ".."): return "", fmt.Errorf("%q is not a repository, /", a.Repository) case !tryHeadOK.MatchString(head): return "", fmt.Errorf("the head %q is not a full commit, 40 or 64 hex digits", a.Head) case a.Level != "" && a.Level != delivery.LevelUnit && a.Level != delivery.LevelIntegration: return "", fmt.Errorf("the level %q is neither unit nor integration", a.Level) case a.Tests != "" && !tryTestsOK.MatchString(a.Tests): return "", fmt.Errorf("the tests %q are not a go test -run pattern", a.Tests) } trunkHead := "" for r, h := range trunks { if strings.EqualFold(r, repo) { trunkHead = h } } if trunkHead == "" { return "", NotChecked(repo) } t := Try{Repository: repo, Head: head, Tests: a.Tests, Level: a.Level, State: TryQueued, Asked: now.UTC()} switch trunk := strings.TrimSpace(a.Trunk); { case trunk == "": t.Trunk = trunkHead case tryHeadOK.MatchString(strings.ToLower(trunk)): t.Trunk = strings.ToLower(trunk) case tryTrunkOK.MatchString(trunk) && !strings.Contains(trunk, ".."): // The checker judges pull requests into the trunk alone (ADR 0484 clause 11): a branch named is the trunk's head now. t.Trunk, t.Branch = trunkHead, trunk default: return "", fmt.Errorf("the trunk %q is neither a full commit nor a branch", a.Trunk) } keys, err := store.Keys() if err != nil { return "", fmt.Errorf("the tries cannot be listed: %w", err) } sort.Strings(keys) var open []string for _, k := range keys { raw, err := store.Get(k) if err != nil || len(raw) == 0 { continue } cur, err := DecodeTry(raw) if err != nil || cur.Over() { continue } if cur.Repository == t.Repository && cur.Head == t.Head && cur.Trunk == t.Trunk && cur.Tests == t.Tests && cur.Level == t.Level { return cur.ID, nil } open = append(open, fmt.Sprintf("%s (%s@%.12s, %s)", cur.ID, cur.Repository, cur.Head, cur.State)) } if len(open) >= MaxTries { return "", fmt.Errorf("%d tries wait or run, the most at once: %s; ask again when one ends", len(open), strings.Join(open, ", ")) } t.ID = "try-" + strconv.FormatInt(now.UnixNano(), 10) data, err := encodeTry(t) if err != nil { return "", err } if err := store.Create(t.ID, data); err != nil { return "", fmt.Errorf("the try %s cannot be written: %w", t.ID, err) } return t.ID, nil } // ReadTry is a try's value, as stored. func ReadTry(store TryStore, id string) (json.RawMessage, error) { if !tryKeyOK.MatchString(id) { return nil, fmt.Errorf("%q is not a try's id, try-", id) } raw, err := store.Get(id) if err != nil { return nil, err } if len(raw) == 0 { return nil, fmt.Errorf("no try %s: it was never asked, or it ended more than %s ago and was removed", id, TryKeep) } return raw, nil } // TryOptions narrow what a try runs; a pull request's check has none. type TryOptions struct { Level string Run string } // Tryer runs the tries: it takes a queued try, judges it through the same Judge as a pull request's check (a check ask, ADR // 0525 §1), and writes its end. It holds no timer: a try starts when its key is written. type Tryer struct { Tries KV Judge Judge Bound time.Duration Now func() time.Time Logf func(format string, args ...any) mu sync.Mutex wg sync.WaitGroup inflight map[string]bool seen map[string]bool // looked at since this process started: a try left running is taken over the first time } func (tr *Tryer) now() time.Time { if tr.Now != nil { return tr.Now() } return time.Now() } func (tr *Tryer) logf(format string, args ...any) { if tr.Logf != nil { tr.Logf(format, args...) } } // Wait returns when every try started so far has ended. func (tr *Tryer) Wait() { tr.wg.Wait() } // Put takes a value of the tries state in: a queued try is started, and a try left running is taken over the first time // this process sees it (the checker that ran it stopped; there is one checker). func (tr *Tryer) Put(ctx context.Context, key string, value []byte) { t, err := DecodeTry(value) if err != nil { tr.logf("tries.%s holds what is not a try, left: %v", key, err) return } if t.Over() { return } tr.mu.Lock() if tr.inflight == nil { tr.inflight, tr.seen = map[string]bool{}, map[string]bool{} } if tr.inflight[key] || (t.State == TryRunning && tr.seen[key]) { tr.mu.Unlock() return } tr.seen[key] = true tr.inflight[key] = true tr.mu.Unlock() tr.wg.Add(1) go func() { defer tr.wg.Done() defer func() { tr.mu.Lock(); delete(tr.inflight, key); tr.mu.Unlock() }() tr.run(ctx, key) }() } // Reconsider looks at every try again (a delivery that held them ended), and removes the ended tries older than TryKeep. func (tr *Tryer) Reconsider(ctx context.Context) { keys, err := tr.Tries.Keys(ctx) if err != nil { tr.logf("the tries cannot be listed: %v", err) return } for _, k := range keys { raw, _, found, err := tr.Tries.Get(ctx, k) if err != nil || !found { continue } if t, err := DecodeTry(raw); err == nil && t.Over() && tr.now().Sub(t.Ended) > TryKeep { if d, ok := tr.Tries.(interface { Delete(ctx context.Context, key string) error }); ok { if err := d.Delete(ctx, k); err != nil { tr.logf("%s ended %s ago and could not be removed: %v", k, tr.now().Sub(t.Ended).Round(time.Hour), err) } } continue } tr.Put(ctx, k, raw) } } // run claims the try by compare-and-set, judges it within its bound and writes its end. func (tr *Tryer) run(ctx context.Context, key string) { raw, rev, found, err := tr.Tries.Get(ctx, key) if err != nil || !found { return } t, err := DecodeTry(raw) if err != nil || t.Over() { return } t.State, t.Started = TryRunning, tr.now().UTC() data, err := encodeTry(t) if err != nil { return } if rev, err = tr.Tries.Update(ctx, key, data, rev); err != nil { if !errors.Is(err, ErrConflict) { tr.logf("%s: %v", key, err) } return } cs := &Case{Key: key, Group: key, Pulls: []Pull{{Repository: t.Repository, Head: t.Head, Trunk: t.Trunk}}, Trunk: map[string]string{t.Repository: t.Trunk}, Queued: t.Asked, Progress: &Progress{}, Try: &TryOptions{Level: t.Level, Run: t.Tests}} out, judged := judgeWithin(ctx, tr.Judge, cs, tr.Bound) if ctx.Err() != nil { return // left running: the checker takes it over when it starts } v := out.Verdict t.Ended = tr.now().UTC() t.Took = t.Ended.Sub(t.Started).Round(time.Second).String() t.Levels, t.Machines = v.Levels, v.Machines t.Said = Report(v, cs, cs.Pulls[0], out.Builds, out.Slowest, out.Judged) for _, n := range cs.Notes() { t.Said += " | " + n } switch { case !judged || IsNodesFailure(v): t.State = TryError case v.State == delivery.VerdictPass: t.State = TryPass default: t.State = TryFail if f := firstFailed(v); f != "" { t.Failed = []string{f} } } if t.Tests != "" { t.Said += " | only the tests " + t.Tests + " (Go); nothing kept" } if data, err = encodeTry(t); err == nil { _, err = tr.Tries.Update(ctx, key, data, rev) } if err != nil { tr.logf("%s: its end could not be written: %v", key, err) return } tr.logf("%s: %s", key, t.State) } // judgeWithin runs a case within the bound (none when 0: the asks' steps are bounded from their start, ADR 0525 §6). A // judge that fails to judge, and a check that ran past its bound, are verdicts of failure naming the machine and the step, // and judged says false for them. func judgeWithin(ctx context.Context, j Judge, cs *Case, bound time.Duration) (Outcome, bool) { run, cancel := context.WithCancel(ctx) if bound > 0 { run, cancel = context.WithTimeoutCause(ctx, bound, errBound) } defer cancel() out, err := j.Judge(run, cs) stage, node := cs.Progress.Now() if node == "" { node = "checker" } switch { case ctx.Err() != nil: return Outcome{}, false case context.Cause(run) == errBound: if stage == "" { stage = "starting" } return Outcome{Verdict: failure(node, fmt.Sprintf("the check ran past its bound of %s, stuck at: %s", bound, stage))}, false case err != nil: return Outcome{Verdict: failure(node, err.Error())}, false } return out, true }