package check import ( "context" "errors" "fmt" "strings" "sync" "time" "git.novox.be/novox/mesh-sdk/go/delivery" ) // DefaultBound is how long a check may run before it is ended failed naming the step it was stuck at (ADR 0484, issue 1678): // none since R4g. A check's asks wait in the build seats' streams as long as the backlog is (ADR 0525 §9), and each ask's // step is bounded from its start on the node that took it (buildrun.Asker.Bound, DefaultStepBound); a bound set by // MESH_CHECKER_BOUND still ends a check that ran past it. const DefaultBound time.Duration = 0 // UnheardWords end the Words of a failure because a part of the mesh the check reads was not heard (the forge, issue 1624): // like a failure of the nodes, it is not the change's, and is judged again when the nodes change. const UnheardWords = "not heard; judged again when the nodes change" // The events the checker says on its stream checks (module.json): a preview or a status was written, so a part that reads // the bucket, which is made when the checker is assigned and may not exist when that part starts, finds it (ADR 0437 §8: no // timer). The body is {"key": ...}. const ( EventPreviewWritten = "preview.written" EventOwedWritten = "owed.written" ) // Announcer says an event of the checker's stream, with the key written. type Announcer func(ctx context.Context, event, key string) // Outcome is what judging a case gives: the verdict (state, failure, levels, times, composition; the checker adds the // group, the heads and the trunk) and, for the line, what the check knows of the builds of the modules it touches. type Outcome struct { Verdict delivery.Verdict Builds []BuildNote // Slowest names the slowest tests of a level that went over its bound ("unit pkg.Test 1.2s"). Slowest []string // Judged says, by pull request (pullKey), which program judged a change to a build module (ADR 0535 §5); a pull request // that changes none is not named. Judged map[string]string } // BuildNote is the state of a candidate's module build, by module: built, failed or waiting (ADR 0484 clause 7). type BuildNote struct { Module string State string } // Judge runs the check of a case: the merge, the levels. It writes what it is doing to c.Progress. It answers an error // only when it could not judge at all; a failure of the change is a Verdict. type Judge interface { Judge(ctx context.Context, c *Case) (Outcome, error) } // Checker judges the candidates the indexer writes, dark (ADR 0453 clause 4, ADR 0484): for each candidate it claims a // verdict key, runs the check through its Judge, writes the verdict and, for each pull request judged, the owed status // `mesh/check` the forge bridge performs. It holds no timer; a check starts from a candidate or from a change of the // nodes, the trunk or the controller's plan, and ends when its judge does or its bound passes. type Checker struct { Verdicts KV Owed KV Judge Judge Trunks interface{ Heads() map[string]string } // Bound is how long one check may run; none when 0 (DefaultBound). Bound time.Duration // Link is the base of the address a status points at; the verdict key follows. Link string // Announce says a status was written (EventOwedWritten); nil says nothing. Announce Announcer Now func() time.Time Logf func(format string, args ...any) once sync.Once wg sync.WaitGroup mu sync.Mutex known map[string]delivery.Candidate // by candidates key inflight map[string]bool // by verdict key again map[string]bool // by verdict key: asked again while it ran tried map[string]bool // by verdict key: looked at since this process started epoch uint64 // changes of the nodes seen since this process started nodesFail map[string]uint64 // by verdict key: the epoch a failure of the nodes was written at } func (ch *Checker) init() { ch.once.Do(func() { ch.known, ch.inflight, ch.again, ch.tried = map[string]delivery.Candidate{}, map[string]bool{}, map[string]bool{}, map[string]bool{} ch.nodesFail = map[string]uint64{} }) } func (ch *Checker) logf(format string, args ...any) { if ch.Logf != nil { ch.Logf(format, args...) } } func (ch *Checker) now() time.Time { if ch.Now != nil { return ch.Now() } return time.Now() } func (ch *Checker) bound() time.Duration { if ch.Bound > 0 { return ch.Bound } return DefaultBound } // Wait returns when every check started so far has ended. func (ch *Checker) Wait() { ch.wg.Wait() } // Candidate takes one value of the indexer's candidates bucket in (the key is the group) and checks it if it is to be // checked. A value that is not a candidate is said and left. func (ch *Checker) Candidate(ctx context.Context, key string, value []byte) { ch.init() c, err := delivery.DecodeCandidate(value) if err != nil { ch.logf("candidates.%s holds what is not a candidate, left: %v", key, err) return } ch.mu.Lock() ch.known[key] = c ch.mu.Unlock() ch.consider(ctx, key, c) } // Forget takes a candidate out (its key was deleted). func (ch *Checker) Forget(key string) { ch.init() ch.mu.Lock() delete(ch.known, key) ch.mu.Unlock() } // NodesChanged is Reconsider after a change of the nodes: one came online or went, a build module became healthy or // stopped being, or a toolchain moved. A check that failed because no node qualified is run again only after such a change. func (ch *Checker) NodesChanged(ctx context.Context) { ch.init() ch.mu.Lock() ch.epoch++ ch.mu.Unlock() ch.Reconsider(ctx) } // Reconsider looks at every candidate again: the trunk moved, the nodes changed or a delivery ended. A candidate whose // verdict stands for the trunk it was judged against is left. func (ch *Checker) Reconsider(ctx context.Context) { ch.init() ch.mu.Lock() known := make(map[string]delivery.Candidate, len(ch.known)) for k, c := range ch.known { known[k] = c } ch.mu.Unlock() for k, c := range known { ch.consider(ctx, k, c) } } func (ch *Checker) heads() map[string]string { if ch.Trunks == nil { return nil } return ch.Trunks.Heads() } func (ch *Checker) consider(ctx context.Context, key string, c delivery.Candidate) { cs := CaseOf(c, ch.heads()) if cs == nil { return // nothing to judge: no trunk known } cs.Queued = ch.now() ch.mu.Lock() if ch.inflight[cs.Key] { ch.again[cs.Key] = true // looked at once more when the check ends, so a change of the trunk is never lost ch.mu.Unlock() return } ch.inflight[cs.Key] = true ch.mu.Unlock() ch.wg.Add(1) go func() { defer ch.wg.Done() for { // Every check is asked at once: the build seats' streams hold the backlog, a new head's check and a check after a // failure of the nodes on check, a re-judge after a trunk move on recheck, pulled after them (ADR 0525 §4, §9; // ADR 0536 §4). The checker holds no place and no capacity of its own. cs.Kind = ch.kindOf(ctx, cs) ch.try(ctx, key, cs) ch.mu.Lock() more := ch.again[cs.Key] delete(ch.again, cs.Key) if !more || ctx.Err() != nil { delete(ch.inflight, cs.Key) ch.mu.Unlock() return } ch.mu.Unlock() if next := CaseOf(ch.candidate(key, c), ch.heads()); next != nil { cs = next } cs.Queued = ch.now() } }() } // The kinds of a check (ADR 0536 §4): a head never judged and a head asked again after a failure of the nodes are asked on // the build seats' check stream, a head asked again only because its trunk moved on the recheck stream, pulled after. const ( KindNew = "new" KindNodes = "nodes" KindTrunk = "trunk" ) // kindOf is the kind of a case's check, read from its verdict key: none (or one left running) is a head never judged; a // failure of the nodes is asked again after the nodes changed; a verdict on other trunk heads is a re-judge after a trunk // move. A verdict that stands is ranked with the re-judges: its claim finds nothing to do. func (ch *Checker) kindOf(ctx context.Context, cs *Case) string { value, _, found, err := ch.Verdicts.Get(ctx, cs.Key) if err != nil || !found { return KindNew } v, err := delivery.DecodeVerdict(value) switch { case err != nil || v.State == delivery.VerdictRunning: return KindNew case IsNodesFailure(v): return KindNodes } return KindTrunk } func (ch *Checker) candidate(key string, fallback delivery.Candidate) delivery.Candidate { ch.mu.Lock() defer ch.mu.Unlock() if c, ok := ch.known[key]; ok { return c } return fallback } // claimed is a verdict key the checker holds: the revision the next write is made at. type claimed struct{ rev uint64 } // claim takes the check of the case, or answers nil when it is not this checker's to run now: // - no key: it is created running; a create that finds the key made meanwhile gets nothing; // - a key left running is taken over as it stands when this process looks at it for the first time (it was left by the // checker that stopped; there is one instance, as there is one builder per node, ADR 0478 clause 6), and left otherwise; // - a pass or a failure for the trunk heads the case has stands; one for other trunk heads (the trunk moved) is taken // again by compare-and-set on the same key, and so is a failure of the nodes after the nodes changed. func (ch *Checker) claim(ctx context.Context, cs *Case, first bool, epoch uint64) (*claimed, error) { running := delivery.Verdict{Group: cs.Group, Heads: cs.Heads, Trunk: cs.Trunk, State: delivery.VerdictRunning} data, err := delivery.EncodeVerdict(running) if err != nil { return nil, err } value, rev, found, err := ch.Verdicts.Get(ctx, cs.Key) if err != nil { return nil, fmt.Errorf("%s: %w", cs.Key, err) } if !found { rev, err := ch.Verdicts.Create(ctx, cs.Key, data) if errors.Is(err, ErrConflict) { return nil, nil // another checker made it first } if err != nil { return nil, fmt.Errorf("%s: %w", cs.Key, err) } return &claimed{rev}, nil } cur, err := delivery.DecodeVerdict(value) if err != nil { return nil, fmt.Errorf("%s holds what is not a verdict: %w", cs.Key, err) } retake := false switch { case cur.State == delivery.VerdictRunning: if first { return &claimed{rev}, nil } return nil, nil case !sameTrunk(cur.Trunk, cs.Trunk): retake = true case cur.State == delivery.VerdictFail && IsNodesFailure(cur): ch.mu.Lock() at, ok := ch.nodesFail[cs.Key] ch.mu.Unlock() retake = !ok || epoch > at } if !retake { return nil, nil } rev, err = ch.Verdicts.Update(ctx, cs.Key, data, rev) if errors.Is(err, ErrConflict) { return nil, nil } if err != nil { return nil, fmt.Errorf("%s: %w", cs.Key, err) } return &claimed{rev}, nil } // IsNodesFailure says whether a verdict failed because a part of the mesh the check reads was not heard (UnheardWords): // it is judged again when the nodes change. No check fails for want of a build node any more: its asks wait in the build // seats' streams (ADR 0525 §6, issue 1912). func IsNodesFailure(v delivery.Verdict) bool { return v.State == delivery.VerdictFail && strings.Contains(v.Failed.Words, UnheardWords) } // try claims the check, judges it within its bound and writes the verdict and the owed statuses. A check that ends // because ctx did is left running: the checker takes it again when it starts. func (ch *Checker) try(ctx context.Context, key string, cs *Case) { ch.mu.Lock() first := !ch.tried[cs.Key] ch.tried[cs.Key] = true epoch := ch.epoch ch.mu.Unlock() cl, err := ch.claim(ctx, cs, first, epoch) if err != nil { ch.logf("%s: %v", cs.Key, err) return } if cl == nil { return } began := ch.now() ch.writeOwed(ctx, cs, delivery.Verdict{State: delivery.VerdictRunning}, Outcome{}) out := ch.judge(ctx, cs) if ctx.Err() != nil { return } v := out.Verdict v.Group, v.Heads, v.Trunk = cs.Group, cs.Heads, cs.Trunk if v.Took.Queue == 0 && !cs.Queued.IsZero() && began.After(cs.Queued) { v.Took.Queue = began.Sub(cs.Queued).Milliseconds() } data, err := delivery.EncodeVerdict(v) if err != nil { ch.logf("%s: the verdict cannot be written: %v", cs.Key, err) v = delivery.Verdict{Group: cs.Group, Heads: cs.Heads, Trunk: cs.Trunk, State: delivery.VerdictFail, Failed: delivery.FailedFirst{Machine: "checker", Words: "the verdict could not be written: " + err.Error()}} if data, err = delivery.EncodeVerdict(v); err != nil { return } } if _, err := ch.Verdicts.Update(ctx, cs.Key, data, cl.rev); err != nil { ch.logf("%s: the verdict could not be written: %v", cs.Key, err) return } if IsNodesFailure(v) { ch.mu.Lock() ch.nodesFail[cs.Key] = epoch ch.mu.Unlock() } if ch.movedSince(cs) { // Judged on a trunk that moved while it ran: the verdict is kept as judged, and the status stays pending until the // check of the new trunk, which Reconsider starts, ends. ch.writeOwed(ctx, cs, delivery.Verdict{State: delivery.VerdictRunning}, Outcome{}) ch.logf("%s: %s on a trunk that moved while it ran; the status stays pending", cs.Key, v.State) return } ch.writeOwed(ctx, cs, v, out) ch.logf("%s: %s", cs.Key, v.State) } // judge runs the case within the bound. A judge that fails to judge, and a check that ran past its bound, are verdicts of // failure naming the machine and the step. func (ch *Checker) judge(ctx context.Context, cs *Case) Outcome { out, _ := judgeWithin(ctx, ch.Judge, cs, ch.bound()) return out } var errBound = errors.New("the check ran past its bound") func failure(machine, words string) delivery.Verdict { return delivery.Verdict{State: delivery.VerdictFail, Failed: delivery.FailedFirst{Machine: machine, Words: words}} } // writeOwed writes, for each pull request judged, the status the forge bridge performs: pending while the check runs, success // or failure at its end (ADR 0484 clause 2). A key already holding the same value is not written again, so the bridge // performs one act for each change; an older head's key is never touched. func (ch *Checker) writeOwed(ctx context.Context, cs *Case, v delivery.Verdict, out Outcome) { for _, p := range cs.Pulls { key, err := delivery.CheckerOwedKey(p.Repository, p.Number, p.Head) if err != nil { ch.logf("%s: %v", cs.Key, err) continue } state := map[string]string{delivery.VerdictRunning: "pending", delivery.VerdictPass: "success", delivery.VerdictFail: "failure"}[v.State] o := delivery.Owed{Act: delivery.OwedStatus, Context: delivery.StatusContextCheck, State: state, Target: strings.TrimSuffix(ch.Link, "/") + "/" + cs.Key, Text: Line(v, cs, p, out.Builds, out.Slowest, out.Judged)} data, err := delivery.EncodeCheckerOwed(o) if err != nil { ch.logf("%s: the status of %s cannot be written: %v", cs.Key, key, err) continue } wrote, err := putOnce(ctx, ch.Owed, key, data, func(cur []byte) bool { held, err := delivery.DecodeCheckerOwed(cur) o.V = delivery.Version return err == nil && delivery.SameOwed(held, o) }) if err != nil { ch.logf("%s: the status of %s cannot be written: %v", cs.Key, key, err) } else if wrote && ch.Announce != nil { ch.Announce(ctx, EventOwedWritten, key) } } } // putOnce writes data under key by create, or by compare-and-set when the key exists, and writes nothing when same says the // key holds the value already. func putOnce(ctx context.Context, kv KV, key string, data []byte, same func(cur []byte) bool) (bool, error) { for try := 0; try < 4; try++ { cur, rev, found, err := kv.Get(ctx, key) if err != nil { return false, err } if found && same(cur) { return false, nil } if !found { _, err = kv.Create(ctx, key, data) } else { _, err = kv.Update(ctx, key, data, rev) } if errors.Is(err, ErrConflict) { continue } return err == nil, err } return false, fmt.Errorf("%s changed under every one of 4 tries", key) } // Sweep closes, at start, every verdict left running for a candidate that no longer holds those heads: nothing will check // them again, and the key would stay running for ever (the builder's Sweep, issue 1660). Call it once the candidates have been // read. A key of heads a candidate holds now is not touched: the check takes it over. func (ch *Checker) Sweep(ctx context.Context) error { ch.init() keys, err := ch.Verdicts.Keys(ctx) if err != nil { return err } ch.mu.Lock() current := map[string]bool{} for _, c := range ch.known { if k, err := delivery.CheckerVerdictKey(c.Group, c.HeadsHash); err == nil { current[k] = true } } ch.mu.Unlock() for _, key := range keys { if current[key] { continue } value, rev, found, err := ch.Verdicts.Get(ctx, key) if err != nil || !found { continue } cur, err := delivery.DecodeVerdict(value) if err != nil || cur.State != delivery.VerdictRunning { continue } cur.State = delivery.VerdictFail cur.Failed = delivery.FailedFirst{Machine: "checker", Words: "closed at start: left running, and no candidate holds these heads any more, so nothing will check them"} data, err := delivery.EncodeVerdict(cur) if err != nil { return err } if _, err := ch.Verdicts.Update(ctx, key, data, rev); err != nil && !errors.Is(err, ErrConflict) { return fmt.Errorf("%s: %w", key, err) } ch.logf("%s: closed, left running for heads no candidate holds", key) } return nil } // TrunkMoved is Reconsider after a move of a trunk, which first owes mesh/check pending on every open pull request of a // candidate whose verdict was judged on other trunk heads, every member's for a group (the lead's decision L4, the checker's // part of issue 1302): the forge's success of the old trunk never stands while the check waits for a place or for a delivery // that holds it, so the gate never reads it as the head's. func (ch *Checker) TrunkMoved(ctx context.Context) { ch.init() ch.mu.Lock() known := make(map[string]delivery.Candidate, len(ch.known)) for k, c := range ch.known { known[k] = c } ch.mu.Unlock() heads := ch.heads() for _, c := range known { cs := CaseOf(c, heads) if cs == nil { continue } value, _, found, err := ch.Verdicts.Get(ctx, cs.Key) if err != nil || !found { continue // never judged: no status to put out of date } if v, err := delivery.DecodeVerdict(value); err == nil && sameTrunk(v.Trunk, cs.Trunk) { continue } ch.writeOwed(ctx, cs, delivery.Verdict{State: delivery.VerdictRunning}, Outcome{}) } ch.Reconsider(ctx) } // movedSince says whether a trunk the case was judged on is no longer the trunk's head. func (ch *Checker) movedSince(cs *Case) bool { now := ch.heads() for repo, head := range cs.Trunk { if now[repo] != head { return true } } return false }