A check that could not run says so, releases its claim and waits for its part to run (milestone 47/G W6.9, issue 1373)
A check that failed for a part of the mesh not heard was judged again at every change of the nodes. It now says could not run, naming the part, and is kept in the checker's own state could-not-run; only the running of that part asks it again, once per heads and part at one fingerprint (PartRunning). Not asked again within an hour it is the finding check-could-not-run, which clears when the heads are checked again, replaced, or their pull request closes. checker_status reads the state. The node-engine seat's running event is wired once its definition lands. Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ Builds: design 47 G §253, §315
This commit is contained in:
@@ -361,9 +361,10 @@ func Tools(r check.Reader) []stdio.Tool {
|
||||
"and the candidates whose check is owed by kind (waiting_by_kind: new, nodes, and trunk, a re-judge pulled after the other two). A key a " +
|
||||
"build module left running (started past the step bound, or its build module not running on its node) is listed apart under " +
|
||||
"left_running with why, and not counted. Nothing holds the checker and it has no capacity of its own: a check is an ask on the " +
|
||||
"build seats' streams, whose backlog the delivery view shows (novox/hq ADR 0525). held_by_turn names the candidates whose check an approved " +
|
||||
"merge's turn holds off (novox/hq ADR 0548 §3), with the repository and the merge that waits; the gate's turns tool reads every turn and its " +
|
||||
"hand-now tool steers one. dead_letters says whether the checker hears the asks the build seats give up on (issue 734), and why not: " +
|
||||
"build seats' streams, whose backlog the delivery view shows (novox/hq ADR 0525). No merge holds a check off (novox/hq ADR 0661). could_not_run names each check " +
|
||||
"that could not run because a part of the mesh was not heard: the part, since when, the running that asked it again, and whether it " +
|
||||
"is the finding check-could-not-run (an hour unasked); it is asked again when that part runs. " +
|
||||
"dead_letters says whether the checker hears the asks the build seats give up on (issue 734), and why not: " +
|
||||
"deaf, a try or check whose ask is given up waits until its bound. builds_not_ended names, of each running check, the builder's builds of " +
|
||||
"its candidate's changed modules that are neither done nor failed: the check waits on them before it composes, within its build-wait-bound " +
|
||||
"(novox/hq ADR 0603 §10); a failed one fails the check: rebuild it with the builder's builder_build_again, and the check asks itself again when the build is done. Read-only.",
|
||||
@@ -422,8 +423,8 @@ func main() {
|
||||
PreviewState: read(stdio.State("previews")), InputState: read(stdio.State(check.InputsStateName)),
|
||||
StepStates: map[string]check.StateRead{"mesh-build-go": read(stdio.State("mesh-build-go.steps")), "mesh-build-node": read(stdio.State("mesh-build-node.steps"))},
|
||||
HaveState: read(stdio.State("node-engine.have")), StepBound: stepBound(),
|
||||
HearingState: read(stdio.State(buildrun.HearingState)),
|
||||
AskState: read(stdio.State(check.AsksState)), BuildState: read(stdio.State("mesh-delivery-builder.builds"))}
|
||||
CouldNotRunState: read(stdio.State(check.CouldNotRunState)), HearingState: read(stdio.State(buildrun.HearingState)),
|
||||
AskState: read(stdio.State(check.AsksState)), BuildState: read(stdio.State("mesh-delivery-builder.builds"))}
|
||||
r.Runs = runsOf{ownerRead(stdio.Ask, indexerModulesTool)} // the node-engine at the commit the mesh runs, as the checker reads it
|
||||
tries := stdio.State(check.TriesState)
|
||||
trunks := func() (map[string]string, error) {
|
||||
|
||||
@@ -0,0 +1,26 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
"mesh-delivery-checker/internal/check"
|
||||
)
|
||||
|
||||
// waitCouldNotRun raises each check that could not run as the finding check-could-not-run once its bound passed
|
||||
// (check.CouldNotRunBound): it waits for the next one due, read from the checker's state could-not-run, and reads again
|
||||
// each time one is kept. The bound is the only wait: a check is asked again on its part's running, never here.
|
||||
func waitCouldNotRun(ctx context.Context, ch *check.Checker, kept <-chan struct{}) {
|
||||
for ctx.Err() == nil {
|
||||
// The wait is a context's bound (ADR 0437 §8): none while no check waits for its bound.
|
||||
wait, done := ctx, func() {}
|
||||
if next := ch.CouldNotRunDue(ctx, time.Now()); !next.IsZero() {
|
||||
wait, done = context.WithDeadline(ctx, next)
|
||||
}
|
||||
select {
|
||||
case <-wait.Done():
|
||||
case <-kept:
|
||||
}
|
||||
done()
|
||||
}
|
||||
}
|
||||
@@ -26,7 +26,7 @@ type forgeReads struct {
|
||||
}
|
||||
|
||||
// ask calls a forge tool; a call that reached nobody (no responders: the forge's tool runner re-serving, or the forge
|
||||
// down) is marked Unheard, so the check fails at the forge and is judged again when the nodes change (issue 1624).
|
||||
// down) is marked Unheard, so the check fails at the forge and is asked again when the part it could not hear runs (issue 1624).
|
||||
func (f forgeReads) ask(key string, body any) (json.RawMessage, error) {
|
||||
raw, err := f.call(key, body)
|
||||
if errors.Is(err, nats.ErrNoResponders) || errors.Is(err, nats.ErrTimeout) || errors.Is(err, context.DeadlineExceeded) {
|
||||
|
||||
@@ -1265,7 +1265,7 @@ func TestTheCheckerSaysOwedReadableOnTheBus(t *testing.T) { // R2-1
|
||||
}
|
||||
if said.Module != "mesh-delivery-checker" || said.Node != "n1" || !said.Keeps("owed") || !said.Readable("owed") ||
|
||||
!said.Readable("previews") || !said.Readable("verdicts") || !said.Readable("kept") || !said.Readable("tries") || said.Readable("hearing") ||
|
||||
!said.Keeps("asks") || said.Readable("asks") || !said.Keeps("first-views") || said.Readable("first-views") || !said.Keeps("findings") || !said.Keeps("inputs") || said.Readable("inputs") || !said.Keeps("compositions") || said.Readable("compositions") || len(said.States) != 11 {
|
||||
t.Fatalf("the checker said %+v, want its eleven states, asks, hearing, first-views, inputs and compositions its own (DLV-M2), with kept, owed, previews, tries and verdicts readable, tries by the delivery view (1929/AC4; 1887 A5b; build_usage)", said)
|
||||
!said.Keeps("asks") || said.Readable("asks") || !said.Keeps("first-views") || said.Readable("first-views") || !said.Keeps("findings") || !said.Keeps("inputs") || said.Readable("inputs") || !said.Keeps("compositions") || said.Readable("compositions") || !said.Keeps("could-not-run") || said.Readable("could-not-run") || len(said.States) != 12 {
|
||||
t.Fatalf("the checker said %+v, want its twelve states, asks, hearing, first-views, inputs, compositions and could-not-run its own (DLV-M2; W6.9), with kept, owed, previews, tries and verdicts readable, tries by the delivery view (1929/AC4; 1887 A5b; build_usage)", said)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -116,6 +116,8 @@ var keptStates = []announce.State{
|
||||
// findings of one unanswered past its bound, which the condition keeper observes as it does every reader's (ADR 0615 §6).
|
||||
{Name: views.StatusState},
|
||||
{Name: "findings"},
|
||||
// The checks that could not run (check.CouldNotRunState), each asked again when its part runs: read by checker_status.
|
||||
{Name: check.CouldNotRunState},
|
||||
}
|
||||
|
||||
func run(ctx context.Context, getenv func(string) string, logOut io.Writer) error {
|
||||
@@ -224,17 +226,29 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
waiter := buildrun.LostAskWaiter{Inner: heard, Dead: dead, Hearing: hearing}
|
||||
asksKV := process.AwaitBucket(ctx, js, conn, check.AsksBucket, "checker_ask_again refuses every check, as it cannot tell whether its ask waits in a stream", logger.Printf)
|
||||
inputsKV := process.AwaitBucket(ctx, js, conn, check.InputsBucket, "no checking program can be asked, so every check fails naming it", logger.Printf)
|
||||
couldNotRunKV := process.AwaitBucket(ctx, js, conn, check.CouldNotRunBucket, "a check that could not run is not asked again when its part runs", logger.Printf)
|
||||
requestsKV := process.AwaitBucket(ctx, js, conn, check.RequestsBucket, "no composition request is kept, so every composition made with the node-engine's head fails naming it", logger.Printf)
|
||||
ch := &check.Checker{
|
||||
Asks: awaitedKV{asksKV},
|
||||
Verdicts: kvStore{buckets[check.CheckerVerdictsBucket]},
|
||||
Owed: kvStore{buckets[check.CheckerOwedBucket]},
|
||||
Trunks: trunks,
|
||||
Bound: checkBound,
|
||||
Link: link,
|
||||
Logf: logger.Printf,
|
||||
Settle: settleWindow,
|
||||
// A check that could not run waits for its part's running, and is the finding check-could-not-run after its bound.
|
||||
CouldNotRun: awaitedKV{couldNotRunKV},
|
||||
Findings: sdkfindings.Writer(ctx, findings.KV),
|
||||
Trunks: trunks,
|
||||
Bound: checkBound,
|
||||
Link: link,
|
||||
Logf: logger.Printf,
|
||||
Settle: settleWindow,
|
||||
}
|
||||
kept := make(chan struct{}, 1)
|
||||
ch.CouldNotRunKept = func() {
|
||||
select {
|
||||
case kept <- struct{}{}:
|
||||
default:
|
||||
}
|
||||
}
|
||||
go waitCouldNotRun(ctx, ch, kept)
|
||||
// The builder's builds say how the candidates' fingerprints stand (issue 1684); where its bucket is not there yet the line
|
||||
// says nothing of the builds, until the checker starts again.
|
||||
var buildStates check.BuildStates
|
||||
|
||||
@@ -31,6 +31,8 @@ type Pull struct {
|
||||
// Case is one check: a candidate's pull requests that are judged (those of a repository whose trunk is not known are left
|
||||
// out; hq is judged as any repository since R4g, its checks Go tests: ADR 0527), the trunk head of each repository and the key of the verdict.
|
||||
type Case struct {
|
||||
// AskedAt is the running that asked again a check of these heads that could not run (PartRunning), "" otherwise.
|
||||
AskedAt string
|
||||
// Key is the verdict key, `<group>.<heads hash>`.
|
||||
Key string
|
||||
Group string
|
||||
|
||||
@@ -20,6 +20,7 @@ import (
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun"
|
||||
"git.novox.be/novox/mesh-delivery/sdk/buildstep"
|
||||
"git.novox.be/novox/mesh-sdk/go/keys"
|
||||
"git.novox.be/novox/mesh-sdk/go/views"
|
||||
)
|
||||
|
||||
// DefaultBound is how long a check may run before it is ended failed naming the step it was stuck at (ADR 0484, issue 1678):
|
||||
@@ -29,8 +30,8 @@ import (
|
||||
const DefaultBound time.Duration = 0
|
||||
|
||||
// UnheardWords end the Words of a failure because a part of the mesh the check reads was not heard (the forge, issue 1624):
|
||||
// like a failure of the nodes, it is not the change's, and is judged again when the nodes change.
|
||||
const UnheardWords = "not heard; judged again when the nodes change"
|
||||
// like a failure of the nodes, it is not the change's, and is asked again when the part it could not hear runs.
|
||||
const UnheardWords = "could not run: not heard; asked again when it runs"
|
||||
|
||||
// The events the checker says on its stream checks (module.json): a preview or a status was written, so a part that reads
|
||||
// the bucket, which is made when the checker is assigned and may not exist when that part starts, finds it (ADR 0437 §8: no
|
||||
@@ -59,6 +60,9 @@ type Outcome struct {
|
||||
// OutOfStep are, by pull request (pullKey), the generated files out of step with their seat definitions that its steps
|
||||
// named (design 26): the verdict event carries them.
|
||||
OutOfStep map[string][]string
|
||||
// Unheard is the part of the mesh the check could not hear, when its verdict could not run (IsCouldNotRun): its running
|
||||
// asks the check again (PartRunning). The zero Part is a part the judge could not name.
|
||||
Unheard Part
|
||||
}
|
||||
|
||||
// BuildNote is the state of a candidate's module build, by module: built, failed or waiting (ADR 0484 clause 7).
|
||||
@@ -85,6 +89,14 @@ type Checker struct {
|
||||
// Runs says which commit the mesh runs of a module (the indexer's modules): the node-engine a verdict read beside at its
|
||||
// running commit is out of date when that commit moves (ADR 0548 §6); nil reads every sibling at its trunk.
|
||||
Runs interface{ CommitOf(module string) string }
|
||||
// CouldNotRun is the checker's own state could-not-run (CouldNotRunState): the checks that could not run, each asked
|
||||
// again when the part it could not hear runs (PartRunning); nil keeps none, and none is asked again.
|
||||
CouldNotRun KV
|
||||
// Findings writes the checker's finding of a check not asked again within CouldNotRunBound (CouldNotRunDue); nil
|
||||
// writes none.
|
||||
Findings views.FindingWriter
|
||||
// CouldNotRunKept is called each time a check that could not run is kept, so the process waits for its bound anew.
|
||||
CouldNotRunKept func()
|
||||
// Bound is how long one check may run; none when 0 (DefaultBound).
|
||||
Bound time.Duration
|
||||
// Rebuilds hears the builder's build keys a failed verdict names (failedBuilds): once each is written done, the check of
|
||||
@@ -105,19 +117,18 @@ type Checker struct {
|
||||
// away at the next. None when 0; DefaultSettle in the module.
|
||||
Settle time.Duration
|
||||
|
||||
once sync.Once
|
||||
wg sync.WaitGroup
|
||||
mu sync.Mutex
|
||||
known map[string]meshcatalog.CandidatesByGroupValue // by candidates key
|
||||
inflight map[string]bool // by verdict key
|
||||
again map[string]bool // by verdict key: asked again while it ran
|
||||
tried map[string]bool // by verdict key: looked at since this process started
|
||||
epoch uint64 // changes of the nodes seen since this process started
|
||||
nodesFail map[string]uint64 // by verdict key: the epoch a failure of the nodes was written at
|
||||
live map[string]liveCheck // by verdict key: the check running in this process, ended when its heads go
|
||||
settling map[string]*window // by candidates key: the window its heads settle in
|
||||
rebuilt map[string]bool // by verdict key: every failed build its verdict names is done again
|
||||
awaiting map[string]bool // by verdict key: its failed builds are heard (awaitRebuilt)
|
||||
once sync.Once
|
||||
wg sync.WaitGroup
|
||||
mu sync.Mutex
|
||||
known map[string]meshcatalog.CandidatesByGroupValue // by candidates key
|
||||
inflight map[string]bool // by verdict key
|
||||
again map[string]bool // by verdict key: asked again while it ran
|
||||
tried map[string]bool // by verdict key: looked at since this process started
|
||||
release map[string]string // by verdict key: the running that released a check that could not run
|
||||
live map[string]liveCheck // by verdict key: the check running in this process, ended when its heads go
|
||||
settling map[string]*window // by candidates key: the window its heads settle in
|
||||
rebuilt map[string]bool // by verdict key: every failed build its verdict names is done again
|
||||
awaiting map[string]bool // by verdict key: its failed builds are heard (awaitRebuilt)
|
||||
// outsideSaid are the candidates outside the mesh said in the log, by group and heads.
|
||||
outsideSaid map[string]bool
|
||||
}
|
||||
@@ -177,7 +188,7 @@ func (s superseded) Unwrap() error { return buildrun.Withdrawn{Why: s.Error()} }
|
||||
func (ch *Checker) init() {
|
||||
ch.once.Do(func() {
|
||||
ch.known, ch.inflight, ch.again, ch.tried = map[string]meshcatalog.CandidatesByGroupValue{}, map[string]bool{}, map[string]bool{}, map[string]bool{}
|
||||
ch.nodesFail = map[string]uint64{}
|
||||
ch.release = map[string]string{}
|
||||
ch.live, ch.settling = map[string]liveCheck{}, map[string]*window{}
|
||||
ch.rebuilt, ch.awaiting = map[string]bool{}, map[string]bool{}
|
||||
ch.outsideSaid = map[string]bool{}
|
||||
@@ -237,6 +248,11 @@ func (ch *Checker) Candidate(ctx context.Context, key string, value []byte) {
|
||||
// or leaving.
|
||||
ch.endOthers(key, vkey, GateHeads(c))
|
||||
moved := !had || prev.Heads != c.Heads
|
||||
if had && moved {
|
||||
if old, err := meshdeliverycheckerclient.VerdictKey(prev.Group, prev.Heads); err == nil && old != vkey {
|
||||
defer ch.clearCouldNotRun(ctx, old) // its heads were replaced, after the lock
|
||||
}
|
||||
}
|
||||
if ch.Settle > 0 && (moved || ch.settling[key] != nil) {
|
||||
// The group's heads changed: its check waits until they have been the same for Settle, so a group whose members
|
||||
// are announced one after another is checked once, whole. A later change starts the window again. The wait is a
|
||||
@@ -308,7 +324,13 @@ func (ch *Checker) VerdictDeleted(ctx context.Context, key string) {
|
||||
func (ch *Checker) Forget(key string) {
|
||||
ch.init()
|
||||
ch.mu.Lock()
|
||||
gone, had := ch.known[key]
|
||||
delete(ch.known, key)
|
||||
if had {
|
||||
if vkey, err := meshdeliverycheckerclient.VerdictKey(gone.Group, gone.Heads); err == nil {
|
||||
defer ch.clearCouldNotRun(context.Background(), vkey) // its pull request closed, after the lock
|
||||
}
|
||||
}
|
||||
if w := ch.settling[key]; w != nil {
|
||||
w.abort(errUnsettled)
|
||||
delete(ch.settling, key)
|
||||
@@ -318,12 +340,10 @@ func (ch *Checker) Forget(key string) {
|
||||
}
|
||||
|
||||
// NodesChanged is Reconsider after a change of the nodes: one came online or went, a build module became healthy or
|
||||
// stopped being, or a toolchain moved. A check that failed because no node qualified is run again only after such a change.
|
||||
// stopped being, or a toolchain moved. A check that could not run is not asked again by it, only by the running of the
|
||||
// part it could not hear (PartRunning).
|
||||
func (ch *Checker) NodesChanged(ctx context.Context) {
|
||||
ch.init()
|
||||
ch.mu.Lock()
|
||||
ch.epoch++
|
||||
ch.mu.Unlock()
|
||||
ch.Reconsider(ctx)
|
||||
}
|
||||
|
||||
@@ -550,7 +570,7 @@ func (ch *Checker) prepare(ctx context.Context, group string, cs *Case) {
|
||||
switch {
|
||||
case err != nil || v.State == meshdeliverycheckerclient.VerdictRunning:
|
||||
return
|
||||
case IsNodesFailure(v):
|
||||
case IsCouldNotRun(v):
|
||||
cs.Kind = KindNodes
|
||||
return
|
||||
case failedBuilds(v, cs) != nil:
|
||||
@@ -677,7 +697,7 @@ type claimed struct{ rev uint64 }
|
||||
// checker that stopped; there is one instance, as there is one builder per node, ADR 0478 clause 6), and left otherwise;
|
||||
// - a pass or a failure for the trunk heads the case has stands; one for other trunk heads (the trunk moved) is taken
|
||||
// again by compare-and-set on the same key, and so is a failure of the nodes after the nodes changed.
|
||||
func (ch *Checker) claim(ctx context.Context, cs *Case, first bool, epoch uint64) (*claimed, error) {
|
||||
func (ch *Checker) claim(ctx context.Context, cs *Case, first bool) (*claimed, error) {
|
||||
running := meshdeliverychecker.VerdictsByGroupHeadsValue{Group: cs.Group, Heads: cs.Heads, Trunk: cs.Trunk, State: meshdeliverycheckerclient.VerdictRunning}
|
||||
data, err := meshdeliverycheckerclient.EncodeVerdict(running)
|
||||
if err != nil {
|
||||
@@ -712,11 +732,9 @@ func (ch *Checker) claim(ctx context.Context, cs *Case, first bool, epoch uint64
|
||||
retake = true // heads held again after their check was ended unjudged (issue 1890), or a trunk that moved
|
||||
case cur.State == meshdeliverycheckerclient.VerdictFail && ch.takeRebuilt(cs.Key):
|
||||
retake = true // every failed build it names is done again (ADR 0603 §10)
|
||||
case cur.State == meshdeliverycheckerclient.VerdictFail && IsNodesFailure(cur):
|
||||
ch.mu.Lock()
|
||||
at, ok := ch.nodesFail[cs.Key]
|
||||
ch.mu.Unlock()
|
||||
retake = !ok || epoch > at
|
||||
case IsCouldNotRun(cur):
|
||||
// Its claim was released when it could not run: only the running of the part it could not hear asks it again.
|
||||
cs.AskedAt, retake = ch.takeRelease(cs.Key)
|
||||
}
|
||||
if !retake {
|
||||
return nil, nil
|
||||
@@ -740,13 +758,6 @@ func IsConflictFailure(v meshdeliverychecker.VerdictsByGroupHeadsValue) bool {
|
||||
return v.State == meshdeliverycheckerclient.VerdictFail && strings.Contains(failedOf(v).Words, ConflictWords)
|
||||
}
|
||||
|
||||
// IsNodesFailure says whether a verdict failed because a part of the mesh the check reads was not heard (UnheardWords):
|
||||
// it is judged again when the nodes change. No check fails for want of a build node any more: its asks wait in the build
|
||||
// seats' streams (ADR 0525 §6, issue 1912).
|
||||
func IsNodesFailure(v meshdeliverychecker.VerdictsByGroupHeadsValue) bool {
|
||||
return v.State == meshdeliverycheckerclient.VerdictFail && strings.Contains(failedOf(v).Words, UnheardWords)
|
||||
}
|
||||
|
||||
// try claims the check, judges it within its bound under run and writes the verdict and the owed statuses. A check that ends
|
||||
// because ctx did is left running: the checker takes it again when it starts. A check whose run was ended because no
|
||||
// candidate holds its heads any more (issue 1890) is ended: its verdict says so, and no status is written.
|
||||
@@ -757,9 +768,8 @@ func (ch *Checker) try(ctx, run context.Context, key string, cs *Case) {
|
||||
ch.mu.Lock()
|
||||
first := !ch.tried[cs.Key]
|
||||
ch.tried[cs.Key] = true
|
||||
epoch := ch.epoch
|
||||
ch.mu.Unlock()
|
||||
cl, err := ch.claim(ctx, cs, first, epoch)
|
||||
cl, err := ch.claim(ctx, cs, first)
|
||||
if err != nil {
|
||||
ch.logf("%s: %v", cs.Key, err)
|
||||
return
|
||||
@@ -767,6 +777,7 @@ func (ch *Checker) try(ctx, run context.Context, key string, cs *Case) {
|
||||
if cl == nil {
|
||||
return
|
||||
}
|
||||
ch.clearCouldNotRun(ctx, cs.Key) // checked again: a record that it could not run, and its finding, go
|
||||
began := ch.now()
|
||||
if cs.Watch == nil && ch.Asks != nil {
|
||||
cs.Watch = func(_ string, step *buildstep.Step) {
|
||||
@@ -780,7 +791,7 @@ func (ch *Checker) try(ctx, run context.Context, key string, cs *Case) {
|
||||
out := ch.judge(run, cs)
|
||||
// An ask the build seat gave up on, or whose step the build module's next process ended because it was replaced under
|
||||
// it, judged nothing (issue 734, IsLostAsk): the check is asked again under the same claim, its status pending, and after
|
||||
// LostAskTries it fails as not heard, judged again when the nodes change. A step that ended while its watch was deaf is
|
||||
// LostAskTries it fails as not heard, asked again when the part it could not hear runs. A step that ended while its watch was deaf is
|
||||
// read at the next nudge (HeardEnds) and judged.
|
||||
for try := 1; IsLostAsk(out.Verdict) && run.Err() == nil; try++ {
|
||||
if try >= LostAskTries {
|
||||
@@ -844,10 +855,8 @@ func (ch *Checker) try(ctx, run context.Context, key string, cs *Case) {
|
||||
return
|
||||
}
|
||||
}
|
||||
if IsNodesFailure(v) {
|
||||
ch.mu.Lock()
|
||||
ch.nodesFail[cs.Key] = epoch
|
||||
ch.mu.Unlock()
|
||||
if IsCouldNotRun(v) {
|
||||
ch.keepCouldNotRun(ctx, cs, v, out.Unheard, cs.AskedAt)
|
||||
}
|
||||
if ch.movedSince(cs) || besideMoved(v.Beside, ch.besideNow()) {
|
||||
// Judged on a trunk that moved while it ran: the verdict is kept as judged, and the status stays pending until the
|
||||
@@ -1113,3 +1122,6 @@ func (ch *Checker) takeRebuilt(vkey string) bool {
|
||||
delete(ch.rebuilt, vkey)
|
||||
return was
|
||||
}
|
||||
|
||||
// containsWords says whether words hold the given words.
|
||||
func containsWords(words, part string) bool { return strings.Contains(words, part) }
|
||||
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"git.novox.be/novox/mesh-sdk/go/views"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
@@ -226,33 +227,107 @@ func TestACheckPastItsBoundFailsNamingTheStuckStep(t *testing.T) { // N6/AC3
|
||||
}
|
||||
}
|
||||
|
||||
// A check whose forge was not heard is judged again when the nodes change (issue 1912: no check fails for want of a build
|
||||
// node any more; its asks wait in the streams).
|
||||
func TestAnUnheardForgeFailsWithTheReasonAndIsJudgedAgainWhenTheNodesChange(t *testing.T) { // N6/AC4
|
||||
// findingsKept is a FindingWriter that keeps what is written, by ID.
|
||||
type findingsKept map[string]views.Finding
|
||||
|
||||
func (f findingsKept) Put(x views.Finding) error { f[x.ID] = x; return nil }
|
||||
func (f findingsKept) Delete(x views.Finding) error { delete(f, x.ID); return nil }
|
||||
|
||||
// A check that could not hear a part of the mesh says `could not run`, naming the part, and releases its claim; only the
|
||||
// running of that part asks it again, once per heads and part at one fingerprint; not asked again within an hour it is the
|
||||
// finding check-could-not-run, which clears when the heads are checked again, replaced, or their pull request closes
|
||||
// (design 47 test 58, issue 1373; issue 1912: no check fails for want of a build node).
|
||||
func TestACheckThatCouldNotRunIsAskedAgainWhenItsPartRuns(t *testing.T) { // N6/AC4, 47/58
|
||||
ctx := context.Background()
|
||||
j := &judgeFunc{f: func(_ context.Context, _ *Case, n int) (Outcome, error) {
|
||||
if n == 1 {
|
||||
return Outcome{Verdict: failure("the forge", "novox/mesh-delivery#7: go.mod cannot be read: no responders (the forge was "+UnheardWords+")")}, nil
|
||||
}
|
||||
return Outcome{Verdict: passing()}, nil
|
||||
}}
|
||||
r := newRig(t, j)
|
||||
unheard := Part{Node: "g14", Module: "forge"}
|
||||
newCase := func(t *testing.T, fails int) (*rig, *judgeFunc, findingsKept, *time.Time) {
|
||||
j := &judgeFunc{f: func(_ context.Context, _ *Case, n int) (Outcome, error) {
|
||||
if n <= fails {
|
||||
return Outcome{Verdict: failure("the forge", "novox/mesh-delivery#7: go.mod cannot be read: no responders (the forge was "+UnheardWords+")"), Unheard: unheard}, nil
|
||||
}
|
||||
return Outcome{Verdict: passing()}, nil
|
||||
}}
|
||||
r := newRig(t, j)
|
||||
found, now := findingsKept{}, time.Date(2026, 11, 9, 20, 0, 0, 0, time.UTC)
|
||||
r.ch.CouldNotRun, r.ch.Findings, r.ch.Now = newMemKV(), found, func() time.Time { return now }
|
||||
return r, j, found, &now
|
||||
}
|
||||
cand := candidateOf(t, "feat/x", pullOf(repoA, 7, head1))
|
||||
key := keyOf(t, "feat/x", cand)
|
||||
r.ch.Candidate(ctx, "feat%2Fx", cand)
|
||||
r.ch.Wait()
|
||||
if v := r.verdict(t, key); !IsNodesFailure(v) {
|
||||
t.Fatalf("verdict = %+v, want a failure of the nodes", v)
|
||||
}
|
||||
r.ch.Reconsider(ctx) // nothing changed: not judged again
|
||||
r.ch.Wait()
|
||||
if j.count() != 1 {
|
||||
t.Fatalf("judged %d times with the nodes unchanged, want once", j.count())
|
||||
}
|
||||
r.ch.NodesChanged(ctx)
|
||||
r.ch.Wait()
|
||||
if j.count() != 2 || r.verdict(t, key).State != meshdeliverycheckerclient.VerdictPass {
|
||||
t.Fatalf("after the nodes changed: judged %d times, verdict %+v", j.count(), r.verdict(t, key))
|
||||
for name, run := range map[string]func(t *testing.T){
|
||||
"it says could not run, naming the part, and only its running asks it again, once per fingerprint": func(t *testing.T) {
|
||||
r, j, _, _ := newCase(t, 2)
|
||||
r.ch.Candidate(ctx, "feat%2Fx", cand)
|
||||
r.ch.Wait()
|
||||
if v := r.verdict(t, key); !IsCouldNotRun(v) || !strings.Contains(v.Failed.Words, "could not run") {
|
||||
t.Fatalf("verdict = %+v, want could not run", v)
|
||||
}
|
||||
if rec, _, ok := r.ch.couldNotRunOf(ctx, key); !ok || rec.Part != unheard || rec.AskedAt != "" {
|
||||
t.Fatalf("could-not-run.%s = %+v (%v), want the part %v", key, rec, ok, unheard)
|
||||
}
|
||||
r.ch.Reconsider(ctx) // nothing ran: not asked again
|
||||
r.ch.NodesChanged(ctx)
|
||||
r.ch.PartRunning(ctx, "g14", "mail", "src2:1") // another part ran
|
||||
r.ch.Wait()
|
||||
if j.count() != 1 {
|
||||
t.Fatalf("judged %d times before its part ran, want once", j.count())
|
||||
}
|
||||
r.ch.PartRunning(ctx, "g14", "forge", "src2:1")
|
||||
r.ch.Wait()
|
||||
r.ch.PartRunning(ctx, "g14", "forge", "src2:1") // the same part at the same fingerprint asks nothing more
|
||||
r.ch.Wait()
|
||||
if j.count() != 2 {
|
||||
t.Fatalf("judged %d times after its part ran twice at one fingerprint, want twice", j.count())
|
||||
}
|
||||
if rec, _, _ := r.ch.couldNotRunOf(ctx, key); rec.AskedAt != runningAt("g14", "forge", "src2:1") {
|
||||
t.Fatalf("asked-at = %q after it could not run again", rec.AskedAt)
|
||||
}
|
||||
r.ch.PartRunning(ctx, "g14", "forge", "src2:2") // a new build of the part asks once more
|
||||
r.ch.Wait()
|
||||
if j.count() != 3 || r.verdict(t, key).State != meshdeliverycheckerclient.VerdictPass {
|
||||
t.Fatalf("judged %d times, verdict %+v, want a pass on the third", j.count(), r.verdict(t, key))
|
||||
}
|
||||
if _, _, ok := r.ch.couldNotRunOf(ctx, key); ok {
|
||||
t.Error("checked again, the record that it could not run is kept")
|
||||
}
|
||||
},
|
||||
"not asked again within the hour it is a finding, cleared when it is checked again": func(t *testing.T) {
|
||||
r, _, found, now := newCase(t, 1)
|
||||
r.ch.Candidate(ctx, "feat%2Fx", cand)
|
||||
r.ch.Wait()
|
||||
if due := r.ch.CouldNotRunDue(ctx, now.Add(59*time.Minute)); !due.Equal(now.Add(CouldNotRunBound)) || len(found) != 0 {
|
||||
t.Fatalf("before its bound: due %v, findings %v", due, found)
|
||||
}
|
||||
if due := r.ch.CouldNotRunDue(ctx, now.Add(CouldNotRunBound)); !due.IsZero() || found[key].Kind != CouldNotRunKind || !strings.Contains(found[key].Summary, "forge on g14") {
|
||||
t.Fatalf("at its bound: due %v, findings %+v, want %s naming the part", due, found, CouldNotRunKind)
|
||||
}
|
||||
r.ch.PartRunning(ctx, "g14", "forge", "src2:1")
|
||||
r.ch.Wait()
|
||||
if len(found) != 0 {
|
||||
t.Errorf("checked again, the finding stays: %v", found)
|
||||
}
|
||||
},
|
||||
"its heads replaced, or its pull request closed, the record and the finding go": func(t *testing.T) {
|
||||
r, _, found, now := newCase(t, 1)
|
||||
r.ch.Candidate(ctx, "feat%2Fx", cand)
|
||||
r.ch.Wait()
|
||||
r.ch.CouldNotRunDue(ctx, now.Add(CouldNotRunBound))
|
||||
r.ch.Candidate(ctx, "feat%2Fx", candidateOf(t, "feat/x", pullOf(repoA, 7, head2)))
|
||||
r.ch.Wait()
|
||||
if _, _, ok := r.ch.couldNotRunOf(ctx, key); ok || len(found) != 0 {
|
||||
t.Fatalf("heads replaced: record kept %v, findings %v", ok, found)
|
||||
}
|
||||
r2, _, found2, now2 := newCase(t, 1)
|
||||
r2.ch.Candidate(ctx, "feat%2Fx", cand)
|
||||
r2.ch.Wait()
|
||||
r2.ch.CouldNotRunDue(ctx, now2.Add(CouldNotRunBound))
|
||||
r2.ch.Forget("feat%2Fx")
|
||||
if _, _, ok := r2.ch.couldNotRunOf(ctx, key); ok || len(found2) != 0 {
|
||||
t.Fatalf("pull request closed: record kept %v, findings %v", ok, found2)
|
||||
}
|
||||
},
|
||||
} {
|
||||
t.Run(name, run)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,246 @@
|
||||
package check
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sort"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/contracts/go/clients/meshdeliverycheckerclient"
|
||||
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshdeliverychecker"
|
||||
"git.novox.be/novox/mesh-sdk/go/views"
|
||||
)
|
||||
|
||||
// A check that failed because a part of the mesh was not heard says `could not run`, naming the part, and releases its
|
||||
// claim on the verdict key (design 47, *A check runs to its end*; issue 1373). It is asked again once per set of heads and
|
||||
// per part returning, on the one event that can change its outcome: the node-engine seat's `running.<node>.<module>`
|
||||
// naming that part (PartRunning), never on a timer; a `running` of the same part at the same fingerprint after that asks
|
||||
// nothing. One not asked again within CouldNotRunBound becomes the checker's finding CouldNotRunKind, which clears when the
|
||||
// heads are checked again, replaced, or their pull request closes.
|
||||
//
|
||||
// What the checker knows of each such check is its own state could-not-run (CouldNotRunState), one key per verdict key,
|
||||
// read by checker_status (could_not_run): kept on the bus, so a checker started again asks each one again on the next
|
||||
// `running` and raises its finding at the same time.
|
||||
|
||||
// CouldNotRunState is the checker's own state of its checks that could not run, by verdict key.
|
||||
const CouldNotRunState = "could-not-run"
|
||||
|
||||
// CouldNotRunBucket is the bucket of CouldNotRunState.
|
||||
const CouldNotRunBucket = CheckerModule + "_" + CouldNotRunState
|
||||
|
||||
// CouldNotRunKind is the checker's finding of a check that could not run and was not asked again within CouldNotRunBound.
|
||||
const CouldNotRunKind = "check-could-not-run"
|
||||
|
||||
// CouldNotRunBound is how long a check that could not run waits to be asked again before it is a finding: the check's own
|
||||
// bound of an hour.
|
||||
const CouldNotRunBound = time.Hour
|
||||
|
||||
// Part is a part of the mesh a check could not hear: the node it runs on and its module, as the node-engine seat's
|
||||
// `running.<node>.<module>` names it. An empty field is not known: an empty Module is any part on Node, and an empty Node is
|
||||
// Module wherever it runs; a Part with neither is a part the check could not name, asked again on the first `running`.
|
||||
type Part struct {
|
||||
Node string `json:"node,omitempty"`
|
||||
Module string `json:"module,omitempty"`
|
||||
}
|
||||
|
||||
// String names the part as a verdict says it.
|
||||
func (p Part) String() string {
|
||||
switch {
|
||||
case p.Node != "" && p.Module != "":
|
||||
return p.Module + " on " + p.Node
|
||||
case p.Module != "":
|
||||
return p.Module
|
||||
case p.Node != "":
|
||||
return "the node " + p.Node
|
||||
}
|
||||
return "a part of the mesh"
|
||||
}
|
||||
|
||||
// matches says whether a `running` of module on node is the part's return.
|
||||
func (p Part) matches(node, module string) bool {
|
||||
return (p.Node == "" || p.Node == node) && (p.Module == "" || p.Module == module)
|
||||
}
|
||||
|
||||
// CouldNotRun is one check that could not run, as the state could-not-run keeps it.
|
||||
type CouldNotRun struct {
|
||||
V int `json:"v"`
|
||||
Group string `json:"group"`
|
||||
Heads string `json:"heads"`
|
||||
Part Part `json:"part"`
|
||||
Words string `json:"words"`
|
||||
// Since is when its verdict was written; CouldNotRunBound after it, not asked again, it is a finding.
|
||||
Since time.Time `json:"since"`
|
||||
// AskedAt is the `running` that asked the check of these heads again, `<node>.<module>@<fingerprint>`: a `running` of
|
||||
// the same part at the same fingerprint asks nothing more.
|
||||
AskedAt string `json:"asked-at,omitempty"`
|
||||
// Raised says its finding is written.
|
||||
Raised bool `json:"raised,omitempty"`
|
||||
}
|
||||
|
||||
// IsCouldNotRun says whether a verdict failed because a part of the mesh the check reads was not heard (UnheardWords): its
|
||||
// claim is released, and it is asked again when that part runs (PartRunning).
|
||||
func IsCouldNotRun(v meshdeliverychecker.VerdictsByGroupHeadsValue) bool {
|
||||
return v.State == meshdeliverycheckerclient.VerdictFail && containsWords(failedOf(v).Words, UnheardWords)
|
||||
}
|
||||
|
||||
// running is how a `running` event is kept in AskedAt and in the release of a check.
|
||||
func runningAt(node, module, fingerprint string) string {
|
||||
return node + "." + module + "@" + fingerprint
|
||||
}
|
||||
|
||||
// PartRunning is the node-engine seat's `running.<node>.<module>` at fingerprint: every check that could not run for want
|
||||
// of that part, and was not asked again by the same part at the same fingerprint, is released and asked again once.
|
||||
func (ch *Checker) PartRunning(ctx context.Context, node, module, fingerprint string) {
|
||||
ch.init()
|
||||
if ch.CouldNotRun == nil {
|
||||
return
|
||||
}
|
||||
keys, err := ch.CouldNotRun.Keys(ctx)
|
||||
if err != nil {
|
||||
ch.logf("the checks that could not run cannot be listed, so the running of %s on %s asks none again: %v", module, node, err)
|
||||
return
|
||||
}
|
||||
at := runningAt(node, module, fingerprint)
|
||||
released := 0
|
||||
for _, key := range keys {
|
||||
r, _, ok := ch.couldNotRunOf(ctx, key)
|
||||
if !ok || !r.Part.matches(node, module) || r.AskedAt == at {
|
||||
continue
|
||||
}
|
||||
ch.mu.Lock()
|
||||
ch.release[key] = at
|
||||
ch.mu.Unlock()
|
||||
released++
|
||||
ch.logf("%s: %s runs (%s), which the check could not hear: asked again", key, module, at)
|
||||
}
|
||||
if released > 0 {
|
||||
ch.Reconsider(ctx)
|
||||
}
|
||||
}
|
||||
|
||||
// couldNotRunOf reads one key of the state could-not-run.
|
||||
func (ch *Checker) couldNotRunOf(ctx context.Context, key string) (CouldNotRun, uint64, bool) {
|
||||
raw, rev, found, err := ch.CouldNotRun.Get(ctx, key)
|
||||
if err != nil || !found || len(raw) == 0 {
|
||||
return CouldNotRun{}, 0, false
|
||||
}
|
||||
var r CouldNotRun
|
||||
if json.Unmarshal(raw, &r) != nil {
|
||||
return CouldNotRun{}, 0, false
|
||||
}
|
||||
return r, rev, true
|
||||
}
|
||||
|
||||
// takeRelease answers the `running` that released the check of key, once, and forgets it.
|
||||
func (ch *Checker) takeRelease(key string) (string, bool) {
|
||||
ch.mu.Lock()
|
||||
defer ch.mu.Unlock()
|
||||
at, ok := ch.release[key]
|
||||
delete(ch.release, key)
|
||||
return at, ok
|
||||
}
|
||||
|
||||
// keepCouldNotRun writes the check's record after a verdict that could not run, asked again by askedAt ("" for none).
|
||||
func (ch *Checker) keepCouldNotRun(ctx context.Context, cs *Case, v meshdeliverychecker.VerdictsByGroupHeadsValue, part Part, askedAt string) {
|
||||
if ch.CouldNotRun == nil {
|
||||
return
|
||||
}
|
||||
r := CouldNotRun{V: 1, Group: cs.Group, Heads: cs.Heads, Part: part, Words: failedOf(v).Words, Since: ch.now().UTC(), AskedAt: askedAt}
|
||||
data, err := json.Marshal(r)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
_, rev, found, err := ch.CouldNotRun.Get(ctx, cs.Key)
|
||||
if err == nil && found {
|
||||
_, err = ch.CouldNotRun.Update(ctx, cs.Key, data, rev)
|
||||
} else if err == nil {
|
||||
_, err = ch.CouldNotRun.Create(ctx, cs.Key, data)
|
||||
}
|
||||
if err != nil {
|
||||
ch.logf("%s: could not run, and that cannot be kept, so no running of %s asks it again until it is checked again: %v", cs.Key, part, err)
|
||||
return
|
||||
}
|
||||
if ch.CouldNotRunKept != nil {
|
||||
ch.CouldNotRunKept()
|
||||
}
|
||||
}
|
||||
|
||||
// clearCouldNotRun forgets the check of key that could not run, and clears its finding: its heads are checked again,
|
||||
// replaced, or their pull request closed.
|
||||
func (ch *Checker) clearCouldNotRun(ctx context.Context, key string) {
|
||||
if ch.CouldNotRun == nil {
|
||||
return
|
||||
}
|
||||
r, _, ok := ch.couldNotRunOf(ctx, key)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
if d, can := ch.CouldNotRun.(Deleter); can {
|
||||
if err := d.Delete(ctx, key); err != nil {
|
||||
ch.logf("%s: the record that it could not run cannot be deleted: %v", key, err)
|
||||
return
|
||||
}
|
||||
}
|
||||
if r.Raised && ch.Findings != nil {
|
||||
if err := ch.Findings.Delete(couldNotRunFinding(key, r, ch.now())); err != nil {
|
||||
ch.logf("%s: its finding %s cannot be cleared: %v", key, CouldNotRunKind, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// CouldNotRunDue raises the finding of every check that could not run and was not asked again within CouldNotRunBound by
|
||||
// now, and answers when the next one is due (the zero time when none waits). The checker's process waits for that time;
|
||||
// a test gives now.
|
||||
func (ch *Checker) CouldNotRunDue(ctx context.Context, now time.Time) time.Time {
|
||||
if ch.CouldNotRun == nil {
|
||||
return time.Time{}
|
||||
}
|
||||
keys, err := ch.CouldNotRun.Keys(ctx)
|
||||
if err != nil {
|
||||
ch.logf("the checks that could not run cannot be listed, so none is raised as %s now: %v", CouldNotRunKind, err)
|
||||
return time.Time{}
|
||||
}
|
||||
sort.Strings(keys)
|
||||
var next time.Time
|
||||
for _, key := range keys {
|
||||
r, rev, ok := ch.couldNotRunOf(ctx, key)
|
||||
if !ok || r.Raised {
|
||||
continue
|
||||
}
|
||||
due := r.Since.Add(CouldNotRunBound)
|
||||
if now.Before(due) {
|
||||
if next.IsZero() || due.Before(next) {
|
||||
next = due
|
||||
}
|
||||
continue
|
||||
}
|
||||
if ch.Findings == nil {
|
||||
continue
|
||||
}
|
||||
if err := ch.Findings.Put(couldNotRunFinding(key, r, now)); err != nil {
|
||||
ch.logf("%s: its finding %s cannot be written: %v", key, CouldNotRunKind, err)
|
||||
continue
|
||||
}
|
||||
r.Raised = true
|
||||
if data, err := json.Marshal(r); err == nil {
|
||||
if _, err := ch.CouldNotRun.Update(ctx, key, data, rev); err != nil && !errors.Is(err, ErrConflict) {
|
||||
ch.logf("%s: its finding is written, and that cannot be kept: %v", key, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
return next
|
||||
}
|
||||
|
||||
// couldNotRunFinding is the finding of the check of key that could not run.
|
||||
func couldNotRunFinding(key string, r CouldNotRun, at time.Time) views.Finding {
|
||||
return views.Finding{Observer: CheckerModule, Kind: CouldNotRunKind, ID: key,
|
||||
Summary: fmt.Sprintf("the check of %s (heads %s) could not run since %s: %s was not heard, and has not run since",
|
||||
r.Group, r.Heads, r.Since.UTC().Format(time.RFC3339), r.Part),
|
||||
Headline: fmt.Sprintf("The check of %s could not run", r.Group),
|
||||
Explanation: "A check that cannot hear a part of the mesh it reads says so and is asked again when that part runs. " +
|
||||
"It has not run for an hour: look at " + r.Part.String() + ". The finding clears when the heads are checked again, " +
|
||||
"replaced, or their pull request closes.",
|
||||
At: at.UTC()}
|
||||
}
|
||||
@@ -14,11 +14,11 @@ import (
|
||||
// granted by buscompose DeadLetterReads), and, under buildrun.NotTakenBound, when no build node took the ask within the
|
||||
// bound. What follows is the checker's: a try ends as error (Tryer.run), with the not-taken bound at the step bound
|
||||
// (Tryer.NotTaken); a check is asked again under its claim, at most LostAskTries times, then fails as not heard and is
|
||||
// judged again when the nodes change (Checker.try). A check's ask has no not-taken bound: it waits in the stream as long as
|
||||
// asked again when the part it could not hear runs (Checker.try). A check's ask has no not-taken bound: it waits in the stream as long as
|
||||
// the backlog is (ADR 0525 §9), and a check ended for a backlog would owe its pull request a failure.
|
||||
|
||||
// LostAskTries is how often a check asks again whose ask was given up, before it fails as not heard (UnheardWords) and is
|
||||
// judged again when the nodes change.
|
||||
// asked again when the part it could not hear runs.
|
||||
const LostAskTries = 3
|
||||
|
||||
// IsLostAsk says whether a verdict failed because its ask judged nothing: the build seat gave it up (buildrun.LostWords), or
|
||||
|
||||
@@ -256,7 +256,7 @@ func (a asVerdicts) Update(key string, value []byte, revision uint64) error {
|
||||
// Issue 734 and the lost check of 2026-10-28: a check whose ask judged nothing is asked again, and one whose step ended
|
||||
// unheard is judged. Given up by the build seat once, or its step ended by the build module's next process after a
|
||||
// delivery replaced it, the check is asked again under its claim and passes; given up every time it fails loudly as not
|
||||
// heard, judged again when the nodes change. A step whose watch went deaf is read at the next step end of its build module
|
||||
// heard, could not run, asked again when its part runs. A step whose watch went deaf is read at the next step end of its build module
|
||||
// (a nudge) and judged with no new ask; one no event reaches is named lost by checker_status, refused by checker_ask_again
|
||||
// while its step still runs, and asked again by it once its step ended.
|
||||
func TestACheckWhoseAskJudgedNothingIsAskedAgainOrRead(t *testing.T) {
|
||||
@@ -342,8 +342,8 @@ func TestACheckWhoseAskJudgedNothingIsAskedAgainOrRead(t *testing.T) {
|
||||
if _, _, kept, _ := asksKept.Get(ctx, key); kept {
|
||||
t.Errorf("the asks of the ended check are still kept under %s", key)
|
||||
}
|
||||
if row.state == meshdeliverycheckerclient.VerdictFail && (!IsLostAsk(v) || !IsNodesFailure(v)) {
|
||||
t.Errorf("a check that judged nothing every time = %q, want lost and judged again when the nodes change", v.Failed.Words)
|
||||
if row.state == meshdeliverycheckerclient.VerdictFail && (!IsLostAsk(v) || !IsCouldNotRun(v)) {
|
||||
t.Errorf("a check that judged nothing every time = %q, want lost and could not run", v.Failed.Words)
|
||||
}
|
||||
if row.how == "hand" {
|
||||
rd := Reader{VerdictState: kvRead(r.verdicts), CandidateState: kvRead(newMemKV())}
|
||||
|
||||
@@ -49,6 +49,8 @@ type Reader struct {
|
||||
StepStates map[string]StateRead // the build modules' steps states, by module
|
||||
HaveState StateRead // the node-engine's have: whether a build module is on a node and running
|
||||
StepBound time.Duration // how long the checker waits for one step; DefaultStepBound when 0
|
||||
// CouldNotRunState is the checker's own state could-not-run (CouldNotRunState); without it none is named.
|
||||
CouldNotRunState StateRead
|
||||
// Trunks are the trunk heads of every repository (the gate's source); nil names no re-judge after a trunk move.
|
||||
Trunks func() (map[string]string, error)
|
||||
// Runs says which commit the mesh runs of a module (the indexer's modules), as the checker's decision reads it: the
|
||||
@@ -412,6 +414,9 @@ type Status struct {
|
||||
// CannotBeJudged are the candidates no pull request of which can be judged, by group, with why: the gate's source.main
|
||||
// names no trunk head of their repositories (issue 1422). They are judged once it does; until then their status is error.
|
||||
CannotBeJudged map[string]string `json:"cannot_be_judged"`
|
||||
// CouldNotRun are the checks that could not run, by verdict key: the part not heard, since when, the running that asked
|
||||
// it again, and whether it is the finding check-could-not-run (design 47 test 58). Each is asked again when its part runs.
|
||||
CouldNotRun map[string]CouldNotRun `json:"could_not_run"`
|
||||
// Outside are the candidates outside the mesh, by group, with why (case.go Outside): judged nothing, owed nothing.
|
||||
Outside map[string]string `json:"outside_the_mesh"`
|
||||
// Lost are the candidates whose verdict is running while no step of theirs can still run and their newest step ended, by
|
||||
@@ -446,6 +451,27 @@ func (r Reader) deadLetters() DeadLettersHeard {
|
||||
return DeadLettersHeard{Heard: a.Healthy, Why: a.Why}
|
||||
}
|
||||
|
||||
// couldNotRun reads the checks that could not run, by verdict key; nil when they cannot be read, said in a note.
|
||||
func (r Reader) couldNotRun(st *Status) map[string]CouldNotRun {
|
||||
if r.CouldNotRunState.Keys == nil || r.CouldNotRunState.Get == nil {
|
||||
st.Notes = append(st.Notes, "the checker's state could-not-run is not readable here, so no check that could not run is named")
|
||||
return nil
|
||||
}
|
||||
keys, err := r.CouldNotRunState.Keys()
|
||||
if err != nil {
|
||||
st.Notes = append(st.Notes, "the checker's state could-not-run cannot be listed: "+err.Error())
|
||||
return nil
|
||||
}
|
||||
out := map[string]CouldNotRun{}
|
||||
for _, k := range keys {
|
||||
var c CouldNotRun
|
||||
if raw, err := r.CouldNotRunState.Get(k); err == nil && json.Unmarshal(raw, &c) == nil {
|
||||
out[k] = c
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// Status reads the build modules' steps and the candidates against their verdicts.
|
||||
func (r Reader) Status() (Status, error) {
|
||||
st := Status{RunningByNode: map[string]int{}, Lost: map[string]string{}}
|
||||
@@ -544,7 +570,7 @@ func (r Reader) Status() (Status, error) {
|
||||
}
|
||||
st.BuildsNotEnded[c.Group] = words
|
||||
}
|
||||
case IsNodesFailure(v):
|
||||
case IsCouldNotRun(v):
|
||||
st.Waiting[KindNodes] = append(st.Waiting[KindNodes], c.Group)
|
||||
case IsUntrunked(v):
|
||||
// Its trunk head is known now: it waits for its whole check.
|
||||
@@ -557,6 +583,7 @@ func (r Reader) Status() (Status, error) {
|
||||
}
|
||||
}
|
||||
st.DeadLetters = r.deadLetters()
|
||||
st.CouldNotRun = r.couldNotRun(&st)
|
||||
st.Notes = append(st.Notes, "nothing holds the checker and it holds no place: running_steps are the build modules' keys that can still be running (started within the step bound, their build module running on the node); a candidate without a verdict has its check ask in a build seat's stream, waiting for a node with a free slot, whose backlog the delivery view shows (ADR 0525 §4, §9)")
|
||||
return st, nil
|
||||
}
|
||||
|
||||
@@ -316,6 +316,15 @@ func TestTheStatusCountsAnHQOnlyGroupAsUnjudged(t *testing.T) {
|
||||
if len(st.Unjudged) != 1 || st.Unjudged[0] != "pb-x" {
|
||||
t.Errorf("unjudged = %v, want pb-x", st.Unjudged)
|
||||
}
|
||||
// A check that could not run is named with the part not heard (design 47 test 58).
|
||||
rec := json.RawMessage(`{"v":1,"group":"feat/x","heads":"0123456789ab","part":{"node":"g14","module":"forge"},"since":"2026-11-09T20:00:00Z"}`)
|
||||
st, err = Reader{CandidateState: fakeState{}.read(), VerdictState: fakeState{}.read(), CouldNotRunState: fakeState{"feat/x.0123456789ab": rec}.read()}.Status()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if c := st.CouldNotRun["feat/x.0123456789ab"]; c.Part != (Part{Node: "g14", Module: "forge"}) {
|
||||
t.Errorf("could_not_run = %+v, want the check of feat/x naming forge on g14", st.CouldNotRun)
|
||||
}
|
||||
}
|
||||
|
||||
// slowBucket wraps a state with a Get that takes delay and counts the values read, the lists and the most Gets in flight.
|
||||
|
||||
@@ -678,7 +678,7 @@ func (tr *Tryer) run(ctx context.Context, key string) {
|
||||
t.Said += " | " + n
|
||||
}
|
||||
switch {
|
||||
case !judged || IsNodesFailure(v) || IsLostAsk(v) || IsNotTaken(v):
|
||||
case !judged || IsCouldNotRun(v) || IsLostAsk(v) || IsNotTaken(v):
|
||||
// Not the change's: its ask was given up or never taken (issue 734), or a part of the mesh was not heard.
|
||||
t.State = TryError
|
||||
if IsLostAsk(v) || IsNotTaken(v) {
|
||||
|
||||
@@ -109,7 +109,7 @@ func TestManifestDeclarations(t *testing.T) { // 1678/AC1
|
||||
t.Errorf("module = %q", m.Module)
|
||||
}
|
||||
// The histories of the checker's four seat states (checked against internal/check names below); the names follow the seat mesh-delivery-checker.
|
||||
if want := []state{{Name: "asks", History: 1}, {Name: "hearing", History: 1}, {Name: "inputs", TTL: 86400}, {Name: "compositions", TTL: 259200}, {Name: "kept", History: 1}, {Name: "owed", History: 5}, {Name: "previews", History: 5}, {Name: "tries", History: 1}, {Name: "verdicts", History: 5}, {Name: "first-views", History: 1}, {Name: "findings", History: 1}}; !reflect.DeepEqual(m.State, want) {
|
||||
if want := []state{{Name: "asks", History: 1}, {Name: "hearing", History: 1}, {Name: "inputs", TTL: 86400}, {Name: "compositions", TTL: 259200}, {Name: "kept", History: 1}, {Name: "owed", History: 5}, {Name: "previews", History: 5}, {Name: "tries", History: 1}, {Name: "verdicts", History: 5}, {Name: "first-views", History: 1}, {Name: "findings", History: 1}, {Name: "could-not-run", History: 1}}; !reflect.DeepEqual(m.State, want) {
|
||||
t.Errorf("state = %+v, want %+v", m.State, want)
|
||||
}
|
||||
if want := []string{"mesh-build-go.steps", "mesh-build-go.toolchains", "mesh-build-node.steps", "mesh-build-node.toolchains",
|
||||
|
||||
@@ -45,7 +45,8 @@
|
||||
{ "name": "tries", "history": 1 },
|
||||
{ "name": "verdicts", "history": 5 },
|
||||
{ "name": "first-views", "history": 1 },
|
||||
{ "name": "findings", "history": 1 }
|
||||
{ "name": "findings", "history": 1 },
|
||||
{ "name": "could-not-run", "history": 1 }
|
||||
],
|
||||
"reads": [
|
||||
"mesh-build-go.steps",
|
||||
|
||||
Reference in New Issue
Block a user