jschoubben
16594feff1
Review of fix/lost-check: an ask no node took yet has no step key, so asking again then would run the check twice (issue 1791's kind). The checker keeps the asks each running check put in a new state asks, and the tool refuses unless every one of them has a step key. Asking again under the same ask id is not done: the stream drops a second put of an id within its de-duplication window and keeps both past it, so the waiting copy is never replaced. Tests: the hand row of the lost-ask table refuses while its ask waits; counts unchanged.
344 lines
13 KiB
Go
344 lines
13 KiB
Go
package check
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"strings"
|
|
"sync"
|
|
"testing"
|
|
"time"
|
|
|
|
"git.novox.be/novox/mesh-sdk/go/buildstep"
|
|
"git.novox.be/novox/mesh-sdk/go/delivery"
|
|
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
|
|
)
|
|
|
|
// losingAsks is the build seat over a bus that gives up the first `lose` asks put (DEAD_LETTERS keeps them, as on
|
|
// 2026-10-26 during a bus reload: try-1793017340003993788) and answers the rest; with never set it neither answers nor
|
|
// gives up (an ask no node takes).
|
|
type losingAsks struct {
|
|
*buildrun.FakeAsks
|
|
answer func(delivery.StepAsk) delivery.Step
|
|
dead *buildrun.DeadLetters
|
|
never bool
|
|
|
|
mu sync.Mutex
|
|
lose int
|
|
}
|
|
|
|
func (l *losingAsks) PublishAsk(ctx context.Context, subject, id string, data []byte) error {
|
|
if err := l.FakeAsks.PublishAsk(ctx, subject, id, data); err != nil {
|
|
return err
|
|
}
|
|
l.mu.Lock()
|
|
lost := l.lose > 0
|
|
if lost {
|
|
l.lose--
|
|
}
|
|
l.mu.Unlock()
|
|
switch {
|
|
case l.never:
|
|
case lost:
|
|
h := map[string][]string{buildrun.DeadHeaderMsgID: {id}, buildrun.DeadHeaderDeliveries: {"5"}, buildrun.DeadHeaderStream: {"SEAT_NODE_BUILD_GO"},
|
|
buildrun.DeadHeaderGaveUp: {"2026-10-26T12:26:40Z"}}
|
|
g, ok := buildrun.GivenUpOf(h)
|
|
if !ok {
|
|
return nil
|
|
}
|
|
go l.dead.Heard(g)
|
|
default:
|
|
a, err := delivery.DecodeStepAsk(data)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
l.Write(l.answer(a))
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func losing(b *bench, lose int) *losingAsks {
|
|
l := &losingAsks{FakeAsks: &buildrun.FakeAsks{}, answer: b.answer, dead: &buildrun.DeadLetters{}, lose: lose}
|
|
b.ex.Asks = buildrun.Asker{Publisher: l, Waiter: buildrun.LostAskWaiter{Inner: l.FakeAsks, Dead: l.dead}, Bound: 5 * time.Second}
|
|
return l
|
|
}
|
|
|
|
// endable is the tries state as the end tool sees it.
|
|
type endable struct{ kv *memKV }
|
|
|
|
func (e endable) Entry(key string) (json.RawMessage, uint64, error) {
|
|
v, rev, _, err := e.kv.Get(context.Background(), key)
|
|
return v, rev, err
|
|
}
|
|
func (e endable) Update(key string, value []byte, revision uint64) error {
|
|
_, err := e.kv.Update(context.Background(), key, value, revision)
|
|
return err
|
|
}
|
|
|
|
// Issue 734: a try whose ask the build seat gave up on ends as error saying so, and the next ask of its head is a new try;
|
|
// one no node takes ends at the step bound; one ended by hand frees its place and stops being waited for.
|
|
func TestAnAskGivenUpOrNeverTakenEndsItsTryAsErrorAndItsHeadIsAskedAnew(t *testing.T) {
|
|
ctx := context.Background()
|
|
for _, row := range []struct {
|
|
name string
|
|
lose int
|
|
never bool
|
|
want string
|
|
}{
|
|
{"given up by the build seat", 1, false, buildrun.LostWords + " after 5 deliveries at 2026-10-26T12:26:40Z"},
|
|
{"taken by no build node within the bound", 0, true, buildrun.NotTakenWords + " within 200ms"},
|
|
} {
|
|
t.Run(row.name, func(t *testing.T) {
|
|
kv := newMemKV()
|
|
at := time.Date(2026, 10, 26, 12, 22, 20, 0, time.UTC)
|
|
id, err := AskTry(memTries{kv}, tryTrunks, TryAsk{Repository: repoA, Head: head1}, at)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
b := newBench(t)
|
|
l := losing(b, row.lose)
|
|
l.never = row.never
|
|
tr := &Tryer{Tries: kv, Judge: b.ex, NotTaken: 200 * time.Millisecond}
|
|
raw, _, _, _ := kv.Get(ctx, id)
|
|
tr.Put(ctx, id, raw)
|
|
tr.Wait()
|
|
got := readTry(t, kv, id)
|
|
if got.State != TryError || len(got.Failed) != 1 || !strings.Contains(got.Failed[0], row.want) || !strings.Contains(got.Said, row.want) {
|
|
t.Fatalf("try = %+v, want error saying %q", got, row.want)
|
|
}
|
|
again, err := AskTry(memTries{kv}, tryTrunks, TryAsk{Repository: repoA, Head: head1}, at.Add(time.Minute))
|
|
if err != nil || again == id {
|
|
t.Errorf("the head asked again answered %q, %v, want a new try", again, err)
|
|
}
|
|
})
|
|
}
|
|
t.Run("ended by hand", func(t *testing.T) {
|
|
kv := newMemKV()
|
|
id, _ := AskTry(memTries{kv}, tryTrunks, TryAsk{Repository: repoA, Head: head1}, time.Now())
|
|
b := newBench(t)
|
|
losing(b, 0).never = true
|
|
tr := &Tryer{Tries: kv, Judge: b.ex} // no bound: only the hand ends it
|
|
raw, _, _, _ := kv.Get(ctx, id)
|
|
tr.Put(ctx, id, raw)
|
|
deadline := time.Now().Add(2 * time.Second)
|
|
for readTry(t, kv, id).State != TryRunning && time.Now().Before(deadline) {
|
|
time.Sleep(5 * time.Millisecond)
|
|
}
|
|
if _, err := EndTry(endable{kv}, id, "", time.Now()); err == nil {
|
|
t.Error("an end without a why was taken")
|
|
}
|
|
ended, err := EndTry(endable{kv}, id, "its ask was lost in the bus reload", time.Now())
|
|
if err != nil || ended.State != TryError || !strings.HasPrefix(ended.Said, "ended by hand ("+TryEndTool+"): its ask was lost") {
|
|
t.Fatalf("ended = %+v, %v", ended, err)
|
|
}
|
|
raw, _, _, _ = kv.Get(ctx, id)
|
|
tr.Put(ctx, id, raw) // the watch of the tries hands the end over
|
|
tr.Wait() // returns: the try's judging stopped
|
|
if got := readTry(t, kv, id); got.State != TryError || got.Said != ended.Said {
|
|
t.Errorf("try = %+v, want the hand's end kept", got)
|
|
}
|
|
if _, err := EndTry(endable{kv}, id, "again", time.Now()); err == nil || !strings.Contains(err.Error(), "ended already") {
|
|
t.Errorf("a second end answered %v", err)
|
|
}
|
|
})
|
|
}
|
|
|
|
// troubled is the build seat as a check meets it when its step's end does not reach its wait: the first `restarts` asks'
|
|
// steps are ended by the build module's next process (buildstep.RestartReason), and the first `deaf` asks' watches hear
|
|
// nothing though their steps end (the lost check of 2026-10-28). Its steps state is read through AskSteps and Steps.
|
|
type troubled struct {
|
|
*buildrun.FakeAsks
|
|
answer func(delivery.StepAsk) delivery.Step
|
|
|
|
mu sync.Mutex
|
|
restarts, deaf int
|
|
deafIDs map[string]bool
|
|
steps map[string]delivery.Step
|
|
}
|
|
|
|
func (tr *troubled) PublishAsk(ctx context.Context, subject, id string, data []byte) error {
|
|
if err := tr.FakeAsks.PublishAsk(ctx, subject, id, data); err != nil {
|
|
return err
|
|
}
|
|
a, err := delivery.DecodeStepAsk(data)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
step := tr.answer(a)
|
|
step.What = buildstep.AskWhat(a)
|
|
tr.mu.Lock()
|
|
if tr.restarts > 0 {
|
|
tr.restarts--
|
|
step.State, step.Reason, step.Result = delivery.StepFailed, buildstep.RestartReason, nil
|
|
}
|
|
deaf := tr.deaf > 0
|
|
if deaf {
|
|
tr.deaf--
|
|
tr.deafIDs[id] = true
|
|
}
|
|
tr.steps[id] = step
|
|
tr.mu.Unlock()
|
|
if !deaf {
|
|
tr.Write(step)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (tr *troubled) WatchAsk(ctx context.Context, module, askID string) (<-chan delivery.Step, error) {
|
|
tr.mu.Lock()
|
|
deaf := tr.deafIDs[askID]
|
|
tr.mu.Unlock()
|
|
if deaf {
|
|
return make(chan delivery.Step), nil // open, and silent for good
|
|
}
|
|
return tr.FakeAsks.WatchAsk(ctx, module, askID)
|
|
}
|
|
|
|
func (tr *troubled) AskSteps(_ context.Context, _, askID string) ([]delivery.Step, error) {
|
|
tr.mu.Lock()
|
|
defer tr.mu.Unlock()
|
|
if s, ok := tr.steps[askID]; ok {
|
|
return []delivery.Step{s}, nil
|
|
}
|
|
return nil, nil
|
|
}
|
|
|
|
// Steps is the steps state as the read tools read it, with running in place of every step the hand has not let end.
|
|
func (tr *troubled) Steps(running bool) StateRead {
|
|
tr.mu.Lock()
|
|
defer tr.mu.Unlock()
|
|
vals := map[string]json.RawMessage{}
|
|
for id, s := range tr.steps {
|
|
if running {
|
|
s.State, s.Ended, s.Started = delivery.StepRunning, time.Time{}, time.Now()
|
|
}
|
|
raw, _ := delivery.EncodeStep(s)
|
|
vals[s.Node+"."+id] = raw
|
|
}
|
|
return StateRead{Keys: func() ([]string, error) {
|
|
var out []string
|
|
for k := range vals {
|
|
out = append(out, k)
|
|
}
|
|
return out, nil
|
|
}, Get: func(k string) (json.RawMessage, error) { return vals[k], nil }}
|
|
}
|
|
|
|
// kvRead is a bucket in memory as the read tools read it, live.
|
|
func kvRead(kv *memKV) StateRead {
|
|
return StateRead{Keys: func() ([]string, error) { return kv.Keys(context.Background()) },
|
|
Get: func(k string) (json.RawMessage, error) {
|
|
v, _, _, err := kv.Get(context.Background(), k)
|
|
return v, err
|
|
}}
|
|
}
|
|
|
|
// asVerdicts is the rig's verdicts as the ask-again tool reads and writes them.
|
|
type asVerdicts struct{ kv *memKV }
|
|
|
|
func (a asVerdicts) Entry(key string) (json.RawMessage, uint64, error) {
|
|
v, rev, _, err := a.kv.Get(context.Background(), key)
|
|
return v, rev, err
|
|
}
|
|
func (a asVerdicts) Update(key string, value []byte, revision uint64) error {
|
|
_, err := a.kv.Update(context.Background(), key, value, revision)
|
|
return err
|
|
}
|
|
|
|
// Issue 734 and the lost check of 2026-10-28: a check whose ask judged nothing is asked again, and one whose step ended
|
|
// unheard is judged. Given up by the build seat once, or its step ended by the build module's next process after a
|
|
// delivery replaced it, the check is asked again under its claim and passes; given up every time it fails loudly as not
|
|
// heard, judged again when the nodes change. A step whose watch went deaf is read at the next step end of its build module
|
|
// (a nudge) and judged with no new ask; one no event reaches is named lost by checker_status, refused by checker_ask_again
|
|
// while its step still runs, and asked again by it once its step ended.
|
|
func TestACheckWhoseAskJudgedNothingIsAskedAgainOrRead(t *testing.T) {
|
|
ctx := context.Background()
|
|
for _, row := range []struct {
|
|
name string
|
|
lose, restarts, deaf int
|
|
how string // "nudge": a step of the build module ends; "hand": checker_ask_again
|
|
state string
|
|
asks int
|
|
}{
|
|
{name: "given up once", lose: 1, state: delivery.VerdictPass, asks: 2},
|
|
{name: "given up every time", lose: LostAskTries, state: delivery.VerdictFail, asks: LostAskTries},
|
|
{name: "its build module replaced under it", restarts: 1, state: delivery.VerdictPass, asks: 2},
|
|
{name: "replaced under every ask", restarts: LostAskTries, state: delivery.VerdictFail, asks: LostAskTries},
|
|
{name: "its watch deaf, a step of its build module ends", deaf: 1, how: "nudge", state: delivery.VerdictPass, asks: 1},
|
|
{name: "its watch deaf, asked again by hand", deaf: 1, how: "hand", state: delivery.VerdictPass, asks: 2},
|
|
} {
|
|
t.Run(row.name, func(t *testing.T) {
|
|
b := newBench(t)
|
|
var asked func() int
|
|
var heard *buildrun.HeardEnds
|
|
var tr *troubled
|
|
if row.lose > 0 {
|
|
l := losing(b, row.lose)
|
|
asked = func() int { return len(l.Asks()) }
|
|
} else {
|
|
tr = &troubled{FakeAsks: &buildrun.FakeAsks{}, answer: b.answer, restarts: row.restarts, deaf: row.deaf,
|
|
deafIDs: map[string]bool{}, steps: map[string]delivery.Step{}}
|
|
heard = &buildrun.HeardEnds{Inner: tr, Read: tr}
|
|
b.ex.Asks = buildrun.Asker{Publisher: tr, Waiter: heard, Bound: 5 * time.Second}
|
|
asked = func() int { return len(tr.Asks()) }
|
|
}
|
|
r := newRig(t, b.ex)
|
|
asksKept := newMemKV()
|
|
r.ch.Asks = asksKept
|
|
cand := candidateOf(t, "feat/x", pullOf(repoA, 7, head1))
|
|
key := keyOf(t, "feat/x", cand)
|
|
r.ch.Candidate(ctx, "feat%2Fx", cand)
|
|
if row.deaf > 0 {
|
|
for deadline := time.Now().Add(2 * time.Second); heard.Waiting() == 0 && time.Now().Before(deadline); {
|
|
time.Sleep(2 * time.Millisecond)
|
|
}
|
|
if v := r.verdict(t, key); v.State != delivery.VerdictRunning {
|
|
t.Fatalf("a check whose step ended unheard = %s, want running until it is read", v.State)
|
|
}
|
|
}
|
|
switch row.how {
|
|
case "nudge":
|
|
heard.Nudge(buildrun.NodeBuildModule) // another build module's step: not this one's to read
|
|
heard.Nudge(buildrun.GoBuildModule)
|
|
case "hand":
|
|
cands := newMemKV()
|
|
ckey, _ := delivery.CandidatesKey("feat/x")
|
|
_, _ = cands.Create(ctx, ckey, cand)
|
|
none := StateRead{Keys: func() ([]string, error) { return nil, nil }, Get: func(string) (json.RawMessage, error) { return nil, nil }}
|
|
rd := Reader{CandidateState: kvRead(cands), VerdictState: kvRead(r.verdicts), AskState: kvRead(asksKept),
|
|
StepStates: map[string]StateRead{buildrun.GoBuildModule: none}} // no node took its ask yet
|
|
if _, err := rd.AskAgain(asVerdicts{r.verdicts}, "feat/x", "its watch went deaf"); err == nil || !strings.Contains(err.Error(), "waits in its build seat's stream") {
|
|
t.Errorf("asked again while its ask waits in the stream (issue 1791's kind): %v", err)
|
|
}
|
|
rd.StepStates[buildrun.GoBuildModule] = tr.Steps(true)
|
|
if _, err := rd.AskAgain(asVerdicts{r.verdicts}, "feat/x", "its watch went deaf"); err == nil || !strings.Contains(err.Error(), "a step of it runs") {
|
|
t.Errorf("asked again while its step runs: %v", err)
|
|
}
|
|
rd.StepStates[buildrun.GoBuildModule] = tr.Steps(false)
|
|
if st, _ := rd.Status(); !strings.Contains(st.Lost["feat/x"], AskAgainTool) {
|
|
t.Errorf("status lost = %v, want feat/x named with %s", st.Lost, AskAgainTool)
|
|
}
|
|
got, err := rd.AskAgain(asVerdicts{r.verdicts}, "feat/x", "its watch went deaf")
|
|
if err != nil || got.Key != key || !strings.HasPrefix(got.Said, AskedAgainWords) {
|
|
t.Fatalf("asked again = %+v, %v", got, err)
|
|
}
|
|
raw, _, _, _ := r.verdicts.Get(ctx, key)
|
|
r.ch.VerdictWritten(ctx, key, raw) // the checker's watch of its verdicts hands it over
|
|
}
|
|
r.ch.Wait()
|
|
v := r.verdict(t, key)
|
|
if v.State != row.state || asked() != row.asks {
|
|
t.Errorf("verdict %s (%s) after %d asks, want %s after %d", v.State, v.Failed.Words, asked(), row.state, row.asks)
|
|
}
|
|
if row.state == delivery.VerdictFail && (!IsLostAsk(v) || !IsNodesFailure(v)) {
|
|
t.Errorf("a check that judged nothing every time = %q, want lost and judged again when the nodes change", v.Failed.Words)
|
|
}
|
|
if row.how == "hand" {
|
|
rd := Reader{VerdictState: kvRead(r.verdicts), CandidateState: kvRead(newMemKV())}
|
|
if _, err := rd.AskAgain(asVerdicts{r.verdicts}, "feat/x", "again"); err == nil {
|
|
t.Error("a judged check was asked again")
|
|
}
|
|
}
|
|
})
|
|
}
|
|
}
|