package check import ( "context" "encoding/json" "errors" "fmt" "slices" "strings" "testing" "time" "git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient" "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient" "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshdeliverycheckerclient" "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshwants" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-checker/checkermsg" "git.novox.be/novox/mesh-delivery/sdk/buildstep" ) // memTries is the tries state as the tools see it, over a memKV. type memTries struct{ kv *memKV } func (m memTries) Keys() ([]string, error) { return m.kv.Keys(context.Background()) } func (m memTries) Get(key string) (json.RawMessage, error) { v, _, _, err := m.kv.Get(context.Background(), key) return v, err } func (m memTries) Create(key string, value []byte) error { _, err := m.kv.Create(context.Background(), key, value) return err } // writtenTries is a memKV that keeps every running try it was given to write, in order. type writtenTries struct { *memKV running []Try } func (w *writtenTries) Update(ctx context.Context, key string, value []byte, rev uint64) (uint64, error) { if t, err := DecodeTry(value); err == nil && !t.Over() { w.running = append(w.running, t) } return w.memKV.Update(ctx, key, value, rev) } var tryTrunks = map[string]string{repoA: trunkA, repoSDK: trunkS, hqRepo: trunkA} func readTry(t testing.TB, kv *memKV, id string) Try { t.Helper() raw, _, found, _ := kv.Get(context.Background(), id) if !found { t.Fatalf("no try %s", id) } tr, err := DecodeTry(raw) if err != nil { t.Fatal(err) } return tr } func TestATryIsAskedQueuedOncePerHeadAndAtMostFourAtOnce(t *testing.T) { // ADR 0501 §5 kv := newMemKV() store := memTries{kv} at := time.Date(2026, 10, 22, 18, 0, 0, 0, time.UTC) id, err := AskTry(store, tryTrunks, TryAsk{Repository: repoA, Head: head1, Trunk: "main"}, at) if err != nil || !strings.HasPrefix(id, "try-") { t.Fatalf("id %q, %v", id, err) } tr := readTry(t, kv, id) if tr.State != TryQueued || tr.Trunk != trunkA || tr.Branch != "main" || tr.Head != head1 || tr.Repository != repoA { t.Errorf("try = %+v, want queued on the trunk's head", tr) } again, err := AskTry(store, tryTrunks, TryAsk{Repository: repoA, Head: head1, Trunk: "main"}, at.Add(time.Second)) if err != nil || again != id { t.Errorf("a second ask of the same head answered %q, %v, want %q", again, err, id) } // Issue 1890: tries on a build node (a step of theirs running, or none named yet) and tries waiting for one (queued, or // their asks not taken) are bounded apart, MaxTries and MaxTriesWaiting; the refusal names both counts. // A group try is the same try whatever the order of its members; the commits asked are the commits tried. group := TryAsk{Members: []TryAskMember{{Repository: repoSDK, Head: head2}, {Repository: repoA, Head: head1}}} gid, err := AskTry(store, tryTrunks, group, at.Add(time.Second)) if err != nil || gid == id { t.Fatalf("the group try = %q, %v, want a try of its own", gid, err) } group.Members[0], group.Members[1] = group.Members[1], group.Members[0] if again, err := AskTry(store, tryTrunks, group, at.Add(time.Minute)); err != nil || again != gid { t.Errorf("the group asked in another order answered %q, %v, want %q", again, err, gid) } if g := readTry(t, kv, gid); len(g.Members) != 2 || g.Members[0].Repository != repoA || g.Members[0].Head != head1 || g.Members[1].Trunk != trunkS { t.Errorf("group try = %+v, want its members by repository at the commits asked", g.Members) } if open, err := OpenTries(store); err != nil || len(open) != 2 || !open[0].Whole || len(open[1].Members) != 2 && len(open[0].Members) != 2 { t.Errorf("open tries = %+v, %v, want the two queued", open, err) } if !meshcatalogclient.IsTryGroup(gid) || meshcatalogclient.IsTryGroup("feat/x") || meshcatalogclient.IsTryGroup("fix/1921-try-1") { t.Errorf("which candidates keys are a try's group: %s is, a branch is not", gid) } onNode := func(i int) Try { return Try{ID: fmt.Sprintf("try-%d", i), Repository: repoA, Head: fmt.Sprintf("%040x", i), Trunk: trunkA, State: TryRunning, Steps: []TryStep{{Ask: "a", State: buildstep.StepRunning}}} } waitingFor := func(i int) Try { return Try{ID: fmt.Sprintf("try-%d", i), Repository: repoA, Head: fmt.Sprintf("%040x", i), Trunk: trunkA, State: TryRunning, Steps: []TryStep{{Ask: "a", State: TryAsked}}} } for _, c := range []struct { what string holding, wait int refused string }{ {"four queued behind the checks, none on a node: a fifth is taken", 0, 4, ""}, {"four on a node", MaxTries, 0, "now 4 run and 0 wait"}, {"three on a node, eight waiting", 3, MaxTriesWaiting, "a new try is refused while 4 tries run on a build node or 8 wait for one; now 3 run and 8 wait"}, } { kv := newMemKV() for i := range c.holding + c.wait { tr := waitingFor(i + 1) if i < c.holding { tr = onNode(i + 1) } data, _ := encodeTry(tr) _, _ = kv.Create(context.Background(), tr.ID, data) } _, err := AskTry(memTries{kv}, tryTrunks, TryAsk{Repository: repoSDK, Head: head1}, at.Add(time.Minute)) if (c.refused == "") != (err == nil) || (err != nil && !strings.Contains(err.Error(), c.refused)) { t.Errorf("%s: answered %v, want refused %q", c.what, err, c.refused) } } } func TestATryOfARepositoryWithNoCheckedLevelIsRefused(t *testing.T) { // the agreed error of the try store := memTries{newMemKV()} for _, repo := range []string{"novox/no-trunk", "novox/mesh-lab"} { // hq is checked since R4g (ADR 0527): see tryTrunks _, err := AskTry(store, tryTrunks, TryAsk{Repository: repo, Head: head1, Trunk: "main"}, time.Now()) if err == nil || err.Error() != "no level of "+repo+" is checked by the checker" { t.Errorf("%s: %v", repo, err) } } for _, bad := range []TryAsk{{Repository: repoA, Head: "abc"}, {Repository: repoA, Head: head1, Level: "e2e"}, {Repository: repoA, Head: head1, Tests: "a b"}, {Repository: "x", Head: head1}, {Repository: repoA, Head: head1, Trunk: "-x"}, // A group try (issue 1921): members or one repository, never both or none; one commit per repository; every member checked. {}, {Repository: repoA, Head: head1, Members: []TryAskMember{{Repository: repoSDK, Head: head2}}}, {Members: []TryAskMember{{Repository: repoA, Head: head1}, {Repository: repoA, Head: head2}}}, {Members: []TryAskMember{{Repository: repoA, Head: head1}, {Repository: repoSDK, Head: "main"}}}, {Members: []TryAskMember{{Repository: repoA, Head: head1}, {Repository: "novox/no-trunk", Head: head2}}}} { if _, err := AskTry(store, tryTrunks, bad, time.Now()); err == nil { t.Errorf("%+v was taken", bad) } } } // fakeTryCandidates answers the indexer's candidate of a try's group from memory, recording the keys asked. type fakeTryCandidates struct { cand meshcatalog.CandidatesByGroupValue err error asked []string // during is called while the try waits for the candidate. during func() } func (f *fakeTryCandidates) TryCandidate(_ context.Context, group, heads string, _ time.Duration) (meshcatalog.CandidatesByGroupValue, error) { if f.during != nil { f.during() } f.asked = append(f.asked, group+"."+heads) return f.cand, f.err } // wholeBench makes the bench judge a whole try as the checker does: the composition on every placed node and the // indexer's candidate of the try's group. func wholeBench(b *bench, cand meshcatalog.CandidatesByGroupValue) (*fakeCompositions, *fakeTryCandidates) { comps := &fakeCompositions{v: meshwants.CompositionsByGroupHeadsValue{State: meshdeliverycheckerclient.CompositionPass, Sdk: "v0.1.14-x", Machines: []string{"g14", "shanks"}, Took: 40}} cands := &fakeTryCandidates{cand: cand} b.ex.Compositions, b.ex.Candidates = comps, cands return comps, cands } // A group try (issue 1921): several repositories each at the exact commit asked, judged by the code of a group's pull-request // check; the two cannot drift, and a later check of the same heads keeps what the try passed. func TestATryIsJudgedAsAGroupsCheckAndOwesTheForgeNothing(t *testing.T) { // ADR 0501 §5, issue 1921 ctx := context.Background() kv := newMemKV() id, err := AskTry(memTries{kv}, tryTrunks, TryAsk{Members: []TryAskMember{{Repository: repoSDK, Head: head2, Trunk: trunkS}, {Repository: repoA, Head: head1, Trunk: trunkA}}}, time.Now()) if err != nil { t.Fatal(err) } b := newBench(t) comps, cands := wholeBench(b, meshcatalog.CandidatesByGroupValue{Fpset: "fps-of-the-try"}) written := &writtenTries{memKV: kv} var waiting string cands.during = func() { waiting = readTry(t, kv, id).Waiting } // ADR 0639 §5: the stored try says what it waits for var said []string tr := &Tryer{Tries: written, Judge: b.ex, Say: func(_ context.Context, event string, e checkermsg.Event) error { said = append(said, fmt.Sprintf("%s %s %d %s", event, e.ID, len(e.Members), e.State)) return nil }} raw, _, _, _ := kv.Get(ctx, id) tr.Put(ctx, id, raw) tr.Wait() // The indexer hears the start with the members at their commits, and the end (it writes and deletes the group's candidate). if len(said) != 2 || said[0] != "try.started "+id+" 2 " || said[1] != "try.ended "+id+" 0 "+TryPass { t.Errorf("events = %v, want the start with 2 members and the end", said) } // What the indexer reads: checkermsg.Decode accepts the events as said and refuses a body that is no try's. for _, row := range []struct { name, event, body string ok bool }{ {"a start with its members", checkermsg.Started, `{"id":"` + id + `","members":[{"repository":"` + repoA + `","head":"` + head1 + `","trunk":"` + trunkA + `"}]}`, true}, {"an end with its state", checkermsg.Ended, `{"id":"` + id + `","state":"pass"}`, true}, {"an id that is no try's", checkermsg.Ended, `{"id":"pull-7","state":"pass"}`, false}, {"a start with no member", checkermsg.Started, `{"id":"` + id + `"}`, false}, {"a body that is no JSON", checkermsg.Ended, `try-7`, false}, } { if e, err := checkermsg.Decode(row.event, []byte(row.body)); (err == nil) != row.ok || (row.ok && e.ID != id) { t.Errorf("Decode of %s = %+v, %v, want ok=%v", row.name, e, err, row.ok) } } got := readTry(t, kv, id) if !strings.Contains(waiting, "candidate") || got.Waiting != "" { t.Errorf("waiting while the candidate was awaited = %q, after the end %q: want the reason, then none", waiting, got.Waiting) } if got.Told { // the end was said: nothing is owed t.Errorf("the ended try still owes its end: %+v", got) } if open, err := OpenTries(memTries{kv}); err != nil || len(open) != 0 { t.Errorf("open tries after the end = %v, %v", open, err) } if got.State != TryPass || got.Took == "" || len(got.Members) != 2 || got.Members[0].Repository != repoA || got.Members[1].Head != head2 || got.Said == "" || got.Levels[buildstep.LevelUnit].Run+got.Levels[buildstep.LevelUnit].Kept != 4 { // the bench gives both members the same digests t.Fatalf("try = %+v", got) } if got.Composition == nil || got.Composition.State != meshdeliverycheckerclient.CompositionPass || len(got.Machines) == 0 { t.Errorf("the try's composition = %+v, want the composition on every placed node, as a check's", got.Composition) } // The same key the indexer's candidate of the group has: the try's id and the heads of its members. if len(comps.asked) != 1 || comps.asked[0] != id || len(cands.asked) != 1 || !strings.HasPrefix(cands.asked[0], id+".") { t.Errorf("composed %v, candidate asked %v, want the try's key %s", comps.asked, cands.asked, id) } // Issue 1890: while it runs, the try names its asks as soon as they are put, and then where each step stands. if len(written.running) < 3 || len(written.running[1].Asks) != 1 || written.running[1].Steps[0].State != TryAsked { t.Errorf("the running try's writes = %+v, want its ask named while it ran", written.running) } if len(got.Steps) != 2 || got.Steps[0].Node != "shanks" || got.Steps[0].State != buildstep.StepPassed || len(got.Asks) != 2 { t.Errorf("the ended try's steps = %+v, asks %v", got.Steps, got.Asks) } for _, a := range got.Asks { // the delivery view counts their steps as a try's (build_usage) if _, err := buildstep.AskStepKey("shanks", a); err != nil { t.Errorf("the try names %q, which is no ask id: %v", a, err) } } // Each member is fetched at the exact commit asked, merged onto its own trunk, with no pull request's ref. fetch := b.steps.tool("fetch") if len(fetch) != 2 { t.Fatalf("fetch = %+v, want one per member", fetch) } for _, f := range fetch { want := map[string][2]string{repoA: {head1, trunkA}, repoSDK: {head2, trunkS}}[f.args["repository"].(string)] if f.args["commit"] != want[0] || f.args["base"] != want[1] || f.args["ref"] != nil { t.Errorf("fetch = %+v, want the exact commit %s merged onto %s", f.args, want[0], want[1]) } } // A whole try writes the merge previews the indexer makes its candidate from; it writes no status and asks no Approve. if keys, _ := b.prev.Keys(ctx); len(keys) != 2 { t.Errorf("a whole try wrote the previews %v, want one per member", keys) } // An ended try is not run again. raw, _, _, _ = kv.Get(ctx, id) tr.Put(ctx, id, raw) tr.Wait() if n := len(b.steps.tool("fetch")); n != 2 { t.Errorf("an ended try was fetched again: %d fetches", n) } // The kept verdicts: what the try passed is kept under the digests of the merged trees, so the pull-request check of // the same head on the same trunk runs none of it again (and a push of other code runs only what changed). before := len(b.steps.tool("test")) out := b.judge(t, caseOf(t, pullOf(repoA, 7, head1))) if out.Verdict.State != meshdeliverycheckerclient.VerdictPass || out.Verdict.Levels[buildstep.LevelUnit].Kept == 0 || out.Verdict.Levels[buildstep.LevelUnit].Run != 0 || out.Verdict.Levels[buildstep.LevelIntegration].Run != 0 { t.Errorf("the check of the tried head = %+v, want everything the try passed kept and nothing run", out.Verdict.Levels) } if after := len(b.steps.tool("test")); after != before { t.Errorf("the check of the tried head ran %d test steps, want none: the try's passes are kept", after-before) } } func TestATryOfOneLevelAndSomeTestsRunsThemAloneAndKeepsNothing(t *testing.T) { ctx := context.Background() kv := newMemKV() id, err := AskTry(memTries{kv}, tryTrunks, TryAsk{Repository: repoA, Head: head1, Level: buildstep.LevelIntegration, Tests: "TestBus|TestStore"}, time.Now()) if err != nil { t.Fatal(err) } b := newBench(t) tr := &Tryer{Tries: kv, Judge: b.ex, Say: func(context.Context, string, checkermsg.Event) error { t.Error("a try of one level and some tests said an event: it has no candidate") return nil }} raw, _, _, _ := kv.Get(ctx, id) tr.Put(ctx, id, raw) tr.Wait() if open, _ := OpenTries(memTries{kv}); len(open) != 0 { t.Errorf("open = %v", open) } tests := b.steps.tool("test") if len(tests) != 1 || tests[0].args["level"] != buildstep.LevelIntegration || tests[0].args["run"] != "TestBus|TestStore" { t.Fatalf("tests = %+v, want the integration level alone with the run pattern", tests) } if n := len(b.steps.tool("digest")); n != 0 { t.Errorf("%d digests for a try of some tests", n) } if keys, _ := b.kept.Keys(ctx); len(keys) != 0 { t.Errorf("kept %v from a try of some tests", keys) } if got := readTry(t, kv, id); got.State != TryPass || !strings.Contains(got.Said, "only the tests TestBus|TestStore") { t.Errorf("try = %+v", got) } t.Run("a try of one level keeps only that level", func(t *testing.T) { kv := newMemKV() id, _ := AskTry(memTries{kv}, tryTrunks, TryAsk{Repository: repoA, Head: head1, Level: buildstep.LevelUnit}, time.Now()) b := newBench(t) tr := &Tryer{Tries: kv, Judge: b.ex} raw, _, _, _ := kv.Get(ctx, id) tr.Put(ctx, id, raw) tr.Wait() keys, _ := b.kept.Keys(ctx) if got := readTry(t, kv, id); got.State != TryPass || len(keys) == 0 { t.Fatalf("try = %+v, kept %v", got, keys) } for _, k := range keys { if !strings.HasPrefix(k, buildstep.LevelUnit+".") { t.Errorf("a try of the unit level kept %s", k) } } }) } func TestAFailingTryNamesWhatFailedAndAStepThatCannotRunIsAnError(t *testing.T) { ctx := context.Background() kv := newMemKV() b := newBench(t) b.test = func(c stepCall) (any, error) { return captured[Tested](t, "go-test-failing"), nil } id, _ := AskTry(memTries{kv}, tryTrunks, TryAsk{Repository: repoA, Head: head1}, time.Now()) tr := &Tryer{Tries: kv, Judge: b.ex} raw, _, _, _ := kv.Get(ctx, id) tr.Put(ctx, id, raw) tr.Wait() if got := readTry(t, kv, id); got.State != TryFail || len(got.Failed) != 2 || !strings.Contains(got.Failed[0], "unit: fixture/failing/bad.TestThreeIsFour") || !strings.Contains(got.Failed[1], "integration: fixture/failing/bad.TestThreeIsFour") { // both levels ran (the operator, 2026-11-02) t.Errorf("try = %+v", got) } // A whole try that could not judge the manifests and the composition is no pass; a failed composition is named beside // the failed tests (the group's check does as much), and a candidate of other heads is not the try's. for _, row := range []struct { name string cand *fakeTryCandidates comp meshwants.CompositionsByGroupHeadsValue state, said string }{ {name: "the indexer wrote no candidate", cand: &fakeTryCandidates{err: errors.New("the indexer wrote no candidate within 5m0s")}, state: TryError, said: "the indexer wrote no candidate within 5m0s"}, {name: "a node that gave no composition answer", cand: &fakeTryCandidates{}, comp: meshwants.CompositionsByGroupHeadsValue{State: meshdeliverycheckerclient.CompositionPass, Sdk: "x", Warnings: []string{"g14: " + nodeengineclient.NoAnswer}}, state: TryError, said: "g14: " + nodeengineclient.NoAnswer}, {name: "a composition refused on a node", cand: &fakeTryCandidates{}, comp: meshwants.CompositionsByGroupHeadsValue{State: meshdeliverycheckerclient.CompositionFail, Machine: "g14", Words: "not composed on g14: web: needs a state mail does not keep"}, state: TryFail, said: "not composed on g14: web: needs a state mail does not keep"}, } { kv := newMemKV() b := newBench(t) comps, _ := wholeBench(b, meshcatalog.CandidatesByGroupValue{}) b.ex.Candidates, comps.v = row.cand, row.comp if row.comp.State == "" { comps.v.State = meshdeliverycheckerclient.CompositionPass } id, _ := AskTry(memTries{kv}, tryTrunks, TryAsk{Repository: repoA, Head: head1}, time.Now()) var ends []string tr := &Tryer{Tries: kv, Judge: b.ex, Say: func(_ context.Context, event string, e checkermsg.Event) error { if event == checkermsg.Ended { ends = append(ends, e.State) } return nil }} raw, _, _, _ := kv.Get(ctx, id) tr.Put(ctx, id, raw) tr.Wait() if len(ends) != 1 || ends[0] != row.state { // every end says its state: error as well as fail t.Errorf("%s: ends said = %v, want once %s", row.name, ends, row.state) } if got := readTry(t, kv, id); got.State != row.state || !strings.Contains(strings.Join(got.Failed, " | ")+got.Said, row.said) { t.Errorf("%s: try = %s, failed %v, said %q, want %s naming %q", row.name, got.State, got.Failed, got.Said, row.state, row.said) } } // No build node is no error since R4g: the try's ask waits in the check stream until a node pulls it (ADR 0525 §4). } func TestATryLeftRunningIsTakenOver(t *testing.T) { ctx := context.Background() // Whether the start was said is in the try's own state (Told), so a checker that starts later says the end of a try it did // not start, whatever way the try ended. for _, row := range []struct { name string told bool failFirst bool // the stream does not acknowledge the first end said after func(t *testing.T, kv *memKV, id string) ends []string }{ {name: "the first publish of the end is not acknowledged", told: true, failFirst: true, ends: []string{TryPass}}, {name: "left running after its start was said", told: true, ends: []string{TryPass}}, {name: "left running with no start said", told: false, ends: []string{TryPass}}, {name: "ended by hand after the start was said", told: true, ends: []string{TryError}, after: func(t *testing.T, kv *memKV, id string) { if _, err := EndTry(endable{kv}, id, "stuck", time.Now()); err != nil { t.Fatal(err) } }}, } { t.Run(row.name, func(t *testing.T) { kv := newMemKV() id, _ := AskTry(memTries{kv}, tryTrunks, TryAsk{Repository: repoA, Head: head1}, time.Now()) // Left running by a checker that stopped. raw, rev, _, _ := kv.Get(ctx, id) cur, _ := DecodeTry(raw) cur.State, cur.Told = TryRunning, row.told data, _ := encodeTry(cur) if _, err := kv.Update(ctx, id, data, rev); err != nil { t.Fatal(err) } if row.after != nil { row.after(t, kv, id) } b := newBench(t) wholeBench(b, meshcatalog.CandidatesByGroupValue{}) var ends []string failed := false tr := &Tryer{Tries: kv, Judge: b.ex, Say: func(_ context.Context, event string, e checkermsg.Event) error { if event == checkermsg.Ended { if row.failFirst && !failed { failed = true return errors.New("the stream did not acknowledge") } ends = append(ends, e.State) } return nil }} tr.Reconsider(ctx) tr.Wait() if row.failFirst && !readTry(t, kv, id).Told { t.Errorf("an end the stream did not acknowledge cleared Told: it is never said again") } tr.Reconsider(ctx) // said once: the second look finds nothing owed tr.Wait() if row.after == nil { if got := readTry(t, kv, id); got.State != TryPass { t.Errorf("the try left running = %+v, want taken over and passed", got) } } if !slices.Equal(ends, row.ends) || readTry(t, kv, id).Told { t.Errorf("ends said = %v, want %v, and nothing owed after", ends, row.ends) } }) } // An ended try is kept TryKeep and removed by the next Reconsider after. t.Run("an ended try is kept seven days", func(t *testing.T) { kv := newMemKV() id, _ := AskTry(memTries{kv}, tryTrunks, TryAsk{Repository: repoA, Head: head1}, time.Now()) raw, rev, _, _ := kv.Get(ctx, id) cur, _ := DecodeTry(raw) ended := time.Now() cur.State, cur.Ended = TryPass, ended data, _ := encodeTry(cur) if _, err := kv.Update(ctx, id, data, rev); err != nil { t.Fatal(err) } now := ended.Add(TryKeep - time.Hour) tr := &Tryer{Tries: kv, Now: func() time.Time { return now }} tr.Reconsider(ctx) if _, _, found, _ := kv.Get(ctx, id); !found { t.Fatal("the try was removed before TryKeep") } now = ended.Add(TryKeep + time.Hour) tr.Reconsider(ctx) if _, _, found, _ := kv.Get(ctx, id); found { t.Error("the try was kept past TryKeep") } }) }