diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/conditions.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/conditions.go index 84bb195e..bbce1e24 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/conditions.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/conditions.go @@ -5,6 +5,7 @@ import ( "fmt" "sort" "strings" + "time" stdio "git.novox.be/novox/mesh-sdk/go" "git.novox.be/novox/mesh-sdk/go/asks" @@ -107,6 +108,16 @@ func (h *Holder) sayConditions() { } h.said = said } + h.mu.Lock() + if at := h.nextStall(h.Now()); !at.IsZero() { + h.lookAt(at, "a delivery whose stalled line, or its severity, comes due with no event") + } + forgeRead := h.forgeRead + waitingOnForge := map[string]bool{} + for id := range h.unchecked { + waitingOnForge[id] = true + } + h.mu.Unlock() lines := map[string]StalledLine{} for _, l := range h.Stalled() { // One condition for an id: the most severe of its lines. @@ -134,10 +145,13 @@ func (h *Holder) sayConditions() { h.Logf("[mesh-delivery] %s is said to the operator as %s: %s", id, severityOfLine(l), l.Says) h.said[key], changed = said, true } + // Cleared only on what was read: not before the forge's open pull requests were read once since the gate started (the + // heads waiting on the forge are kept in memory alone), and not while the forge still shows the head open and waiting: + // its line is gone only because its bound runs again from a later first sight, and it comes back with the same words. var gone []string for key := range h.said { id := strings.TrimSuffix(strings.TrimPrefix(key, "mesh-delivery-gate.delivery."), ".stalled") - if _, ok := lines[id]; !ok { + if _, ok := lines[id]; !ok && forgeRead && !waitingOnForge[id] { gone = append(gone, key) } } @@ -160,7 +174,56 @@ func (h *Holder) sayConditions() { h.Logf("[mesh-delivery] the conditions raised cannot be kept: %v", err) } } - if failed { + if !failed { + h.sayFailures = 0 + return + } + // A seat that keeps refusing is tried again after askAgainAfter, up to sayTries looks in a row; then only at the + // next event, and said once, loudly, in the log (the operator hears nothing while the seat refuses). + h.sayFailures++ + switch { + case h.sayFailures < sayTries: h.again("a condition that could not be said to the operator") + case h.sayFailures == sayTries: + h.Logf("[mesh-delivery] NO CONDITION REACHES THE OPERATOR: the operator-channel seat refused %d looks in a row; "+ + "tried again at each event from now on", sayTries) } } + +// sayTries is how many looks in a row a condition that could not be said is tried again on the clock. +const sayTries = 10 + +// nextStall is the earliest time after now when a delivery's stalled line comes due, or its severity changes, with nothing +// else happening: a state's bound, the author's bound of a rejected head, a head never announced, a refusal the gate +// answers, and an out-of-date refusal turning urgent (1275/AC6). Zero when none. Under the lock. +func (h *Holder) nextStall(now time.Time) time.Time { + var next time.Time + consider := func(at time.Time) { + if at.After(now) && (next.IsZero() || at.Before(next)) { + next = at + } + } + for _, d := range h.deliveries { + if d.State.Final() { + continue + } + if b, bounded := boundOf(d); bounded && b.For > 0 { + consider(boundSince(d).Add(b.For).Add(time.Second)) // pastBound is strictly after the bound + } + r := d.Ready + if r == nil || r.Merged != nil || d.MergedAs != "" { + continue + } + if d.Base == "" && d.Number != 0 { + consider(r.Seen.Add(headHeardBound)) + } + if r.Failed != "" && d.State == Rejected { + consider(d.Since.Add(h.authorBound())) + } + if r.RefusedSince != nil { + consider(r.RefusedSince.Add(h.answeredRefusalBound(d.Repository))) + consider(r.RefusedSince.Add(outOfDateFor)) + } + } + return next +} diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/holder.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/holder.go index 851305e0..8c4779ca 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/holder.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/holder.go @@ -42,8 +42,12 @@ type Holder struct { // Say says a condition to the operator, and Conditions keeps which the gate raised (conditions.go); nil says none. Say func(c asks.Condition) error Conditions ConditionStore - // said is what Conditions keeps, read at the first look; touched by React alone. - said map[string]string + // said is what Conditions keeps, read at the first look; sayFailures the looks in a row a condition could not be + // said. Touched by React alone. + said map[string]string + sayFailures int + // forgeRead is whether the forge's open pull requests were read since this owner started (SeeOpenPulls). + forgeRead bool // cancels are asks to take back, gathered under the lock and run with the other asks of the look. cancels []func() @@ -958,7 +962,7 @@ func mergeRank(d *Delivery) int { if strings.EqualFold(d.Repository, SDKRepository) { return 0 } - if d.PathsTruncated { + if d.PathsTruncated || len(d.Paths) == 0 { return 2 // its paths are not all known: it may pin, so it goes after those that do not } for _, p := range d.Paths { diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/issue1275_test.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/issue1275_test.go index 451ea831..91d67e61 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/issue1275_test.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/issue1275_test.go @@ -542,10 +542,121 @@ func TestAnApprovedGroupIsHandedWholeInItsOrderAndAStopIsSaid(t *testing.T) { told := hearConditions(w) told.kept[conditionKeyOf("novox/app@0123456789ab")] = "urgent||" w.settleAll() + if c := told.last("novox/app@0123456789ab"); c != nil { + t.Fatalf("cleared before the forge's open pull requests were read: %+v", c) + } + w.h.wantForge() + w.settleAll() if c := told.last("novox/app@0123456789ab"); c == nil || c.State != asks.StateCleared || len(told.kept) != 0 { t.Fatalf("not cleared: %+v, kept %v", c, told.kept) } }}, + // Review round 1, blocker 1: a pull request closed or merged on the forge with no event heard (de-spiegel #4, + // closed 2026-10-17, ready here eight days later) settles at the next complete read of the open list, and no + // condition is raised for it; a merge learnt so does not move source.main. + {"a pull request gone from the open list settles and raises nothing", func(t *testing.T) { + for _, merged := range []bool{false, true} { + w := newWorld(t) + w.op.auto = false + const repo, head = "novox/de-spiegel", "fdcbaefb3123fdcbaefb3123fdcbaefb3123fdcb" + w.h.PullUpdated(pr(repo, 4, head, "feat/x", "src/a.go")) + w.h.Checked(judgedAt(repo, 4, head, "pass", "t1t1t1t1t1t1")) + w.settleAll() + w.forge.refuse = errors.New("novox/de-spiegel#4 merges into main, which no protection rule requiring the checks guards") + w.op.answer(w.op.open()[0], OptionApprove) + w.settleAll() + id := IDOf(repo, head) + if len(w.h.Stalled()) == 0 { + t.Fatalf("the refused head is not stalled to begin with") + } + p := ForgePull{Number: 4, State: "closed", HeadSHA: head, Base: "main"} + if merged { + p.Merged, p.MergeCommit = true, strings.Repeat("e", 40) + } + w.forge.pulls[repo+"#4"] = p + w.forge.open = []OpenPull{{Owner: "novox", Repo: "other", Number: 1, Base: "main", HeadSHA: "abab"}} + told := hearConditions(w) + moves := len(w.source.moves) + w.h.wantForge() + w.settleAll() + want := Closed + if merged { + want = Merged + } + if st := w.state(id); st != want { + t.Fatalf("merged %v: the delivery is %s, wanted %s", merged, st, want) + } + if c := told.last(id); c != nil { + t.Fatalf("merged %v: a condition was said for it: %+v", merged, c) + } + if len(w.source.moves) != moves { + t.Fatalf("merged %v: source.main moved for a merge learnt by its absence: %+v", merged, w.source.moves) + } + } + }}, + // Review round 1, blocker 2 (1275/AC6): an out-of-date refusal at t0 with no other event is said urgent at t0+10 + // minutes: the gate asks itself for a look then. + {"an out-of-date refusal turns urgent on the clock alone", func(t *testing.T) { + w := approvedGroup(t) + told := hearConditions(w) + allGreen(w) + w.forge.refuse = errAskedAgain + w.op.answer(w.op.open()[0], OptionApprove) + w.settleAll() + t0 := w.h.Now() + w.h.wanted.mu.Lock() + at := w.h.wanted.at + w.h.wanted.mu.Unlock() + if at.IsZero() || at.Sub(t0) > outOfDateFor { + t.Fatalf("no look is wanted by t0+%s: %v", outOfDateFor, at) + } + for i := 0; i < 5 && w.h.Now().Before(t0.Add(outOfDateFor)); i++ { // the timer fires a look, and nothing else happens + w.h.wanted.mu.Lock() + at = w.h.wanted.at + w.h.wanted.mu.Unlock() + if at.IsZero() { + break + } + w.later(at.Sub(w.h.Now())) + w.h.React() + } + c := told.last(IDOf(gCat, gHk)) + if c == nil || c.State != asks.StateRaised || c.Severity != asks.SeverityUrgent { + t.Fatalf("at t0+%s with no event: %+v", w.h.Now().Sub(t0), c) + } + }}, + // Review round 1, blocker 3: a gate that restarts while a head still waits on the forge does not clear its + // condition (the head's first sight is in memory alone, so its bound runs again), and does not say it again. + {"a restart keeps the condition of a head still waiting", func(t *testing.T) { + w := newWorld(t) + told := hearConditions(w) + const head = "bfe82315f42c8f4509d0598c32569a202d5eada6" + id := IDOf("novox/hq", head) + w.forge.open = []OpenPull{{Owner: "novox", Repo: "hq", Number: 243, Base: "main", HeadSHA: head, Required: []string{"mesh/repo-check"}}} + w.h.wantForge() + w.settleAll() + w.later(UncheckedAfter + time.Minute) + w.settleAll() + if c := told.last(id); c == nil || c.State != asks.StateRaised { + t.Fatalf("the unchecked head is not raised: %+v", c) + } + before := len(told.told) + again := newWorldOver(t, w.store, w.forge, w.h.Now()) // the gate restarts + again.h.Say, again.h.Conditions = told.say, told + again.h.wantForge() + again.settleAll() + again.later(time.Minute) + again.settleAll() + if len(told.told) != before { + t.Fatalf("the restart said something of a head still waiting: %+v", told.told[before:]) + } + w.forge.open = nil // it closes + again.h.wantForge() + again.settleAll() + if c := told.last(id); c == nil || c.State != asks.StateCleared { + t.Fatalf("closed, and its condition is not cleared: %+v", c) + } + }}, // The kind behind the day's fault: a decision on a group's state named a word GroupState no longer says. Every word // it says is one groupStatusState knows, and the merge goes on exactly in ready and merging. {"every word of a group's state is decided on", func(t *testing.T) { diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/unchecked.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/unchecked.go index 2c9d92ba..17132c32 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/unchecked.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/unchecked.go @@ -3,6 +3,8 @@ package main import ( "fmt" "slices" + "sort" + "strconv" "strings" "time" ) @@ -175,8 +177,24 @@ func (h *Holder) SeeOpenPulls() error { h.Logf("[mesh-delivery] the forge's open pull requests cannot be read: %v", err) return err } + for _, gone := range h.seeOpen(open) { + h.settleGone(gone) + } + return nil +} + +// openPullsMost is the most open pull requests the forge's gitea_open_pulls answers: it reads 20 pages of 50 and says +// nothing of the rest (mesh-catalog modules/gitea client.ts searchOpenPulls). A list that long may be cut, so what is +// missing from it is not taken as closed. +const openPullsMost = 20 * 50 + +// seeOpen keeps what the forge's open pull requests say, and answers the deliveries whose pull request is not among them: +// open here, closed on the forge with no event heard (a close or merge while the gate was away, or an event lost). Only +// from a complete list. Takes the lock. +func (h *Holder) seeOpen(open []OpenPull) []*Delivery { h.mu.Lock() defer h.mu.Unlock() + h.forgeRead = true now := h.Now() seen := map[string]unchecked{} for _, p := range open { @@ -215,7 +233,62 @@ func (h *Holder) SeeOpenPulls() error { seen[id] = u } h.unchecked = seen - return nil + if len(open) >= openPullsMost { + return nil + } + listed := map[string]bool{} + for _, p := range open { + listed[strings.ToLower(p.Owner+"/"+p.Repo)+"#"+strconv.Itoa(p.Number)] = true + } + var gone []*Delivery + for _, d := range h.deliveries { + if d.State.Final() || d.MergedAs != "" || d.Number <= 0 || listed[strings.ToLower(d.Repository)+"#"+strconv.Itoa(d.Number)] { + continue + } + gone = append(gone, d) + } + sort.Slice(gone, func(i, j int) bool { return gone[i].ID < gone[j].ID }) + return gone +} + +// settleGone reads from the forge a pull request the open list does not hold, and takes its merge or its close as the +// forge's event would have said it (issue 1275, review round 1: a pull request closed on 2026-10-17 was still ready here +// eight days later). A merge learnt this way is past and undated: its delivery is merged, and source.main is not moved +// for it (it could move it back); the trunk's newer merges and the catch-up at start move source.main. A forge that +// cannot say changes nothing. Outside the lock. +func (h *Holder) settleGone(d *Delivery) { + h.mu.Lock() + id, owner, repo, number, head, base, branch := d.ID, d.Owner(), d.Repo(), d.Number, d.Commit, d.Base, d.Branch + h.mu.Unlock() + fc, err := h.Forge.Statuses(owner, repo, number, head, base) + if err != nil || fc == nil || fc.Pull == nil { + if err != nil { + h.Logf("[mesh-delivery] %s is not among the forge's open pull requests, and the forge cannot say what became of it: %v", id, err) + } + return + } + p := fc.Pull + if p.State != "closed" || p.Number != number { + return // open again, a state the forge did not say, or not this pull request: the next reading says + } + ev := PullEvent{Owner: owner, Repo: repo, Number: number, Base: base, Head: branch, HeadSHA: head, HTMLURL: p.HTMLURL} + if p.Merged && p.MergeCommit != "" { + ev.MergeCommit, ev.Merged = p.MergeCommit, true + h.Logf("[mesh-delivery] %s is not among the forge's open pull requests: the forge says it merged as %s", id, shortOf(p.MergeCommit, 12)) + h.PullMerged(ev) + h.mu.Lock() + if m := h.deliveries[id]; m != nil && m.MergedAs != "" { + m.InSource = true + if m.Ready != nil { + m.Ready.SourceMoved = true + } + h.keep(m) + } + h.mu.Unlock() + return + } + h.Logf("[mesh-delivery] %s is not among the forge's open pull requests: the forge says it was closed unmerged", id) + h.PullClosed(ev) } // started is whether the head's check started, rather than was never asked: it has any merge check, or a delivery