Gate settles pull requests closed with no event, looks again at a line's bound, and keeps a waiting head's condition over a restart (issue 1275, review round 1)

The conditions the gate now says would otherwise be wrong three ways: - A pull request closed or merged on the forge with no event heard stayed ready here for days (de-spiegel #4, closed 2026-10-17) and would raise a condition. After a complete read of the open list (shorter than the forge tool's 1000), each delivery not final whose pull request is missing is read from the forge and taken as its close or merge; a merge learnt so does not move source.main, as it is undated and could move it back. - A line that comes due on the clock (a state's bound, the author's bound, an answered refusal, an out-of-date refusal turning urgent after 10 minutes) waited for an unrelated event. The gate now asks a look at the earliest such time. - After a restart the heads waiting on the forge start their bound again, so their conditions were cleared and raised again later. A condition is cleared only once the open list was read since start, and not while the forge still shows its head open and waiting. Also: an empty path list ranks with a truncated one in the merge order, and a seat that keeps refusing conditions is retried on the clock 10 looks in a row, then at events only, said once in the log.
2026-10-25 18:32:19 +01:00
parent 67092161bd
commit 145c54d19c
4 changed files with 257 additions and 6 deletions
@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"sort"
 	"strings"
+	"time"

 	stdio "git.novox.be/novox/mesh-sdk/go"
 	"git.novox.be/novox/mesh-sdk/go/asks"
@@ -107,6 +108,16 @@ func (h *Holder) sayConditions() {
 		}
 		h.said = said
 	}
+	h.mu.Lock()
+	if at := h.nextStall(h.Now()); !at.IsZero() {
+		h.lookAt(at, "a delivery whose stalled line, or its severity, comes due with no event")
+	}
+	forgeRead := h.forgeRead
+	waitingOnForge := map[string]bool{}
+	for id := range h.unchecked {
+		waitingOnForge[id] = true
+	}
+	h.mu.Unlock()
 	lines := map[string]StalledLine{}
 	for _, l := range h.Stalled() {
 		// One condition for an id: the most severe of its lines.
@@ -134,10 +145,13 @@ func (h *Holder) sayConditions() {
 		h.Logf("[mesh-delivery] %s is said to the operator as %s: %s", id, severityOfLine(l), l.Says)
 		h.said[key], changed = said, true
 	}
+	// Cleared only on what was read: not before the forge's open pull requests were read once since the gate started (the
+	// heads waiting on the forge are kept in memory alone), and not while the forge still shows the head open and waiting:
+	// its line is gone only because its bound runs again from a later first sight, and it comes back with the same words.
 	var gone []string
 	for key := range h.said {
 		id := strings.TrimSuffix(strings.TrimPrefix(key, "mesh-delivery-gate.delivery."), ".stalled")
-		if _, ok := lines[id]; !ok {
+		if _, ok := lines[id]; !ok && forgeRead && !waitingOnForge[id] {
 			gone = append(gone, key)
 		}
 	}
@@ -160,7 +174,56 @@ func (h *Holder) sayConditions() {
 			h.Logf("[mesh-delivery] the conditions raised cannot be kept: %v", err)
 		}
 	}
-	if failed {
+	if !failed {
+		h.sayFailures = 0
+		return
+	}
+	// A seat that keeps refusing is tried again after askAgainAfter, up to sayTries looks in a row; then only at the
+	// next event, and said once, loudly, in the log (the operator hears nothing while the seat refuses).
+	h.sayFailures++
+	switch {
+	case h.sayFailures < sayTries:
 		h.again("a condition that could not be said to the operator")
+	case h.sayFailures == sayTries:
+		h.Logf("[mesh-delivery] NO CONDITION REACHES THE OPERATOR: the operator-channel seat refused %d looks in a row; "+
+			"tried again at each event from now on", sayTries)
 	}
 }
+
+// sayTries is how many looks in a row a condition that could not be said is tried again on the clock.
+const sayTries = 10
+
+// nextStall is the earliest time after now when a delivery's stalled line comes due, or its severity changes, with nothing
+// else happening: a state's bound, the author's bound of a rejected head, a head never announced, a refusal the gate
+// answers, and an out-of-date refusal turning urgent (1275/AC6). Zero when none. Under the lock.
+func (h *Holder) nextStall(now time.Time) time.Time {
+	var next time.Time
+	consider := func(at time.Time) {
+		if at.After(now) && (next.IsZero() || at.Before(next)) {
+			next = at
+		}
+	}
+	for _, d := range h.deliveries {
+		if d.State.Final() {
+			continue
+		}
+		if b, bounded := boundOf(d); bounded && b.For > 0 {
+			consider(boundSince(d).Add(b.For).Add(time.Second)) // pastBound is strictly after the bound
+		}
+		r := d.Ready
+		if r == nil || r.Merged != nil || d.MergedAs != "" {
+			continue
+		}
+		if d.Base == "" && d.Number != 0 {
+			consider(r.Seen.Add(headHeardBound))
+		}
+		if r.Failed != "" && d.State == Rejected {
+			consider(d.Since.Add(h.authorBound()))
+		}
+		if r.RefusedSince != nil {
+			consider(r.RefusedSince.Add(h.answeredRefusalBound(d.Repository)))
+			consider(r.RefusedSince.Add(outOfDateFor))
+		}
+	}
+	return next
+}
@@ -42,8 +42,12 @@ type Holder struct {
 	// Say says a condition to the operator, and Conditions keeps which the gate raised (conditions.go); nil says none.
 	Say        func(c asks.Condition) error
 	Conditions ConditionStore
-	// said is what Conditions keeps, read at the first look; touched by React alone.
-	said map[string]string
+	// said is what Conditions keeps, read at the first look; sayFailures the looks in a row a condition could not be
+	// said. Touched by React alone.
+	said        map[string]string
+	sayFailures int
+	// forgeRead is whether the forge's open pull requests were read since this owner started (SeeOpenPulls).
+	forgeRead bool
 	// cancels are asks to take back, gathered under the lock and run with the other asks of the look.
 	cancels []func()

@@ -958,7 +962,7 @@ func mergeRank(d *Delivery) int {
 	if strings.EqualFold(d.Repository, SDKRepository) {
 		return 0
 	}
-	if d.PathsTruncated {
+	if d.PathsTruncated || len(d.Paths) == 0 {
 		return 2 // its paths are not all known: it may pin, so it goes after those that do not
 	}
 	for _, p := range d.Paths {
@@ -542,10 +542,121 @@ func TestAnApprovedGroupIsHandedWholeInItsOrderAndAStopIsSaid(t *testing.T) {
 			told := hearConditions(w)
 			told.kept[conditionKeyOf("novox/app@0123456789ab")] = "urgent||"
 			w.settleAll()
+			if c := told.last("novox/app@0123456789ab"); c != nil {
+				t.Fatalf("cleared before the forge's open pull requests were read: %+v", c)
+			}
+			w.h.wantForge()
+			w.settleAll()
 			if c := told.last("novox/app@0123456789ab"); c == nil || c.State != asks.StateCleared || len(told.kept) != 0 {
 				t.Fatalf("not cleared: %+v, kept %v", c, told.kept)
 			}
 		}},
+		// Review round 1, blocker 1: a pull request closed or merged on the forge with no event heard (de-spiegel #4,
+		// closed 2026-10-17, ready here eight days later) settles at the next complete read of the open list, and no
+		// condition is raised for it; a merge learnt so does not move source.main.
+		{"a pull request gone from the open list settles and raises nothing", func(t *testing.T) {
+			for _, merged := range []bool{false, true} {
+				w := newWorld(t)
+				w.op.auto = false
+				const repo, head = "novox/de-spiegel", "fdcbaefb3123fdcbaefb3123fdcbaefb3123fdcb"
+				w.h.PullUpdated(pr(repo, 4, head, "feat/x", "src/a.go"))
+				w.h.Checked(judgedAt(repo, 4, head, "pass", "t1t1t1t1t1t1"))
+				w.settleAll()
+				w.forge.refuse = errors.New("novox/de-spiegel#4 merges into main, which no protection rule requiring the checks guards")
+				w.op.answer(w.op.open()[0], OptionApprove)
+				w.settleAll()
+				id := IDOf(repo, head)
+				if len(w.h.Stalled()) == 0 {
+					t.Fatalf("the refused head is not stalled to begin with")
+				}
+				p := ForgePull{Number: 4, State: "closed", HeadSHA: head, Base: "main"}
+				if merged {
+					p.Merged, p.MergeCommit = true, strings.Repeat("e", 40)
+				}
+				w.forge.pulls[repo+"#4"] = p
+				w.forge.open = []OpenPull{{Owner: "novox", Repo: "other", Number: 1, Base: "main", HeadSHA: "abab"}}
+				told := hearConditions(w)
+				moves := len(w.source.moves)
+				w.h.wantForge()
+				w.settleAll()
+				want := Closed
+				if merged {
+					want = Merged
+				}
+				if st := w.state(id); st != want {
+					t.Fatalf("merged %v: the delivery is %s, wanted %s", merged, st, want)
+				}
+				if c := told.last(id); c != nil {
+					t.Fatalf("merged %v: a condition was said for it: %+v", merged, c)
+				}
+				if len(w.source.moves) != moves {
+					t.Fatalf("merged %v: source.main moved for a merge learnt by its absence: %+v", merged, w.source.moves)
+				}
+			}
+		}},
+		// Review round 1, blocker 2 (1275/AC6): an out-of-date refusal at t0 with no other event is said urgent at t0+10
+		// minutes: the gate asks itself for a look then.
+		{"an out-of-date refusal turns urgent on the clock alone", func(t *testing.T) {
+			w := approvedGroup(t)
+			told := hearConditions(w)
+			allGreen(w)
+			w.forge.refuse = errAskedAgain
+			w.op.answer(w.op.open()[0], OptionApprove)
+			w.settleAll()
+			t0 := w.h.Now()
+			w.h.wanted.mu.Lock()
+			at := w.h.wanted.at
+			w.h.wanted.mu.Unlock()
+			if at.IsZero() || at.Sub(t0) > outOfDateFor {
+				t.Fatalf("no look is wanted by t0+%s: %v", outOfDateFor, at)
+			}
+			for i := 0; i < 5 && w.h.Now().Before(t0.Add(outOfDateFor)); i++ { // the timer fires a look, and nothing else happens
+				w.h.wanted.mu.Lock()
+				at = w.h.wanted.at
+				w.h.wanted.mu.Unlock()
+				if at.IsZero() {
+					break
+				}
+				w.later(at.Sub(w.h.Now()))
+				w.h.React()
+			}
+			c := told.last(IDOf(gCat, gHk))
+			if c == nil || c.State != asks.StateRaised || c.Severity != asks.SeverityUrgent {
+				t.Fatalf("at t0+%s with no event: %+v", w.h.Now().Sub(t0), c)
+			}
+		}},
+		// Review round 1, blocker 3: a gate that restarts while a head still waits on the forge does not clear its
+		// condition (the head's first sight is in memory alone, so its bound runs again), and does not say it again.
+		{"a restart keeps the condition of a head still waiting", func(t *testing.T) {
+			w := newWorld(t)
+			told := hearConditions(w)
+			const head = "bfe82315f42c8f4509d0598c32569a202d5eada6"
+			id := IDOf("novox/hq", head)
+			w.forge.open = []OpenPull{{Owner: "novox", Repo: "hq", Number: 243, Base: "main", HeadSHA: head, Required: []string{"mesh/repo-check"}}}
+			w.h.wantForge()
+			w.settleAll()
+			w.later(UncheckedAfter + time.Minute)
+			w.settleAll()
+			if c := told.last(id); c == nil || c.State != asks.StateRaised {
+				t.Fatalf("the unchecked head is not raised: %+v", c)
+			}
+			before := len(told.told)
+			again := newWorldOver(t, w.store, w.forge, w.h.Now()) // the gate restarts
+			again.h.Say, again.h.Conditions = told.say, told
+			again.h.wantForge()
+			again.settleAll()
+			again.later(time.Minute)
+			again.settleAll()
+			if len(told.told) != before {
+				t.Fatalf("the restart said something of a head still waiting: %+v", told.told[before:])
+			}
+			w.forge.open = nil // it closes
+			again.h.wantForge()
+			again.settleAll()
+			if c := told.last(id); c == nil || c.State != asks.StateCleared {
+				t.Fatalf("closed, and its condition is not cleared: %+v", c)
+			}
+		}},
 		// The kind behind the day's fault: a decision on a group's state named a word GroupState no longer says. Every word
 		// it says is one groupStatusState knows, and the merge goes on exactly in ready and merging.
 		{"every word of a group's state is decided on", func(t *testing.T) {
@@ -3,6 +3,8 @@ package main
 import (
 	"fmt"
 	"slices"
+	"sort"
+	"strconv"
 	"strings"
 	"time"
 )
@@ -175,8 +177,24 @@ func (h *Holder) SeeOpenPulls() error {
 		h.Logf("[mesh-delivery] the forge's open pull requests cannot be read: %v", err)
 		return err
 	}
+	for _, gone := range h.seeOpen(open) {
+		h.settleGone(gone)
+	}
+	return nil
+}
+
+// openPullsMost is the most open pull requests the forge's gitea_open_pulls answers: it reads 20 pages of 50 and says
+// nothing of the rest (mesh-catalog modules/gitea client.ts searchOpenPulls). A list that long may be cut, so what is
+// missing from it is not taken as closed.
+const openPullsMost = 20 * 50
+
+// seeOpen keeps what the forge's open pull requests say, and answers the deliveries whose pull request is not among them:
+// open here, closed on the forge with no event heard (a close or merge while the gate was away, or an event lost). Only
+// from a complete list. Takes the lock.
+func (h *Holder) seeOpen(open []OpenPull) []*Delivery {
 	h.mu.Lock()
 	defer h.mu.Unlock()
+	h.forgeRead = true
 	now := h.Now()
 	seen := map[string]unchecked{}
 	for _, p := range open {
@@ -215,7 +233,62 @@ func (h *Holder) SeeOpenPulls() error {
 		seen[id] = u
 	}
 	h.unchecked = seen
-	return nil
+	if len(open) >= openPullsMost {
+		return nil
+	}
+	listed := map[string]bool{}
+	for _, p := range open {
+		listed[strings.ToLower(p.Owner+"/"+p.Repo)+"#"+strconv.Itoa(p.Number)] = true
+	}
+	var gone []*Delivery
+	for _, d := range h.deliveries {
+		if d.State.Final() || d.MergedAs != "" || d.Number <= 0 || listed[strings.ToLower(d.Repository)+"#"+strconv.Itoa(d.Number)] {
+			continue
+		}
+		gone = append(gone, d)
+	}
+	sort.Slice(gone, func(i, j int) bool { return gone[i].ID < gone[j].ID })
+	return gone
+}
+
+// settleGone reads from the forge a pull request the open list does not hold, and takes its merge or its close as the
+// forge's event would have said it (issue 1275, review round 1: a pull request closed on 2026-10-17 was still ready here
+// eight days later). A merge learnt this way is past and undated: its delivery is merged, and source.main is not moved
+// for it (it could move it back); the trunk's newer merges and the catch-up at start move source.main. A forge that
+// cannot say changes nothing. Outside the lock.
+func (h *Holder) settleGone(d *Delivery) {
+	h.mu.Lock()
+	id, owner, repo, number, head, base, branch := d.ID, d.Owner(), d.Repo(), d.Number, d.Commit, d.Base, d.Branch
+	h.mu.Unlock()
+	fc, err := h.Forge.Statuses(owner, repo, number, head, base)
+	if err != nil || fc == nil || fc.Pull == nil {
+		if err != nil {
+			h.Logf("[mesh-delivery] %s is not among the forge's open pull requests, and the forge cannot say what became of it: %v", id, err)
+		}
+		return
+	}
+	p := fc.Pull
+	if p.State != "closed" || p.Number != number {
+		return // open again, a state the forge did not say, or not this pull request: the next reading says
+	}
+	ev := PullEvent{Owner: owner, Repo: repo, Number: number, Base: base, Head: branch, HeadSHA: head, HTMLURL: p.HTMLURL}
+	if p.Merged && p.MergeCommit != "" {
+		ev.MergeCommit, ev.Merged = p.MergeCommit, true
+		h.Logf("[mesh-delivery] %s is not among the forge's open pull requests: the forge says it merged as %s", id, shortOf(p.MergeCommit, 12))
+		h.PullMerged(ev)
+		h.mu.Lock()
+		if m := h.deliveries[id]; m != nil && m.MergedAs != "" {
+			m.InSource = true
+			if m.Ready != nil {
+				m.Ready.SourceMoved = true
+			}
+			h.keep(m)
+		}
+		h.mu.Unlock()
+		return
+	}
+	h.Logf("[mesh-delivery] %s is not among the forge's open pull requests: the forge says it was closed unmerged", id)
+	h.PullClosed(ev)
 }

 // started is whether the head's check started, rather than was never asked: it has any merge check, or a delivery