mesh-delivery-gate: an Approve is acted on or said within 60 s, on the bus and as an urgent condition

On 2026-10-30 the group feat/774-l3-reland was approved at 10:14 and nothing was handed for an hour with no word: six members kept a Decline of the earlier ask beside the new Approve, because the fix of 2026-10-29 (add6f143, cherry-picked before this commit) was reverted with L3 in c8240c38. Beside that fix, every Approve is now said as mesh-delivery-gate.approval {state: approved}, and an approved merge whose pass has not started 60 s after it is said once as {state: not-handed} with what it waits on and is an urgent stalled line until it starts (the operator: "you should know about it immediately"). One table test, two rows: a Decline kept before the Approve merges; a merge that cannot start is said within its bound.
This commit is contained in:
2026-10-30 11:40:09 +01:00
parent d6a069ae49
commit 10789ce9fd
7 changed files with 270 additions and 2 deletions
@@ -716,6 +716,9 @@ func (h *Holder) answered(key string, ans sdk.Answer) []func() {
}
h.keep(d)
}
if ans.Option == OptionApprove {
h.oweApproved(covered[0], now) // said on the bus at once, and again at its bound if its merge has not started (unacted.go)
}
return said
}
@@ -1,6 +1,7 @@
package main
import (
"fmt"
"strings"
"testing"
"time"
@@ -453,3 +454,92 @@ func TestADeclineOfAnEarlierAskDoesNotHoldTheApproveOfTheNext(t *testing.T) {
})
}
}
// 2026-10-30 (feat/774-l3-reland): approved at 10:14 and never handed, with no word. Each row an Approve: one whose runs
// were kept by a gate before the fix above, a Decline of an earlier ask beside the Approve, merges; one whose merge
// cannot start is said on the bus (mesh-delivery-gate.approval) and as an urgent line within approvalBound, and merges
// once what held it ends.
func TestAnApproveIsActedOnOrSaidWithinItsBound(t *testing.T) {
for _, row := range []struct {
name string
kept bool // the runs carry a Decline of an earlier ask, kept before answers named their ask
forgeDown bool // the members' statuses cannot be read, so the pass cannot start
}{
{name: "a Decline kept before the Approve, as the gate kept it before answers named their ask", kept: true},
{name: "an Approve whose merge cannot start is said within its bound", forgeDown: true},
} {
t.Run(row.name, func(t *testing.T) {
w := newWorld(t)
w.op.auto = false
heads := map[string]string{"novox/mesh-controller": "c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1", "novox/mesh-catalog": "d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2"}
paths := map[string]string{"novox/mesh-controller": "internal/a.go", "novox/mesh-catalog": "modules/app/module.json"}
numbers := map[string]int{"novox/mesh-controller": 1, "novox/mesh-catalog": 2}
trunks := map[string]string{"novox/mesh-controller": "t1t1t1t1t1t1", "novox/mesh-catalog": "t3t3t3t3t3t3"}
var open []OpenPull
for _, repo := range []string{"novox/mesh-controller", "novox/mesh-catalog"} {
w.h.PullUpdated(pr(repo, numbers[repo], heads[repo], "feat/x", paths[repo]))
w.h.Checked(judgedAt(repo, numbers[repo], heads[repo], "pass", trunks[repo]))
open = append(open, openPull(repo, numbers[repo], heads[repo]))
}
w.forge.open = open
w.h.SeeOpenPulls()
w.settleAll()
w.h.Checked(CheckedEvent{Group: "feat/x", ID: "check-1", Verdict: "pass", Summary: "both compose"})
w.settleAll()
asked := w.op.open()
if len(asked) != 1 {
t.Fatalf("the group is not asked once: %v", asked)
}
w.op.answer(asked[0], OptionApprove)
w.forge.mu.Lock()
w.forge.down = row.forgeDown
w.forge.mu.Unlock()
w.h.hearAsks()
if row.kept {
w.h.mu.Lock()
for _, d := range w.h.deliveries {
if d.Ready != nil && d.Ready.Approved != nil {
before := d.Ready.Approved.Add(-38 * time.Minute)
d.Ready.Declined, d.Ready.AnsweredAsk = &before, ""
}
}
w.h.mu.Unlock()
}
w.settleAll()
w.later(approvalBound + time.Second)
w.settleAll()
states := map[string]bool{}
for _, e := range w.said {
if e["event-kind"] == approvalEvent && e["group"] == "feat/x" {
states[fmt.Sprint(e["state"])] = true
}
}
if !states["approved"] {
t.Fatalf("the Approve is not said on the bus: %v", w.said)
}
var urgent []StalledLine
for _, l := range w.h.Stalled() {
if l.Bound == approvalBound.String() && l.Severity == "urgent" && strings.Contains(l.Says, "feat/x was approved") {
urgent = append(urgent, l)
}
}
if !row.forgeDown {
if len(w.forge.merges) != 2 || states["not-handed"] || len(urgent) != 0 {
t.Fatalf("approved, yet merged %v (said not handed: %v, lines %+v)", w.forge.merges, states["not-handed"], urgent)
}
return
}
if len(w.forge.merges) != 0 || !states["not-handed"] || len(urgent) != 1 {
t.Fatalf("an Approve held past its bound: merged %v, said not handed %v, urgent lines %+v", w.forge.merges, states["not-handed"], urgent)
}
w.forge.mu.Lock()
w.forge.down = false
w.forge.mu.Unlock()
w.later(askAgainAfter)
w.settleAll()
if len(w.forge.merges) != 2 {
t.Fatalf("the forge back, yet merged %v; stalled %+v", w.forge.merges, w.h.Stalled())
}
})
}
}
@@ -89,7 +89,7 @@ func TestItClaimsTheDeliverySeatAndFollowsWhatItHandles(t *testing.T) {
if !reflect.DeepEqual(m.Consumes, followed) || !slices.Contains(m.Consumes, "mesh-delivery-forge-bridge.forge.**") {
t.Fatalf("consumes %v, follows %v (the bridge's forge messages among them)", m.Consumes, followed)
}
if !reflect.DeepEqual(m.Emits, []string{"transition", "group", "merged"}) {
if !reflect.DeepEqual(m.Emits, []string{"transition", "group", "merged", "approval"}) {
t.Fatalf("emits %v", m.Emits)
}
// Nine buckets, and no `asked` (the teller asks, ADR 0437 §6): ADR 0450 §3, the conditions it raised (issue 1275) and the
@@ -144,6 +144,9 @@ type ReadyRun struct {
// cannot be handed, and why), NotHandedSince since when: said once in the log and as a stalled line, never silent.
NotHanded string `json:"not_handed,omitempty"`
NotHandedSince *time.Time `json:"not_handed_since,omitempty"`
// UnactedSaid is the ask whose Approve was said not acted on, approvalBound after it (unacted.go): said once per ask.
UnactedSaid string `json:"unacted_said,omitempty"`
}
// declined is whether the operator's Decline stands for this run: it answered the ask the run holds now, or the run holds
@@ -194,6 +197,9 @@ func (h *Holder) runOf(d *Delivery, now time.Time) *ReadyRun {
func (h *Holder) readyAsks(now time.Time) []func() {
var asks []func()
h.endStaleAsks()
// An Approve whose merge has not started is said, whatever holds it (unacted.go): read before this look hands, so a pass
// handed and given back within the last look (a read that failed) counts as not started.
h.sayUnacted(now)
gates := h.checkGates()
ids := make([]string, 0, len(h.deliveries))
for id := range h.deliveries {
@@ -0,0 +1,167 @@
package main
import (
"fmt"
"sort"
"strings"
"time"
)
// **An Approve is acted on, or said at once** (the operator, 2026-10-30: "an approval given by me must get handled
// accordingly and you should know about it immediately"). That morning the group feat/774-l3-reland was approved at 10:14
// and nothing was handed for an hour, with no word: six of its seven members kept a Decline of an earlier ask beside the
// new Approve (the fix of 2026-10-29, reverted with L3, came back with this file), readyAsks skipped them and handTurn
// returned nil for the seventh.
//
// So, whatever holds it: every Approve is said on the bus as `mesh-delivery-gate.approval` {state: approved} when it is
// taken; an approved merge whose pass has not started approvalBound after its Approve is said again as {state: not-handed}
// with what it waits on, once per ask, and is an urgent stalled line (so an urgent condition) until its pass starts, unless
// a line of its own says it already (a held group; a merge waiting for another delivery's check, said at its bound). A
// rejected delivery is its author's, not an Approve unacted. A session waits on `mesh-delivery-gate.approval` to hear both.
// approvalBound is how long after the operator's Approve a merge's pass may take to start before it is said.
const approvalBound = 60 * time.Second
// approvalEvent is the event family of an Approve taken and of an Approve not acted on.
const approvalEvent = "approval"
// unacted is an approved ask whose pass has not started: the run holding its Approve, its owner, its members, and what
// the gate says it waits on.
type unacted struct {
owner *Delivery
run *ReadyRun
name string
members []string
waits string
// checkHeld is whether a member waits for the check of other heads (ADR 0548 §2): said by the event at once, and as a
// line by turnLines at its own bound, not twice.
checkHeld bool
}
// approvalStands is whether a run's Approve still stands: no Decline after it, not merged, into the merge branch.
func approvalStands(d *Delivery) bool {
r := d.Ready
if r == nil || r.Approved == nil || r.AskID == "" || len(r.Heads) == 0 || r.Merged != nil || d.MergedAs != "" ||
d.State.Final() || d.State == Rejected || d.Base != MergeBranch {
return false
}
return r.Declined == nil || !r.Declined.After(*r.Approved)
}
// unactedAsks are the approved asks whose pass has not started, each once, in the order of their ids. A pass has started
// when a member it covers is handed (merging), was tried (refused or asked again), or merged. Under the lock.
func (h *Holder) unactedAsks() []unacted {
ids := make([]string, 0, len(h.deliveries))
for id := range h.deliveries {
ids = append(ids, id)
}
sort.Strings(ids)
seen := map[string]bool{}
var out []unacted
for _, id := range ids {
d := h.deliveries[id]
if !approvalStands(d) || seen[d.Ready.AskID] {
continue
}
if g := h.groups[d.Group]; d.Group != "" && g != nil && g.Hold != nil {
seen[d.Ready.AskID] = true
continue // a held group is said by its own urgent line, naming the member and why (groupLines)
}
r := d.Ready
seen[r.AskID] = true
u := unacted{owner: d, run: r, name: askGroup(d)}
started := false
var says []string
for mid := range r.Heads {
u.members = append(u.members, mid)
m := h.deliveries[mid]
if m == nil {
continue
}
if m.MergedAs != "" || (m.Ready != nil && (m.Ready.Merged != nil || m.Ready.merging || !m.Ready.LastTry.IsZero() ||
m.Ready.RefusedSince != nil)) {
started = true
break
}
if m.Ready != nil {
u.checkHeld = u.checkHeld || m.Ready.HeldSince != nil
if w := firstOf(m.Ready.NotHanded, m.Ready.Waits); w != "" {
says = append(says, mid+": "+w)
}
}
}
if started {
continue
}
sort.Strings(u.members)
sort.Strings(says)
u.waits = firstOf(strings.Join(says, "; "), "nothing the gate says: its merge was never handed")
out = append(out, u)
}
return out
}
// sayUnacted owes the event {state: not-handed} once per ask for every approved merge past approvalBound whose pass has
// not started, and wants a look at the bound of each not yet due. Under the lock; the event is owed, said by Flush.
func (h *Holder) sayUnacted(now time.Time) {
for _, u := range h.unactedAsks() {
due := u.run.Approved.Add(approvalBound)
if now.Before(due) {
h.lookAt(due, "an Approve whose merge has not started, said at its bound")
continue
}
if u.run.UnactedSaid == u.run.AskID {
continue
}
u.run.UnactedSaid = u.run.AskID
h.Logf("[mesh-delivery] %s is approved since %s and its merge is NOT handed: %s", u.name,
u.run.Approved.UTC().Format(time.RFC3339), u.waits)
u.owner.Owed = append(u.owner.Owed, Effect{Kind: EffectEmit, Event: approvalEvent, Since: now,
Line: string(mustJSON(approvalBody(u.owner, "not-handed", u.members, now, u.waits)))})
h.keep(u.owner)
}
}
// oweApproved owes the event {state: approved} for an ask the operator approved, on the first delivery it covers, and wants
// a look at its bound. Under the lock.
func (h *Holder) oweApproved(d *Delivery, now time.Time) {
r := d.Ready
members := make([]string, 0, len(r.Heads))
for id := range r.Heads {
members = append(members, id)
}
sort.Strings(members)
d.Owed = append(d.Owed, Effect{Kind: EffectEmit, Event: approvalEvent, Since: now,
Line: string(mustJSON(approvalBody(d, "approved", members, now, "")))})
h.lookAt(now.Add(approvalBound), "an Approve whose merge has not started, said at its bound")
}
// approvalBody is what `mesh-delivery-gate.approval` says: no secret, no address.
func approvalBody(d *Delivery, state string, members []string, now time.Time, waits string) map[string]any {
r := d.Ready
e := map[string]any{"state": state, "group": askGroup(d), "ask": r.AskID, "approved": r.Approved.UTC(), "at": now.UTC(),
"members": members, "by": r.Warrant}
if waits != "" {
e["waits"] = waits
e["for"] = now.Sub(*r.Approved).Round(time.Second).String()
}
return e
}
// unactedLines are the urgent stalled lines of approved merges whose pass has not started past approvalBound. Under the
// lock.
func (h *Holder) unactedLines(now time.Time) []StalledLine {
var out []StalledLine
for _, u := range h.unactedAsks() {
since := now.Sub(*u.run.Approved)
if since < approvalBound || u.checkHeld {
continue
}
out = append(out, StalledLine{ID: u.owner.ID, Number: u.owner.Number, State: u.owner.State,
For: since.Round(time.Second).String(), Bound: approvalBound.String(), Severity: "urgent",
H2: "none: the merge is handed once what it waits on ends; delivery_turns and mesh-delivery.show say what holds it",
Says: fmt.Sprintf("%s was approved at %s (%d pull requests) and its merge is not handed: %s", u.name,
u.run.Approved.UTC().Format(time.RFC3339), len(u.members), u.waits)})
}
return out
}
@@ -230,6 +230,7 @@ func (h *Holder) Stalled() []StalledLine {
out = append(out, h.readyLines(now)...)
out = append(out, h.turnLines(now)...) // an approved merge held past its bound by another delivery's check (ADR 0548 §4)
out = append(out, h.groupLines(now)...)
out = append(out, h.unactedLines(now)...) // an Approve whose merge has not started past its bound (unacted.go)
sort.Slice(out, func(i, j int) bool { return out[i].ID < out[j].ID })
return out
}
+2 -1
View File
@@ -33,7 +33,8 @@
"emits": [
"transition",
"group",
"merged"
"merged",
"approval"
],
"uses": [
"operator-channel"