Files
mesh-catalog/modules/mesh-delivery/cmd/mesh-delivery/ready.go
T

490 lines
21 KiB
Go

package main
import (
"fmt"
"strings"
"time"
"git.novox.be/novox/mesh-sdk/go/asks"
)
// Merge on ready (novox/hq issue 722).
//
// The operator, 2026-10-14: "the reviewer task should merge the request or mark the MR as ready to merge in any case".
// Before, a reviewed pull request with green checks waited until a session called the forge's merge tool, and again
// after every check the tool asked because the trunk had moved. Now the independent reviewer marks the pull request
// ready at the head it approved — the forge label `mesh/ready` and a comment "ready at <full head sha>" (approving
// through the forge is the operator's) — and this owner merges it, through the forge's own merge tool
// (gitea_merge_pull_request), the one path every merge takes:
//
// - only at the head the mark names, and only while that is the pull request's head: a push after the mark clears
// it — the label taken off, and said on the pull request — and the new head needs a mark of its own;
// - only when its delivery is ready here: its check passed against its trunk's head. A verdict out of date is judged
// again by the merge tool, which asks the check against the trunk's head itself (novox/hq ADR 0323 §5); the merge
// is asked again once a minute, and lands on the passing verdict;
// - a group member only at its turn in the group's order, and only once the group's composed check passed for its
// members' heads as they are;
// - a check that failed merges nothing: it is said once on the pull request, and as a stalled line, which the
// controller raises as the condition `delivery.<id>.stalled`. So is a mark the merge tool refused for an hour.
//
// What this owner does with a mark is kept on the delivery (Ready), said in its log, and the merge's transitions name
// it as their author.
// ReadyMark is a pull request's ready mark as the forge reads it (the gitea module's gitea_open_pulls): the head the
// newest "ready at" comment names, who wrote it and when, and the label's id. Head is empty when the label is on and
// no comment names a head.
type ReadyMark struct {
Head string `json:"head"`
By string `json:"by,omitempty"`
At string `json:"at,omitempty"`
LabelID int `json:"label_id"`
}
// ReadyLabel is the label that marks a pull request ready.
const ReadyLabel = "mesh/ready"
// mergeRetryAfter is how long after the merge tool refused a marked head's merge it is asked again, when nothing
// happened meanwhile that decides whether it may merge (a verdict, an Approve, a group's turn are events, and ask it at
// once): a check it asked takes minutes, and a refusal costs the forge a few reads. A retry of a bounded step (askMerge).
const mergeRetryAfter = time.Minute
// markGrace is how long a mark not complete (the label, no "ready at" comment yet), and one this owner has just
// cleared, are left before they are judged again: the comment follows the label, and a clearing takes a moment.
const markGrace = time.Minute
// readyRefusedFor is how long the merge tool may refuse a marked head before it is said as stalled: longer than a
// check takes (the controller's checkTimeout is 45 minutes), so a check on its way is never said.
const readyRefusedFor = time.Hour
// ReadyRun is what this owner did with a delivery's ready mark: kept on the delivery and shown whole by `show`.
type ReadyRun struct {
// Head, By and MarkedAt are the mark: the head it names, who marked it, when.
Head string `json:"head"`
By string `json:"by,omitempty"`
MarkedAt string `json:"marked_at,omitempty"`
// Seen is when this owner first read the mark.
Seen time.Time `json:"seen"`
// Waits is what the merge waits on, when it waits: a verdict, the group's turn or its composed check.
Waits string `json:"waits,omitempty"`
// Tries is how many times the merge was asked; LastTry when, and Answer what the merge tool last answered.
Tries int `json:"tries,omitempty"`
LastTry time.Time `json:"last_try"`
Answer string `json:"answer,omitempty"`
// merging is whether a merge ask is on its way, outside the lock: not asked twice meanwhile. Not kept.
merging bool
// RefusedSince is when the merge tool began refusing it without a break.
RefusedSince *time.Time `json:"refused_since,omitempty"`
// Merged is when the merge tool merged it.
Merged *time.Time `json:"merged,omitempty"`
// Failed is why it is not merged and will not be on this head: its check failed. Said once on the pull request.
Failed string `json:"failed,omitempty"`
// OtherBase is the branch its pull request merges into when that is not main: never merged here, said once on the
// pull request (novox/hq ADR 0398 §4, issue 948).
OtherBase string `json:"other_base,omitempty"`
// The operator's Approve of the merge into main (novox/hq ADR 0398 §3, issue 948, approval.go). AskID is the ask
// waiting for an answer, Ask the ask as made (a warrant is held to it), Heads the heads it covers by delivery id (a
// group's members together), AskedAt when and Asks how many were made for this head.
AskID string `json:"ask_id,omitempty"`
Ask *asks.Ask `json:"ask,omitempty"`
Heads map[string]string `json:"heads,omitempty"`
AskedAt time.Time `json:"asked_at"`
Asks int `json:"asks,omitempty"`
// Approved and Declined are when the operator chose, Warrant what the router's warrant says of who and how; Refused
// why an answer was not taken (it did not hold to the ask).
Approved *time.Time `json:"approved,omitempty"`
Declined *time.Time `json:"declined,omitempty"`
Warrant string `json:"warrant,omitempty"`
Refused string `json:"refused,omitempty"`
}
// MergeBranch is the one branch this owner merges into (novox/hq ADR 0398 §4): a merge into any other is its author's.
const MergeBranch = "main"
// mergedBy is who a merge this owner made on a ready mark is said to be by, in its transitions.
func (r *ReadyRun) mergedBy() string {
if r == nil || r.Merged == nil {
return ""
}
by := byOwner + ", on the ready mark at " + shortOf(r.Head, 12)
if r.By != "" {
by += " by " + r.By
}
if r.Warrant != "" {
by += "; the merge into main approved: " + r.Warrant
}
return by
}
// marked is a pull request read with a ready mark.
type marked struct {
pull OpenPull
// cleared is when the mark was cleared, the label taken off: not cleared again within markGrace.
cleared time.Time
// firstEmpty is when a mark with no head (a label no "ready at" comment follows yet) was first read so: it is a
// mark not yet complete, and is cleared only once it has stayed so for markGrace (novox/hq issue 873).
firstEmpty time.Time
}
// pullKey is a pull request as a key: owner/repo#number.
func pullKey(p OpenPull) string {
return strings.ToLower(p.Owner+"/"+p.Repo) + fmt.Sprintf("#%d", p.Number)
}
// seeMarks keeps the open pull requests that carry a ready mark; under the lock. A pull request no longer open or
// no longer marked drops out.
func (h *Holder) seeMarks(open []OpenPull) {
seen := map[string]*marked{}
for _, p := range open {
if p.Ready == nil || p.HeadSHA == "" {
continue
}
k := pullKey(p)
m := &marked{pull: p}
if was := h.marks[k]; was != nil && was.pull.Ready != nil && was.pull.Ready.Head == p.Ready.Head &&
was.pull.HeadSHA == p.HeadSHA {
m.cleared = was.cleared
}
if p.Ready.Head == "" {
m.firstEmpty = h.Now()
if was := h.marks[k]; was != nil && was.pull.Ready != nil && was.pull.Ready.Head == "" &&
was.pull.HeadSHA == p.HeadSHA && !was.firstEmpty.IsZero() {
m.firstEmpty = was.firstEmpty
}
}
seen[k] = m
}
h.marks = seen
}
// sameFull is whether a mark names a head: the whole sha, or — read from an older forge module — at least twelve
// characters of it.
func sameFull(mark, head string) bool {
mark, head = strings.ToLower(mark), strings.ToLower(head)
return len(mark) >= 12 && strings.HasPrefix(head, mark)
}
// readyAsks works out, under the lock, what each ready mark needs: a mark cleared, a merge asked, a failure said.
// The asking happens outside the lock.
func (h *Holder) readyAsks(now time.Time) []func() {
var asks []func()
h.endStaleAsks()
for k, m := range h.marks {
p, mark := m.pull, *m.pull.Ready
repository := strings.ToLower(p.Owner + "/" + p.Repo)
if !sameFull(mark.Head, p.HeadSHA) {
if mark.Head == "" && now.Sub(m.firstEmpty) < markGrace {
h.lookAt(m.firstEmpty.Add(markGrace), "a ready label whose comment naming the head has not followed yet")
continue // the label first, its comment not yet: a mark not complete, not an old one
}
if d := h.deliveries[IDOf(repository, p.HeadSHA)]; d != nil && d.Ready != nil &&
sameFull(d.Ready.Head, p.HeadSHA) && now.Sub(d.Ready.Seen) < markGrace {
h.lookAt(d.Ready.Seen.Add(markGrace), "a mark made through mark-ready, its comment possibly not yet read")
continue // marked through mark-ready a moment ago: this read may predate its comment
}
if !m.cleared.IsZero() && now.Sub(m.cleared) < markGrace {
h.lookAt(m.cleared.Add(markGrace), "a stale ready mark whose clearing may not have taken")
continue
}
m.cleared = now
if old := h.deliveries[IDOf(repository, mark.Head)]; old != nil && old.Ready != nil && old.Ready.AskID != "" {
asks = append(asks, h.dropAsk(old.Ready.AskID, "its head moved on to "+shortOf(p.HeadSHA, 12)))
}
asks = append(asks, h.clearMark(k, p, mark))
continue
}
d := h.deliveries[IDOf(repository, p.HeadSHA)]
if d == nil || d.State.Final() || d.MergedAs != "" {
continue // not announced yet, or past its merge
}
if d.Ready == nil || d.Ready.Head != mark.Head || (d.Ready.Declined != nil && mark.At != d.Ready.MarkedAt) {
d.Ready = &ReadyRun{Head: mark.Head, By: mark.By, MarkedAt: mark.At, Seen: now}
h.Poke() // its check, if none is asked, is worth asking now (askCheck)
h.Logf("[mesh-delivery] %s is marked ready at %s by %s: merged once its check passes against its trunk's head",
d.ID, shortOf(mark.Head, 12), orNone(mark.By))
h.keep(d)
}
r := d.Ready
if r.Merged != nil {
continue
}
if r.Declined != nil {
// The operator declined: nothing is merged, and the mark is taken off (said once when it was declined).
if m.cleared.IsZero() || now.Sub(m.cleared) >= markGrace {
m.cleared = now
asks = append(asks, h.unmark(k, p, mark))
}
h.lookAt(m.cleared.Add(markGrace), "a declined ready mark whose clearing may not have taken")
continue
}
if d.State == Rejected {
if r.Failed == "" {
r.Failed = "its check failed"
if d.Check != nil {
r.Failed += ": " + oneLine(d.Check.Summary)
if j := d.Check.Judged(); j != "" {
r.Failed += " (" + j + ")"
}
}
r.Waits = ""
h.Logf("[mesh-delivery] %s is marked ready and not merged: %s", d.ID, r.Failed)
h.keep(d)
asks = append(asks, h.sayOn(d, fmt.Sprintf("mesh-delivery: **not merged.** This pull request is marked "+
"ready at %s, and %s. A new head, marked ready once reviewed, or a recheck that passes, merges it.",
mark.Head, r.Failed)))
}
continue
}
if p.Base != MergeBranch {
if r.OtherBase != p.Base {
r.OtherBase = p.Base
h.Logf("[mesh-delivery] %s is marked ready and not merged: it merges into %s, not %s", d.ID, p.Base, MergeBranch)
h.keep(d)
asks = append(asks, h.sayOn(d, fmt.Sprintf("mesh-delivery: **not merged.** This pull request is marked ready "+
"at %s, and it merges into %s. Only a merge into %s is this owner's; a merge into another branch is its "+
"author's (novox/hq ADR 0398 §4). The mark stays.", mark.Head, p.Base, MergeBranch)))
}
continue
}
r.OtherBase = ""
r.Failed = ""
waits := h.readyWaits(d)
if waits != r.Waits {
r.Waits = waits
if waits != "" {
h.Logf("[mesh-delivery] %s is marked ready and waits: %s", d.ID, waits)
}
h.keep(d)
}
var approvalAsk func()
if waits == "" && p.Base == MergeBranch {
waits, approvalAsk = h.approval(d, now)
}
if waits != r.Waits {
r.Waits = waits
h.keep(d)
}
if approvalAsk != nil {
asks = append(asks, approvalAsk)
}
if waits != "" || r.merging {
continue
}
if !r.LastTry.IsZero() && now.Sub(r.LastTry) < mergeRetryAfter {
h.lookAt(r.LastTry.Add(mergeRetryAfter), "a merge the forge refused")
continue
}
r.Tries++
r.LastTry, r.merging = now, true
h.keep(d)
asks = append(asks, h.askMerge(d.ID, p, mark.Head))
}
asks = append(h.cancels, asks...)
h.cancels = nil
return asks
}
// mergeAgainNow lets a marked delivery's merge be asked at once: something that decides whether it may merge happened (a
// verdict, an Approve, a group's turn), so the wait after a refusal does not hold it. Under the lock.
func (h *Holder) mergeAgainNow(d *Delivery) {
if d != nil && d.Ready != nil && d.Ready.Merged == nil {
d.Ready.LastTry = time.Time{}
}
}
// readyWaits is what a marked delivery's merge waits on; empty when it is asked now. A ready delivery is asked; so is
// a proposed one whose verdict went out of date and whose check nobody asked again — the merge tool asks it.
func (h *Holder) readyWaits(d *Delivery) string {
switch {
case d.State == Ready:
case d.State == Proposed && d.outOfDate() != "" && d.AskedAgain == nil:
case d.State == Proposed && d.AskedAgain != nil:
return "its check against the trunk's head is asked and has not answered"
default:
return "its check has not passed yet (" + string(d.State) + ")"
}
if d.Group == "" {
return ""
}
return h.groupTurnToMerge(d)
}
// groupTurnToMerge is why a group member may not merge yet; empty at its turn: the group's composed check passed for
// its members' heads as they are and is not out of date, and every member before it in the group's order merged.
func (h *Holder) groupTurnToMerge(d *Delivery) string {
g := h.groups[d.Group]
if g == nil {
return ""
}
members := h.membersOf(g)
if state, why := GroupState(g, members); state != "ready" && state != "delivering" {
return fmt.Sprintf("its group %s is %s: %s", g.ID, state, why)
}
if c := g.Check; c == nil || (c.Verdict != "pass" && c.Verdict != "warning") || c.OutOfDate != "" ||
!sameMembers(c.Commits, commitsOf(members)) {
return "its group " + g.ID + "'s composed check has not passed for its members' heads"
}
if !containsString(g.Order, d.ID) {
return "its group " + g.ID + "'s order is not worked out for it yet"
}
for _, id := range g.Order {
if id == d.ID {
return ""
}
if o := h.deliveries[id]; o != nil && o.MergedAs == "" && !o.State.Final() {
return "its group " + g.ID + "'s order merges " + id + " first"
}
}
return ""
}
// askMerge asks the forge's merge tool to merge a marked head, and keeps its answer.
func (h *Holder) askMerge(id string, p OpenPull, head string) func() {
return func() {
err := h.Forge.Merge(p.Owner, p.Repo, p.Number, head)
h.mu.Lock()
defer h.mu.Unlock()
d := h.deliveries[id]
if d == nil || d.Ready == nil {
return
}
r, now := d.Ready, h.Now()
r.merging = false
if err == nil {
// Issue 1025: the announcement is owed, kept with the delivery, so a death right after the merge
// still says it once.
d.Owed = append(d.Owed, Effect{Kind: EffectEmit, Event: "merged", Line: string(mustJSON(mergedEvent(d, p, head))), Since: now})
r.Merged, r.Answer, r.RefusedSince = &now, "merged", nil
h.Logf("[mesh-delivery] %s merged on its ready mark at %s, its check passed against its trunk's head", id,
shortOf(head, 12))
h.keep(d)
return
}
answer := oneLine(err.Error())
if r.RefusedSince == nil {
r.RefusedSince = &now
}
if answer != r.Answer {
h.Logf("[mesh-delivery] %s is marked ready, and the merge tool did not merge it (asked again in %s): %s", id,
mergeRetryAfter, answer)
}
h.lookAt(now.Add(mergeRetryAfter), "a merge the forge refused")
r.Answer = answer
h.keep(d)
}
}
// clearMark takes a mark off a pull request whose head moved after it, and says so on it.
func (h *Holder) clearMark(k string, p OpenPull, mark ReadyMark) func() {
return func() {
err := h.Forge.Unlabel(p.Owner, p.Repo, p.Number, mark.LabelID)
if err == nil {
err = h.Forge.Comment(p.Owner, p.Repo, p.Number, clearedSaying(mark, p))
}
if mark.Head == "" {
if err != nil {
h.Logf("[mesh-delivery] %s's ready label has no \"ready at <head>\" comment of a listed account after it, and could not be taken off yet: %v", k, err)
h.again("a ready label that could not be taken off")
return
}
h.Logf("[mesh-delivery] %s's ready label taken off: no \"ready at <head>\" comment of a listed account was written after it; mark-ready writes both in order", k)
} else {
if err != nil {
h.Logf("[mesh-delivery] %s's ready mark at %s names an older head than %s, and could not be cleared yet: %v",
k, shortOf(mark.Head, 12), shortOf(p.HeadSHA, 12), err)
h.again("a stale ready mark that could not be cleared")
return
}
h.Logf("[mesh-delivery] %s's ready mark at %s cleared: its head is now %s", k, shortOf(mark.Head, 12),
shortOf(p.HeadSHA, 12))
}
h.mu.Lock()
delete(h.marks, k)
h.mu.Unlock()
}
}
// unmark takes the mark off a pull request whose merge the operator declined (already said on it).
func (h *Holder) unmark(k string, p OpenPull, mark ReadyMark) func() {
return func() {
if err := h.Forge.Unlabel(p.Owner, p.Repo, p.Number, mark.LabelID); err != nil {
h.Logf("[mesh-delivery] %s's ready mark could not be taken off after the operator's Decline yet: %v", k, err)
h.again("a declined ready mark that could not be taken off")
return
}
h.mu.Lock()
delete(h.marks, k)
h.mu.Unlock()
}
}
// clearedSaying is what a cleared mark says on its pull request. A mark with no head is a label no "ready at <head>"
// comment of a listed account followed (novox/hq issue 873); one with a head names an older head than the pull's.
func clearedSaying(mark ReadyMark, p OpenPull) string {
if mark.Head == "" {
return "mesh-delivery: the label " + ReadyLabel + " is taken off: no \"ready at\" comment by a listed account " +
"follows both the label's last addition and the last push; the head is now " + p.HeadSHA + ". Review it and " +
"mark that head with `mesh-delivery.mark-ready`, which writes label and comment in the order the forge counts."
}
return fmt.Sprintf("mesh-delivery: the ready mark at %s is cleared: "+
"the head is now %s, pushed after it. Mark the new head ready once it is reviewed.", mark.Head, p.HeadSHA)
}
// sayOn comments on a delivery's pull request.
func (h *Holder) sayOn(d *Delivery, body string) func() {
owner, repo, number, id := d.Owner(), d.Repo(), d.Number, d.ID
return func() {
if err := h.Forge.Comment(owner, repo, number, body); err != nil {
h.Logf("[mesh-delivery] %s: could not say on its pull request: %v", id, err)
}
}
}
// readyLines are the marked heads that will not merge: a check that failed at once, a merge refused past its bound;
// as stalled lines, which the controller raises as conditions. Under the lock.
func (h *Holder) readyLines(now time.Time) []StalledLine {
var out []StalledLine
for _, d := range h.deliveries {
r := d.Ready
if r == nil || r.Merged != nil || d.MergedAs != "" || d.State.Final() {
continue
}
switch {
case r.Failed != "" && d.State == Rejected:
out = append(out, StalledLine{ID: d.ID, Number: d.Number, State: d.State, For: now.Sub(r.Seen).Round(time.Second).String(),
Bound: "0s", H2: "none: a new head marked ready, or a recheck that passes — the operator's",
Says: fmt.Sprintf("%s#%d is marked ready at %s and not merged: %s", d.Repository, d.Number,
shortOf(r.Head, 12), r.Failed)})
case r.Refused != "":
out = append(out, StalledLine{ID: d.ID, Number: d.Number, State: d.State, For: now.Sub(r.Seen).Round(time.Second).String(),
Bound: "0s", H2: "none: a new head marked ready, which asks the operator again — the operator's",
Says: fmt.Sprintf("%s#%d is marked ready at %s and not merged: the operator's answer %s", d.Repository,
d.Number, shortOf(r.Head, 12), r.Refused)})
case r.RefusedSince != nil && now.Sub(*r.RefusedSince) >= readyRefusedFor:
out = append(out, StalledLine{ID: d.ID, Number: d.Number, State: d.State,
For: now.Sub(*r.RefusedSince).Round(time.Second).String(), Bound: readyRefusedFor.String(),
H2: "none: the merge is asked again after each verdict, Approve or turn, and each minute — the operator's",
Says: fmt.Sprintf("%s#%d is marked ready at %s, and the merge tool has refused it since %s: %s", d.Repository,
d.Number, shortOf(r.Head, 12), r.RefusedSince.UTC().Format(time.RFC3339), r.Answer)})
}
}
return out
}
// mergedEvent is the module event `merged` (issue 1025): the pull request merged, its head and group, and the ask,
// option and heads the operator's Approve was bound to (the warrant the controller reads back from the router). A merge
// without an Approve names no ask and no option: no warrant is invented. Under the lock, before anything clears them.
func mergedEvent(d *Delivery, p OpenPull, head string) map[string]any {
r := d.Ready
heads := map[string]string{}
for k, v := range r.Heads {
heads[k] = v
}
ev := map[string]any{"owner": p.Owner, "repo": p.Repo, "number": p.Number, "head_sha": head, "group": d.Group,
"ask_id": "", "option": "", "heads": heads}
if r.Approved != nil {
ev["ask_id"], ev["option"] = r.AskID, OptionApprove
}
return ev
}