mesh-catalog/modules/mesh-delivery/cmd/mesh-delivery/ready.go

package main

import (
	"fmt"
	"sort"
	"strings"
	"time"

	"git.novox.be/novox/mesh-sdk/go/asks"
)

// The Approve is asked as soon as the checks pass (novox/hq ADR 0421, issue 1022).
//
// Before, a reviewer marked a pull request ready (a label and a comment) and only then was the operator's Approve asked
// (issue 722, ADR 0364). The review now finishes before the push (playbook 07) and its record, a line in the pull
// request's description, is given when the pull request is opened, so the mark repeated what the push already meant and
// added a step and a wait. This owner asks the operator's Approve itself, within the event that makes it possible: the
// verdict that passes, the commit status that sets the checks. Then:
//
//   - the Approve is the only act: nothing is merged before it, and nothing else is needed after it (no label, comment
//     or second Approve); it is asked once for a head, and once for a group's heads together;
//   - it merges through the forge's merge tool (gitea_merge_pull_request), the one path every merge takes, at the head
//     the Approve names, handing the tool the heads the Approve covers (ADR 0421 §9);
//   - only when its delivery is ready here: its check passed against its trunk's head. A verdict out of date is asked
//     again by this owner at once (askFresh), and an Approve already asked stays through a trunk move (ADR 0421 §3): the
//     merge is asked again on the passing verdict, with no second Approve;
//   - a group member only once the group's composed check passed for its members' heads as they are;
//   - a push ends the Approve and says so once on the pull request (ADR 0421 §6); the new head is asked on its own;
//   - a check that failed merges nothing: it is said once on the pull request, and as a stalled line, which the
//     controller raises as the condition `delivery.<id>.stalled`. So is a merge the tool refused for an hour.
//
// The review record is read and put in the ask; it is never a gate (ADR 0421 §5). What this owner does is kept on the
// delivery (Ready), said in its log, and the merge's transitions name the Approve as their warrant.

// mergeRetryAfter is how long after the merge tool refused a head's merge it is asked again, when nothing happened
// meanwhile that decides whether it may merge (a verdict, an Approve, a group's composed check are events, and ask it at
// once): a check it asked takes minutes, and a refusal costs the forge a few reads. A retry of a bounded step (askMerge).
const mergeRetryAfter = time.Minute

// readyRefusedFor is how long the merge tool may refuse an approved head before it is said as stalled: longer than a
// check takes (the controller's checkTimeout is 45 minutes), so a check on its way is never said.
const readyRefusedFor = time.Hour

// ReadyRun is what this owner did with a delivery's Approve and merge: kept on the delivery and shown whole by `show`.
type ReadyRun struct {
	// Head is the head the run is for.
	Head string `json:"head"`
	// Seen is when this owner first made the run: the checks passed.
	Seen time.Time `json:"seen"`
	// Waits is what the merge waits on, when it waits: a verdict, the group's composed check or the operator's Approve.
	Waits string `json:"waits,omitempty"`
	// Tries is how many times the merge was asked; LastTry when, and Answer what the merge tool last answered.
	Tries   int       `json:"tries,omitempty"`
	LastTry time.Time `json:"last_try"`
	Answer  string    `json:"answer,omitempty"`
	// merging is whether a merge ask is on its way, outside the lock: not asked twice meanwhile. Not kept.
	merging bool
	// RefusedSince is when the merge tool began refusing it without a break.
	RefusedSince *time.Time `json:"refused_since,omitempty"`
	// Merged is when the merge tool merged it.
	Merged *time.Time `json:"merged,omitempty"`
	// Failed is why it is not merged and will not be on this head: its check failed. Said once on the pull request.
	Failed string `json:"failed,omitempty"`
	// OtherBase is the branch its pull request merges into when that is not main: never asked, never merged here, said
	// once on the pull request (novox/hq ADR 0398 §4, issue 948).
	OtherBase string `json:"other_base,omitempty"`

	// The operator's Approve of the merge into main (novox/hq ADR 0398 §3, issue 948, approval.go). AskID is the ask
	// waiting for an answer, Ask the ask as made (a warrant is held to it), Heads the heads it covers by delivery id (a
	// group's members together), AskedAt when and Asks how many were made for this head.
	AskID   string            `json:"ask_id,omitempty"`
	Ask     *asks.Ask         `json:"ask,omitempty"`
	Heads   map[string]string `json:"heads,omitempty"`
	AskedAt time.Time         `json:"asked_at"`
	Asks    int               `json:"asks,omitempty"`
	// Approved and Declined are when the operator chose, Warrant what the router's warrant says of who and how; Refused
	// why an answer was not taken (it did not hold to the ask).
	Approved *time.Time `json:"approved,omitempty"`
	Declined *time.Time `json:"declined,omitempty"`
	Warrant  string     `json:"warrant,omitempty"`
	Refused  string     `json:"refused,omitempty"`
}

// MergeBranch is the one branch this owner merges into (novox/hq ADR 0398 §4): a merge into any other is its author's.
const MergeBranch = "main"

// mergedBy is who a merge this owner made on an Approve is said to be by, in its transitions.
func (r *ReadyRun) mergedBy() string {
	if r == nil || r.Merged == nil {
		return ""
	}
	by := byOwner + ", on the operator's Approve at " + shortOf(r.Head, 12)
	if r.Warrant != "" {
		by += "; the merge into main approved: " + r.Warrant
	}
	return by
}

// pullKey is a pull request as a key: owner/repo#number.
func pullKey(p OpenPull) string {
	return strings.ToLower(p.Owner+"/"+p.Repo) + fmt.Sprintf("#%d", p.Number)
}

// runOf is the delivery's run, made when it has none.
func (h *Holder) runOf(d *Delivery, now time.Time) *ReadyRun {
	if d.Ready == nil {
		d.Ready = &ReadyRun{Head: d.Commit, Seen: now}
		h.keep(d)
	}
	return d.Ready
}

// readyAsks works out, under the lock, what each delivery whose checks passed needs: the operator asked, a merge asked,
// a failure said, an out of date verdict's check asked again. The asking happens outside the lock.
func (h *Holder) readyAsks(now time.Time) []func() {
	var asks []func()
	h.endStaleAsks()
	ids := make([]string, 0, len(h.deliveries))
	for id := range h.deliveries {
		ids = append(ids, id)
	}
	sort.Strings(ids)
	for _, id := range ids {
		d := h.deliveries[id]
		if d.Number == 0 || d.State.Final() || d.MergedAs != "" {
			continue // not of a pull request, or past its merge
		}
		if d.Ready == nil && d.State == Ready {
			h.runOf(d, now)
			h.Poke() // its check, if none is asked, is worth asking now (askCheck)
			h.Logf("[mesh-delivery] %s: its checks passed; the operator's Approve of its merge into main is asked once it may merge", d.ID)
		}
		if a := h.sayAfterLine(d); a != nil {
			asks = append(asks, a)
		}
		if a := h.askFresh(d, now); a != nil {
			asks = append(asks, a)
		}
		r := d.Ready
		if r == nil || r.Merged != nil || r.Declined != nil {
			continue
		}
		if d.State == Rejected {
			if r.Failed == "" {
				r.Failed = "its check failed"
				if d.Check != nil {
					r.Failed += ": " + oneLine(d.Check.Summary)
					if j := d.Check.Judged(); j != "" {
						r.Failed += " (" + j + ")"
					}
				}
				r.Waits = ""
				h.Logf("[mesh-delivery] %s passed its checks and is not merged: %s", d.ID, r.Failed)
				h.keep(d)
				asks = append(asks, h.sayOn(d, fmt.Sprintf("mesh-delivery: **not merged.** This pull request's checks passed at "+
					"%s, and now %s. A new head, or a recheck that passes, asks the operator again.", d.Commit, r.Failed)))
			}
			continue
		}
		if d.Base != MergeBranch {
			if r.OtherBase != d.Base {
				r.OtherBase = d.Base
				h.Logf("[mesh-delivery] %s passed its checks and is not asked: it merges into %s, not %s", d.ID, d.Base, MergeBranch)
				h.keep(d)
				asks = append(asks, h.sayOn(d, fmt.Sprintf("mesh-delivery: **not merged.** This pull request's checks passed at "+
					"%s, and it merges into %s. Only a merge into %s is this owner's; a merge into another branch is its "+
					"author's (novox/hq ADR 0398 §4).", d.Commit, d.Base, MergeBranch)))
			}
			continue
		}
		r.OtherBase = ""
		r.Failed = ""
		waits := h.readyWaits(d)
		if waits != r.Waits {
			r.Waits = waits
			if waits != "" {
				h.Logf("[mesh-delivery] %s waits: %s", d.ID, waits)
			}
			h.keep(d)
		}
		var approvalAsk func()
		if waits == "" {
			waits, approvalAsk = h.approval(d, now)
		}
		if waits != r.Waits {
			r.Waits = waits
			h.keep(d)
		}
		if approvalAsk != nil {
			asks = append(asks, approvalAsk)
		}
		if waits != "" || r.merging {
			continue
		}
		if !r.LastTry.IsZero() && now.Sub(r.LastTry) < mergeRetryAfter {
			h.lookAt(r.LastTry.Add(mergeRetryAfter), "a merge the forge refused")
			continue
		}
		r.Tries++
		r.LastTry, r.merging = now, true
		h.keep(d)
		asks = append(asks, h.askMerge(d.ID, r.Heads))
	}
	asks = append(h.cancels, asks...)
	h.cancels = nil
	return asks
}

// sayAfterLine tells a pull request, once, that its description holds an `after: <repository>` line, which is not read
// (novox/hq ADR 0419 §2): the order of a delivery comes from the module graph alone, and no refusal asks for the line.
// Once per pull request, not per head. Under the lock; the comment is said outside it.
func (h *Holder) sayAfterLine(d *Delivery) func() {
	if len(d.After) == 0 || d.AfterSaid {
		return nil
	}
	d.AfterSaid = true
	for _, o := range h.deliveries {
		if o != d && o.Number == d.Number && o.Repository == d.Repository && o.AfterSaid {
			h.keep(d)
			return nil // said on this pull request for an earlier head
		}
	}
	h.keep(d)
	h.Logf("[mesh-delivery] %s: its description holds an after: line (%s), which is not read", d.ID, strings.Join(d.After, ", "))
	return h.sayOn(d, "mesh-delivery: this description holds an `after:` line ("+strings.Join(d.After, ", ")+"). It is not read: "+
		"the order of a delivery comes from the module graph alone, and nothing asks for the line (novox/hq ADR 0419).")
}

// askFresh is the ask of a head's check against its trunk's head, when its passing verdict is out of date and either
// arrived so, or has an Approve asked or given for it (novox/hq ADR 0421 §3, issue 1022; issue 1047 for a group's
// member): the Approve stays through a trunk move, and the merge is asked again on the verdict this brings, with no
// second Approve. Nil when nothing needs asking. Under the lock; the ask runs outside it.
func (h *Holder) askFresh(d *Delivery, now time.Time) func() {
	if d.State != Proposed || d.outOfDate() == "" || d.AskedAgain != nil || !d.Check.Passes() || d.Base != MergeBranch ||
		d.Number == 0 || d.TrunkNow == "" {
		return nil
	}
	if !d.FreshWanted && h.covering(d) == nil {
		return nil
	}
	if tried, was := h.askTried[d.ID]; was && now.Sub(tried) < time.Minute {
		h.lookAt(tried.Add(time.Minute), "a check against the trunk's head that could not be asked")
		return nil
	}
	h.askTried[d.ID] = now
	h.lookAt(now.Add(time.Minute), "a check against the trunk's head that could not be asked")
	id, m := d.ID, MemberOf(d)
	m.BaseSHA = d.TrunkNow
	return func() {
		asked, err := h.Controller.Check("", []Member{m})
		h.mu.Lock()
		defer h.mu.Unlock()
		if err != nil {
			h.Logf("[mesh-delivery] %s's check against the trunk's head cannot be asked yet: %v", id, err)
			return
		}
		delete(h.askTried, id)
		d := h.deliveries[id]
		if d == nil || d.State != Proposed || d.outOfDate() == "" {
			return
		}
		t := h.Now()
		d.AskedAgain, d.FreshWanted = &t, false
		h.Logf("[mesh-delivery] %s: its verdict is out of date; its check against the trunk's head %s asked as %s", id,
			shortOf(d.TrunkNow, 12), asked)
		h.keep(d)
	}
}

// mergeAgainNow lets an approved delivery's merge be asked at once: something that decides whether it may merge happened (a
// verdict, an Approve, a group's composed check), so the wait after a refusal does not hold it. Under the lock.
func (h *Holder) mergeAgainNow(d *Delivery) {
	if d != nil && d.Ready != nil && d.Ready.Merged == nil {
		d.Ready.LastTry = time.Time{}
	}
}

// readyWaits is what a delivery's merge waits on; empty when the Approve may be asked and the merge asked. A ready
// delivery is asked. So is a proposed one whose verdict went out of date, when an Approve was asked for it already (it
// stays through a trunk move, ADR 0421 §3) and its check against the trunk's head is not on its way; one without an
// Approve waits for that check first.
func (h *Holder) readyWaits(d *Delivery) string {
	switch {
	case d.State == Ready:
	case d.State == Proposed && d.outOfDate() != "" && d.AskedAgain != nil:
		return "its check against the trunk's head is asked and has not answered"
	case d.State == Proposed && d.outOfDate() != "" && d.Check.Passes() && h.covering(d) != nil:
	case d.State == Proposed && d.outOfDate() != "":
		return "its verdict is out of date: its check against the trunk's head is asked before the operator is"
	case d.State == Proposed && d.AskedAgain != nil:
		return "its check against the trunk's head is asked and has not answered"
	default:
		return "its check has not passed yet (" + string(d.State) + ")"
	}
	if d.Group == "" {
		return ""
	}
	return h.groupTurnToMerge(d)
}

// groupTurnToMerge is why a group member may not merge yet; empty when it may: the group's composed check passed for
// its members' heads as they are and is not out of date. No member waits for another's merge or delivery (novox/hq
// ADR 0418 §4): once the composed check passed and the Approve covers the heads, every member is merged in one pass.
func (h *Holder) groupTurnToMerge(d *Delivery) string {
	g := h.groups[d.Group]
	if g == nil {
		return ""
	}
	members := h.membersOf(g)
	if state, why := GroupState(g, members); state != "ready" && state != "delivering" {
		return fmt.Sprintf("its group %s is %s: %s", g.ID, state, why)
	}
	if c := g.Check; c == nil || (c.Verdict != "pass" && c.Verdict != "warning") || c.OutOfDate != "" ||
		!sameMembers(c.Commits, commitsOf(members)) {
		return "its group " + g.ID + "'s composed check has not passed for its members' heads"
	}
	return ""
}

// askMerge asks the forge's merge tool to merge a head the operator approved, and keeps its answer. The tool is handed
// the heads the Approve covers, each `owner/repo@<full sha>` (novox/hq ADR 0421 §9), and refuses a pull request whose
// head differs from the one approved.
func (h *Holder) askMerge(id string, heads map[string]string) func() {
	return func() {
		h.mu.Lock()
		d := h.deliveries[id]
		if d == nil || d.Ready == nil {
			h.mu.Unlock()
			return
		}
		owner, repo, number, head := d.Owner(), d.Repo(), d.Number, d.Commit
		if sha := heads[id]; sha != "" {
			head = sha
		}
		h.mu.Unlock()
		err := h.Forge.Merge(owner, repo, number, head, approvedHeads(heads))
		h.mu.Lock()
		defer h.mu.Unlock()
		d = h.deliveries[id]
		if d == nil || d.Ready == nil {
			return
		}
		r, now := d.Ready, h.Now()
		r.merging = false
		if err == nil {
			// Issue 1025: the announcement is owed, kept with the delivery, so a death right after the merge
			// still says it once.
			d.Owed = append(d.Owed, Effect{Kind: EffectEmit, Event: "merged", Line: string(mustJSON(mergedEvent(d, head))), Since: now})
			r.Merged, r.Answer, r.RefusedSince = &now, "merged", nil
			h.Logf("[mesh-delivery] %s merged on the operator's Approve at %s, its check passed against its trunk's head", id,
				shortOf(head, 12))
			h.keep(d)
			return
		}
		answer := oneLine(err.Error())
		if r.RefusedSince == nil {
			r.RefusedSince = &now
		}
		if answer != r.Answer {
			h.Logf("[mesh-delivery] %s is approved, and the merge tool did not merge it (asked again in %s): %s", id,
				mergeRetryAfter, answer)
		}
		h.lookAt(now.Add(mergeRetryAfter), "a merge the forge refused")
		r.Answer = answer
		h.keep(d)
	}
}

// approvedHeads are the heads an Approve covers as the merge tool is handed them: `owner/repo@<full sha>`, sorted.
func approvedHeads(heads map[string]string) []string {
	var out []string
	for id, sha := range heads {
		repository, _, _ := strings.Cut(id, "@")
		out = append(out, repository+"@"+sha)
	}
	sort.Strings(out)
	return out
}

// sayOn comments on a delivery's pull request.
func (h *Holder) sayOn(d *Delivery, body string) func() {
	owner, repo, number, id := d.Owner(), d.Repo(), d.Number, d.ID
	return func() {
		if err := h.Forge.Comment(owner, repo, number, body); err != nil {
			h.Logf("[mesh-delivery] %s: could not say on its pull request: %v", id, err)
		}
	}
}

// readyLines are the approved heads that will not merge: a check that failed at once, a merge refused past its bound;
// as stalled lines, which the controller raises as conditions. Under the lock.
func (h *Holder) readyLines(now time.Time) []StalledLine {
	var out []StalledLine
	for _, d := range h.deliveries {
		r := d.Ready
		if r == nil || r.Merged != nil || d.MergedAs != "" || d.State.Final() {
			continue
		}
		switch {
		case r.Failed != "" && d.State == Rejected:
			out = append(out, StalledLine{ID: d.ID, Number: d.Number, State: d.State, For: now.Sub(r.Seen).Round(time.Second).String(),
				Bound: "0s", H2: "none: a new head, or a recheck that passes — the operator's",
				Says: fmt.Sprintf("%s#%d passed its checks at %s and is not merged: %s", d.Repository, d.Number,
					shortOf(r.Head, 12), r.Failed)})
		case r.Refused != "":
			out = append(out, StalledLine{ID: d.ID, Number: d.Number, State: d.State, For: now.Sub(r.Seen).Round(time.Second).String(),
				Bound: "0s", H2: "none: a new head, which asks the operator again — the operator's",
				Says: fmt.Sprintf("%s#%d at %s is not merged: the operator's answer %s", d.Repository,
					d.Number, shortOf(r.Head, 12), r.Refused)})
		case r.RefusedSince != nil && now.Sub(*r.RefusedSince) >= readyRefusedFor:
			out = append(out, StalledLine{ID: d.ID, Number: d.Number, State: d.State,
				For: now.Sub(*r.RefusedSince).Round(time.Second).String(), Bound: readyRefusedFor.String(),
				H2: "none: the merge is asked again after each verdict, Approve or composed verdict, and each minute — the operator's",
				Says: fmt.Sprintf("%s#%d is approved at %s, and the merge tool has refused it since %s: %s", d.Repository,
					d.Number, shortOf(r.Head, 12), r.RefusedSince.UTC().Format(time.RFC3339), r.Answer)})
		}
	}
	return out
}

// mergedEvent is the module event `merged` (issue 1025): the pull request merged, its head and group, and the ask,
// option and heads the operator's Approve was bound to (the warrant the controller reads back from the router). A merge
// without an Approve names no ask and no option: no warrant is invented. Under the lock, before anything clears them.
func mergedEvent(d *Delivery, head string) map[string]any {
	r := d.Ready
	heads := map[string]string{}
	for k, v := range r.Heads {
		heads[k] = v
	}
	ev := map[string]any{"owner": d.Owner(), "repo": d.Repo(), "number": d.Number, "head_sha": head, "group": d.Group,
		"ask_id": "", "option": "", "heads": heads}
	if r.Approved != nil {
		ev["ask_id"], ev["option"] = r.AskID, OptionApprove
	}
	return ev
}