package main // Whether an issue's fix is delivered (novox/hq ADR 0281). The tracker hears mesh-delivery's `transition` event // for every pull request an issue's fixed-by names and keeps, per pull request, the state the delivery last said // — merged or not, held and why, covered by a later merge, delivered. From that it derives one line for `get` and // `list`: a measured fact, never set by a verb. Delivered is not resolved: when every fix of an issue not yet // resolved is delivered, the issue gains a note and the event `ready-to-verify`, and an agent proves its // acceptance live before resolving it. A fix merged and not delivered past a bound is said on the issue and to // the operator's channel. import ( "encoding/json" "errors" "fmt" "sort" "strings" "time" stdio "git.novox.be/novox/mesh-sdk/go" ) // EventReadyToVerify is said once when every fix of an issue not yet resolved is delivered. const EventReadyToVerify = "ready-to-verify" // DefaultStuckAfter is how long a fix may be merged and not delivered before it is said stuck: a walk takes // minutes, a gate ten at most, and a hold waits for a person; two hours is past every ordinary delivery and // still the same working day. const DefaultStuckAfter = 2 * time.Hour // Fix is one pull request of an issue's fixed-by, as the delivery last said it. type Fix struct { PullRequest string `json:"pull-request"` State string `json:"state"` Why string `json:"why,omitempty"` Merged bool `json:"merged,omitempty"` MergedAt *time.Time `json:"merged-at,omitempty"` Since time.Time `json:"since"` // StuckSaid says it was said stuck, on the issue and to the operator's channel. StuckSaid bool `json:"stuck-said,omitempty"` // Class is the class of what its delivery moves, as mesh-delivery says it: core, leaf, or none when it moves // no module; Opened when its pull request was first heard (novox/hq ADR 0313 §2, §4). Class string `json:"class,omitempty"` Opened *time.Time `json:"opened,omitempty"` } // heardFix is what the delivery said of a pull request at one moment: its state, why, whether it is merged, and // what a fix's time is read by — the class of what it moves, when its pull request opened and when it merged, // each when said. type heardFix struct { State string Why string Merged bool At time.Time Class string Opened *time.Time MergedAt *time.Time } // news says whether what was heard adds anything to what a fix keeps. func (h heardFix) news(f *Fix, kept string) bool { if f.State != h.State || f.Why != kept || (h.Merged && !f.Merged) || (h.Class != "" && h.Class != f.Class) { return true } if h.Opened != nil && (f.Opened == nil || !f.Opened.Equal(*h.Opened)) { return true } return h.MergedAt != nil && (f.MergedAt == nil || !f.MergedAt.Equal(*h.MergedAt)) } // reached says whether a delivery's state says the change reached the machines (mesh-delivery's Reached). func reached(state string) bool { return state == "delivered" || state == "covered" || state == "retired" } // Whether a pull request is merged is what the delivery says of the pull request itself — `merged_as` on a // transition, the pull request's `merged` in checks — never read from the state: `stopped` is also what a pull // request closed without merging becomes (mesh-delivery's `closed` event), and read as merged it was said stuck to // the operator for good (novox/hq issue 483). // withdrawn says whether a fix is a pull request closed without merging: no fix at all, neither awaiting delivery, // delivered, nor open. func withdrawn(f *Fix) bool { return f != nil && f.State == "stopped" && !f.Merged } // liveFixes is an issue's fixes less those withdrawn: what its delivery, readiness to verify, the stuck sweep and a // abandoned issue are measured on. fixPullRequests still names the withdrawn ones, so a pull request reopened is heard. func liveFixes(is *Issue) []string { var out []string for _, pr := range fixPullRequests(is) { if !withdrawn(fixOf(is, pr)) { out = append(out, pr) } } return out } // withdrawnFixes is the pull requests of an issue closed without merging. func withdrawnFixes(is *Issue) []string { var out []string for _, pr := range fixPullRequests(is) { if withdrawn(fixOf(is, pr)) { out = append(out, pr) } } return out } func fixOf(is *Issue, pr string) *Fix { for i := range is.Fixes { if is.Fixes[i].PullRequest == pr { return &is.Fixes[i] } } return nil } // allDelivered says whether an issue names a pull request and every one is delivered. func allDelivered(is *Issue) bool { prs := liveFixes(is) if len(prs) == 0 { return false } for _, pr := range prs { if !contains(is.Delivered, pr) { return false } } return true } // DeliveryLine is the derived delivery state of an issue's fix, in words; empty when fixed-by names no pull // request. func DeliveryLine(is *Issue) string { prs := liveFixes(is) if len(prs) == 0 { if w := withdrawnFixes(is); len(w) > 0 { return "fix withdrawn: " + strings.Join(w, ", ") + " closed without merging" } return "" } var held, failed, merged, open, unheard []string covered := false for _, pr := range prs { f := fixOf(is, pr) switch { case contains(is.Delivered, pr): covered = covered || (f != nil && f.State == "covered") case f == nil: unheard = append(unheard, pr) case f.State == "held": held = append(held, strings.TrimSpace(pr+": "+f.Why)) case f.State == "failed" || f.State == "stopped": failed = append(failed, strings.TrimSpace(pr+" "+f.State+": "+f.Why)) case f.Merged: merged = append(merged, pr) default: open = append(open, pr) } } switch { case len(failed) > 0: return "fix not delivered: " + strings.Join(failed, "; ") case len(held) > 0: return "held: " + strings.Join(held, "; ") case len(merged) > 0: return "fix merged, not delivered: " + strings.Join(merged, ", ") case len(open) > 0: return "fix not merged: " + strings.Join(open, ", ") case len(unheard) > 0: return "nothing heard from the delivery of " + strings.Join(unheard, ", ") case covered: return "delivered to every machine it should reach (by a later merge that contains it)" } return "delivered to every machine it should reach" } // recordFix keeps what the delivery said of a pull request on every issue whose fixed-by names it (only, when // given, on that one issue), and answers the issues it changed. func (t *Tracker) recordFix(pr string, h heardFix, only uint64) ([]uint64, error) { state, why, merged, at := h.State, h.Why, h.Merged, h.At all, err := t.allIssues() if err != nil { return nil, err } var changed []uint64 for n, is := range all { if (only != 0 && n != only) || !contains(fixPullRequests(is), pr) { continue } kept := cut(why, MostReasonKept) if f := fixOf(is, pr); f != nil && (at.Before(f.Since) || !h.news(f, kept)) { // Older than what is kept, or nothing new: transitions can arrive out of order. continue } if contains(is.Delivered, pr) && !reached(state) { // Delivered stays delivered: a later state that has not reached the machines is a newer delivery of // the same pull request at most, and an older one is history. continue } if _, err := t.change(n, Act{Op: "delivery", By: Seat}, func(i *Issue) error { f := fixOf(i, pr) if f == nil { i.Fixes = append(i.Fixes, Fix{PullRequest: pr}) f = &i.Fixes[len(i.Fixes)-1] } f.State, f.Why, f.Since = state, kept, at if merged && !f.Merged { f.Merged = true m := at f.MergedAt = &m } if h.MergedAt != nil && (merged || f.Merged) { m := h.MergedAt.UTC() f.MergedAt = &m // the delivery's own moment of the merge, not when it was heard } if h.Class != "" { f.Class = h.Class } if h.Opened != nil { o := h.Opened.UTC() f.Opened = &o } if reached(state) && !contains(i.Delivered, pr) { i.Delivered = append(i.Delivered, pr) } return nil }); err != nil { return changed, err } changed = append(changed, n) } sort.Slice(changed, func(i, j int) bool { return changed[i] < changed[j] }) return changed, nil } // afterFix is what follows a change of an issue's fixes: ready to verify, once, when every fix is delivered and // the issue is not yet resolved; a stuck fix delivered at last cleared on the operator's channel; and what // waited on it let go. func (t *Tracker) afterFix(n uint64) error { is, _, err := read(t.Store, n) if err != nil { return err } if err := t.clearStuck(is); err != nil { t.Logf("[mesh-issues] the stuck fix of %d could not be cleared on the operator's channel (tried again at the next sweep): %v", n, err) } if err := t.clearNotResolved(is); err != nil { t.Logf("[mesh-issues] the warning that %d is delivered and not resolved could not be cleared on the operator's channel (tried again at the next sweep): %v", n, err) } if allDelivered(is) { // Its fix time, final once its last fix is delivered (novox/hq ADR 0313). if err := t.timeFix(n); err != nil { return err } if unresolved(is) { prs := liveFixes(is) sort.Strings(prs) if strings.Join(is.VerifyFor, ",") != strings.Join(prs, ",") { // The note first, kept once by its words; then the claim; then the event. A failure before the claim // is tried again at the next offer, and the note is not written twice. It names when each fix was // delivered and who works on the issue: no channel of the tracker reaches a worker, so the note // and the event are how the worker hears it (novox/hq issue 483). var when []string for _, pr := range prs { if f := fixOf(is, pr); f != nil { when = append(when, pr+" at "+f.Since.Format(time.RFC3339)) } else { when = append(when, pr) } } prefix := fmt.Sprintf("Ready to verify: every fix of this issue is delivered (%s).", strings.Join(when, ", ")) worker := "" if is.Worker != nil { worker = is.Worker.By } if !noted(is, prefix) { text := prefix if worker != "" { text += fmt.Sprintf(" %s works on it: prove the acceptance live and resolve it, or say what is left. "+ "In fixing and not resolved %s after its delivery, it is said to the operator.", worker, t.deliveredNotResolvedAfter()) } text += " Delivered is not resolved: prove the acceptance live, then resolve it, from fixing: move it " + "to fixing with by first if it is not there. (novox/hq ADR 0281, ADR 0284, issue 483)" if _, err := t.note(GroomBy, n, text); err != nil { return err } } claimed := false if _, err := t.change(n, Act{Op: EventReadyToVerify, By: Seat}, func(i *Issue) error { claimed = false if strings.Join(i.VerifyFor, ",") == strings.Join(prs, ",") { return nil } i.VerifyFor, claimed = prs, true return nil }); err != nil { return err } if claimed { body := map[string]any{"number": n, "fixed-by": prs, "at": t.now(), "delivered-at": deliveredAt(is)} if worker != "" { body["worker"] = worker } t.emit(EventReadyToVerify, body) } } } } return t.letGo(n) } // stillStuck says whether an issue has a fix said stuck that is still named, not delivered, and the issue not // closed. A closed issue is the end of its warning: resolved means its worker proved the acceptance live, so a fix // the tracker never heard delivered is not waited for any more (a failed walk that a later merge carried stays // failed, novox/hq issue 547), and wontfix waits for nothing. func stillStuck(is *Issue) bool { if closedStatus(is.Status) { return false } named := liveFixes(is) for _, f := range is.Fixes { if f.StuckSaid && contains(named, f.PullRequest) && !contains(is.Delivered, f.PullRequest) { return true } } return false } // clearStuck clears the warning said for an issue on the operator's channel once nothing it was said for is // stuck — delivered, no longer named, or the issue closed (resolved or wontfix); one that fails is tried again at // the next sweep. func (t *Tracker) clearStuck(is *Issue) error { if !is.StuckOpen || stillStuck(is) || t.Notify == nil { return nil } if err := t.Notify(stuckKey(is.Number), "", "", "", "", stuckEnded(is), true); err != nil { return err } _, err := t.change(is.Number, Act{Op: "warned", By: Seat}, func(i *Issue) error { i.StuckOpen = false; return nil }) return err } // stuckEnded is what the operator's channel says when a stuck warning ends. func stuckEnded(is *Issue) string { switch is.Status { case StatusResolved: return "resolved: issue " + keyOf(is.Number) + " was proven live by its worker" case StatusWontfix: return "issue " + keyOf(is.Number) + " was closed as wontfix: its fix is no longer waited for" } return "the fix of issue " + keyOf(is.Number) + " is delivered or no longer waited for" } func stuckKey(n uint64) string { return "issue-tracker.fix-stuck." + keyOf(n) } func prKey(repository string, number int) string { _, repo, ok := strings.Cut(repository, "/") if !ok { repo = repository } return fmt.Sprintf("%s#%d", strings.ToLower(repo), number) } // OnTransition is the handler of mesh-delivery's `transition` event: every state a pull request's delivery moves // to is kept on each issue whose fixed-by names it. func (t *Tracker) OnTransition(e stdio.Envelope) error { var body struct { Repository string `json:"repository"` Number int `json:"number"` To string `json:"to"` Why string `json:"why"` MergedAs string `json:"merged_as"` At time.Time `json:"at"` Class string `json:"class"` Opened *time.Time `json:"opened"` MergedAt *time.Time `json:"merged_at"` } if err := json.Unmarshal(e.Body, &body); err != nil || body.To == "" || body.Number <= 0 { return nil } at := body.At.UTC() if at.IsZero() { at = t.now() } merged := body.MergedAs != "" changed, err := t.recordFix(prKey(body.Repository, body.Number), heardFix{State: body.To, Why: body.Why, Merged: merged, At: at, Class: body.Class, Opened: nonZero(body.Opened), MergedAt: nonZero(body.MergedAt)}, 0) if err != nil { return err } for _, n := range changed { if err := t.afterFix(n); err != nil { return err } } return nil } // Delivery is a pull request's delivery as mesh-delivery's `checks` answers it; a test stubs the asking. type Delivery struct { State string Why string // Since is when the delivery entered its state, as it says; zero when it does not. Since time.Time // Merged is the pull request's own merged flag, as the forge says it. Merged bool // Class, Opened and MergedAt are what a fix's time is read by (novox/hq ADR 0313), when the delivery says them. Class string Opened *time.Time MergedAt *time.Time } // heard is what a delivery answered, as a fix keeps it. func (d Delivery) heard(since time.Time) heardFix { return heardFix{State: d.State, Why: d.Why, Merged: d.Merged, At: since, Class: d.Class, Opened: d.Opened, MergedAt: d.MergedAt} } // nonZero is a moment said, or nil for one not said. func nonZero(t *time.Time) *time.Time { if t == nil || t.IsZero() { return nil } u := t.UTC() return &u } // DeliveryState asks mesh-delivery for one pull request's delivery. type DeliveryState func(repository string, number string) (Delivery, error) // busDeliveryState asks mesh-delivery's checks over the bus. func busDeliveryState(owner string) DeliveryState { return func(repository, number string) (Delivery, error) { raw, err := stdio.Ask("seat:mesh-delivery.checks", map[string]any{"repository": owner + "/" + repository, "number": number}) if err != nil { return Delivery{}, err } var out struct { Delivery struct { State string `json:"state"` Waits string `json:"waits"` Since time.Time `json:"since"` Class string `json:"class"` Opened *time.Time `json:"opened"` MergedAt *time.Time `json:"merged_at"` } `json:"delivery"` Pull struct { Merged bool `json:"merged"` } `json:"pull"` } if err := json.Unmarshal(raw, &out); err != nil { return Delivery{}, fmt.Errorf("mesh-delivery's checks answered no delivery: %.200s", raw) } return Delivery{State: out.Delivery.State, Why: out.Delivery.Waits, Since: out.Delivery.Since.UTC(), Merged: out.Pull.Merged, Class: out.Delivery.Class, Opened: nonZero(out.Delivery.Opened), MergedAt: nonZero(out.Delivery.MergedAt)}, nil } } // OnMoved is the handler of the tracker's own `moved` event: each pull request of the issue's fix not heard of // yet is asked of mesh-delivery, at most a few per handling, for a delivery that moved before the issue named // it; then what follows its fixes, and an issue closed lets go what waited on it. func (t *Tracker) OnMoved(e stdio.Envelope) error { var body struct { Number uint64 `json:"number"` } if err := json.Unmarshal(e.Body, &body); err != nil || body.Number == 0 { return nil } is, _, err := read(t.Store, body.Number) if errors.Is(err, ErrNoIssue) { return nil } if err != nil { return err } if t.Deliveries != nil { // Each ask up to the bus's 30 seconds, under the runtime's two minutes; one that fails, or more left to // ask, has the event offered again, so a delivery never asked ends in max-deliveries, not in silence. asked := 0 for _, pr := range fixPullRequests(is) { if fixOf(is, pr) != nil { continue } if asked == MostDeliveryAsks { return fmt.Errorf("issue %d names more pull requests than are asked at once; the rest at the next offer", body.Number) } asked++ repo, number, _ := strings.Cut(pr, "#") d, err := t.Deliveries(repo, number) if err != nil { return fmt.Errorf("the delivery of %s (issue %d) could not be asked: %v", pr, body.Number, err) } state := d.State if state == "" { // The delivery knows nothing of it yet: not kept, so it is asked again at the next move, and its own // events say it when they come. continue } // The delivery's own time, so a transition stamped the same moment is not taken as older. since := d.Since if since.IsZero() { since = t.now() } if _, err := t.recordFix(pr, d.heard(since), body.Number); err != nil { return err } } } return t.afterFix(body.Number) } // Notifier tells the operator's channel something (operator-channel.notify), or clears it. type Notifier func(key, severity, summary, headline, explanation, resolved string, clear bool) error // busNotifier asks the operator's channel over the bus. func busNotifier() Notifier { return func(key, severity, summary, headline, explanation, resolved string, clear bool) error { body := map[string]any{"key": key, "subject": "the issue tracker"} if clear { body["clear"] = true body["resolved"] = resolved } else { body["severity"], body["summary"], body["headline"], body["explanation"] = severity, summary, headline, explanation } _, err := stdio.Ask("seat:operator-channel.notify", body) return err } } // stuckAfter is the setting's bound, or the default. func (t *Tracker) stuckAfter() time.Duration { if s, err := t.Settings(); err == nil && strings.TrimSpace(s.StuckAfter) != "" { if d, err := time.ParseDuration(strings.TrimSpace(s.StuckAfter)); err == nil && d > 0 { return d } } return DefaultStuckAfter } // Sweep says every fix of an issue not closed merged and not delivered past the bound: a note on its issue and a warning on the // operator's channel, once; and every issue in fixing whose fixes were all delivered past its own bound and is not // resolved (novox/hq issue 483). First it asks the delivery of a few fixes never heard of. It answers how many // warnings it said. Run every few minutes. func (t *Tracker) Sweep() (int, error) { if err := t.askUnheard(); err != nil { t.Logf("[mesh-issues] the sweep could not ask a fix's delivery (asked again at the next sweep): %v", err) } all, err := t.allIssues() if err != nil { return 0, err } bound, now := t.stuckAfter(), t.now() said := 0 for n, is := range all { // A fix time missed at its delivery (a failure, or a restart between) is kept now; one of an issue moved to // fixing before the tracker kept the moment is kept only when its fix is heard of again. if is.FixingFrom != nil && is.FixTime == nil && allDelivered(is) { if err := t.timeFix(n); err != nil { t.Logf("[mesh-issues] the fix time of %d could not be kept (tried again at the next sweep): %v", n, err) } } ok, err := t.sayNotResolved(is, now) if err != nil { t.Logf("[mesh-issues] that %d is delivered and not resolved could not be said on the operator's channel (said at the next sweep): %v", n, err) } else if ok { said++ } if err := t.clearStuck(is); err != nil { t.Logf("[mesh-issues] the stuck fix of %d could not be cleared on the operator's channel: %v", n, err) } if closedStatus(is.Status) { // A resolved or wontfix issue waits for no fix: never said stuck (novox/hq issue 547). continue } named := liveFixes(is) for _, f := range is.Fixes { if !contains(named, f.PullRequest) || contains(is.Delivered, f.PullRequest) || !f.Merged || f.MergedAt == nil || reached(f.State) || f.StuckSaid || now.Sub(*f.MergedAt) < bound { continue } state := f.State if f.Why != "" { state += ": " + f.Why } prefix := fmt.Sprintf("Stuck: the fix %s", f.PullRequest) if !noted(is, prefix) { if _, err := t.note(GroomBy, n, fmt.Sprintf("%s was merged %s ago and is not delivered (%s). A merged fix "+ "that does not reach the machines is said on the operator's channel. (novox/hq ADR 0281)", prefix, now.Sub(*f.MergedAt).Round(time.Minute), state)); err != nil { return said, err } } if t.Notify != nil { if err := t.Notify(stuckKey(n), "warning", fmt.Sprintf("issue %d: fix %s merged %s ago, not delivered", n, f.PullRequest, now.Sub(*f.MergedAt).Round(time.Minute)), fmt.Sprintf("A fix is merged and not delivered (issue %d)", n), fmt.Sprintf("The fix of issue %d was merged %s ago and has not reached the machines: %s. Look at its "+ "delivery; nothing else is needed from you unless it waits for a person.", n, now.Sub(*f.MergedAt).Round(time.Minute), state), "", false); err != nil { // Tried again at the next sweep: the note is not written twice. t.Logf("[mesh-issues] the stuck fix of %d could not be said on the operator's channel: %v", n, err) continue } } pr := f.PullRequest if _, err := t.change(n, Act{Op: "warned", By: Seat}, func(i *Issue) error { if g := fixOf(i, pr); g != nil { g.StuckSaid = true } i.StuckOpen = t.Notify != nil return nil }); err != nil { return said, err } said++ } } if err := t.judgeFixBudgets(); err != nil { t.Logf("[mesh-issues] a fix over its budget could not be said or cleared on the operator's channel (tried again at the next sweep): %v", err) } return said, nil }