package main // A fix delivered and an issue left in fixing, and an issue its worker abandoned (novox/hq issue 483). The operator found // 13 of 23 issues in fixing whose fix was merged and delivered hours before, and two claimed by builders that had // stopped: the tracker measured the delivery and said "ready to verify", but nothing said an issue left there, and // nothing told a worked issue from a forgotten one. Here: // // - an issue in fixing whose every fix (fixed-by and the pull requests linked) is delivered is // delivered-not-resolved; past `delivered-not-resolved-after` (1h) it is said to the operator's channel as a // warning, key `issue-tracker.delivered-not-resolved.`, naming the issue and its worker, cleared there // when it leaves fixing or names a fix not yet delivered; // - an issue in diagnosing or fixing whose worker did nothing on it for `worker-stale-after` (24h) and that names // no pull request still open is abandoned (the glossary keeps "claim" for a module's on a seat): get and list // say so, and a move by someone else takes it over without a reason, saying it was abandoned. // // A module that runs only its bundle cannot raise a controller condition (hq ADR 0240 rule 8, ADR 0256), so the // warning is the operator's channel's, as the stuck fix's is (ADR 0281 ยง4). import ( "fmt" "strings" "time" ) // DefaultDeliveredNotResolvedAfter is how long an issue may stay in fixing once its fix is delivered before the // operator hears of it: proving an acceptance live takes minutes, and a worker that went away would otherwise // leave the issue there for good. const DefaultDeliveredNotResolvedAfter = time.Hour // MostSweepAsks is the most deliveries one sweep asks of mesh-delivery, each up to the bus's 30 seconds: a fix // linked after its delivery moved is measured within a few sweeps, and a sweep stays short. const MostSweepAsks = 5 // The flags list filters on (novox/hq issue 483). const ( FlagDeliveredNotResolved = "delivered-not-resolved" FlagAbandoned = "abandoned" ) // Flags is every flag list takes. var Flags = []string{FlagDeliveredNotResolved, FlagAbandoned} // deliveredNotResolvedAfter is the setting's bound, or the default. func (t *Tracker) deliveredNotResolvedAfter() time.Duration { if s, err := t.Settings(); err == nil && strings.TrimSpace(s.DeliveredNotResolvedAfter) != "" { if d, err := time.ParseDuration(strings.TrimSpace(s.DeliveredNotResolvedAfter)); err == nil && d > 0 { return d } } return DefaultDeliveredNotResolvedAfter } // deliveredNotResolved says whether an issue is in fixing and every fix it names is delivered. func deliveredNotResolved(is *Issue) bool { return is.Status == StatusFixing && allDelivered(is) } // deliveredAt is when the last of an issue's delivered fixes was delivered, as the delivery said it. func deliveredAt(is *Issue) time.Time { var last time.Time for _, pr := range liveFixes(is) { if f := fixOf(is, pr); f != nil && contains(is.Delivered, pr) && f.Since.After(last) { last = f.Since } } return last } func notResolvedKey(n uint64) string { return "issue-tracker.delivered-not-resolved." + keyOf(n) } // sayNotResolved warns the operator's channel, once per spell, of an issue delivered and not resolved past the // bound, and answers whether it did; an issue no longer so has its warning cleared. func (t *Tracker) sayNotResolved(is *Issue, now time.Time) (bool, error) { if !deliveredNotResolved(is) { return false, t.clearNotResolved(is) } at := deliveredAt(is) if is.NotResolvedOpen || t.Notify == nil || now.Sub(at) < t.deliveredNotResolvedAfter() { return false, nil } worker := "nobody" if is.Worker != nil { worker = is.Worker.By } ago := now.Sub(at).Round(time.Minute) prs := strings.Join(liveFixes(is), ", ") if err := t.Notify(notResolvedKey(is.Number), "warning", fmt.Sprintf("issue %d: fixed and delivered, not resolved; %s works on it", is.Number, worker), fmt.Sprintf("Fixed and delivered, not resolved (issue %d)", is.Number), fmt.Sprintf("Every fix of issue %d (%s) was delivered %s ago, at %s, and the issue is still in fixing; %s works "+ "on it. Its worker proves the acceptance live and resolves it. When the worker is gone, another agent "+ "takes it over with a move to fixing and resolves it; nothing is needed from you otherwise.", is.Number, prs, ago, at.Format(time.RFC3339), worker), "", false); err != nil { return false, err } _, err := t.change(is.Number, Act{Op: "warned", By: Seat}, func(i *Issue) error { i.NotResolvedOpen = true; return nil }) return err == nil, err } // clearNotResolved clears the warning once the issue is no longer delivered and in fixing: resolved, moved on, or // naming a fix not delivered yet. func (t *Tracker) clearNotResolved(is *Issue) error { if !is.NotResolvedOpen || deliveredNotResolved(is) || t.Notify == nil { return nil } if err := t.Notify(notResolvedKey(is.Number), "", "", "", "", fmt.Sprintf("issue %d left fixing (%s), or names a "+ "fix not delivered yet", is.Number, is.Status), true); err != nil { return err } _, err := t.change(is.Number, Act{Op: "warned", By: Seat}, func(i *Issue) error { i.NotResolvedOpen = false; return nil }) return err } // askUnheard asks mesh-delivery for a few fixes of issues not yet resolved that the tracker never heard of: a pull // request linked after its delivery moved hears no transition again, and OnMoved asks only at a move. func (t *Tracker) askUnheard() error { if t.Deliveries == nil { return nil } all, err := t.allIssues() if err != nil { return err } asked := 0 for n, is := range all { if !unresolved(is) { continue } heard := false for _, pr := range fixPullRequests(is) { if fixOf(is, pr) != nil { continue } if asked == MostSweepAsks { return nil } asked++ repo, number, _ := strings.Cut(pr, "#") d, err := t.Deliveries(repo, number) if err != nil { return fmt.Errorf("the delivery of %s (issue %d): %v", pr, n, err) } if d.State == "" { continue } since := d.Since if since.IsZero() { since = t.now() } if _, err := t.recordFix(pr, d.State, d.Why, d.Merged, since, n); err != nil { return err } heard = true } if heard { if err := t.afterFix(n); err != nil { return err } } } return nil } // openPullRequest says whether an issue names a pull request not merged yet, or one whose delivery the tracker // never heard of: work in review is work, and a fix unheard of is not taken for none. One closed without merging // is not open. func openPullRequest(is *Issue) bool { for _, pr := range liveFixes(is) { f := fixOf(is, pr) if f == nil || (!f.Merged && !contains(is.Delivered, pr)) { return true } } return false } // abandoned says whether an issue's worker went quiet with no pull request open: nothing done on it for the bound, // and nothing in review. func abandoned(is *Issue, now time.Time, after time.Duration) bool { return worked(is.Status) && is.Worker != nil && is.Worker.stale(now, after) && !openPullRequest(is) } // hasFlag says whether an issue carries a list flag. func hasFlag(is *Issue, flag string, now time.Time, staleAfter time.Duration) bool { switch flag { case FlagDeliveredNotResolved: return deliveredNotResolved(is) case FlagAbandoned: return abandoned(is, now, staleAfter) } return false }