package main // A fix delivered and an issue left in fixing, and an issue its worker abandoned (novox/hq issue 483). The operator found // 13 of 23 issues in fixing whose fix was merged and delivered hours before, and two claimed by builders that had // stopped: the tracker measured the delivery and said "ready to verify", but nothing said an issue left there, and // nothing told a worked issue from a forgotten one. Here: // // - an issue in fixing whose every fix (fixed-by and the pull requests linked) is delivered is // delivered-not-resolved; past `delivered-not-resolved-after` (1h) it is said to the operator's channel as a // warning, key `issue-tracker.delivered-not-resolved.`, naming the issue and its worker, cleared there // when it leaves fixing or names a fix not yet delivered; // - an issue in diagnosing or fixing whose worker did nothing on it for `worker-stale-after` (24h) and that names // no pull request still open is abandoned (the glossary keeps "claim" for a module's on a seat): get and list // say so, and a move by someone else takes it over without a reason, saying it was abandoned. // // A module that runs only its bundle cannot raise a controller condition (hq ADR 0240 rule 8, ADR 0256), so the // warning is the operator's channel's, as the stuck fix's is (ADR 0281 ยง4). import ( "fmt" "sort" "strings" "time" ) // DefaultDeliveredNotResolvedAfter is how long an issue may stay in fixing once its fix is delivered before the // operator hears of it: proving an acceptance live takes minutes, and a worker that went away would otherwise // leave the issue there for good. const DefaultDeliveredNotResolvedAfter = time.Hour // MostSweepAsks is the most deliveries one sweep asks of mesh-delivery, each up to the bus's 30 seconds: a fix // linked after its delivery moved is measured within a few sweeps, and a sweep stays short. const MostSweepAsks = 5 // The flags list filters on (novox/hq issue 483). const ( FlagDeliveredNotResolved = "delivered-not-resolved" FlagAbandoned = "abandoned" ) // Flags is every flag list takes. var Flags = []string{FlagDeliveredNotResolved, FlagAbandoned} // deliveredNotResolvedAfter is the setting's bound, or the default. func (t *Tracker) deliveredNotResolvedAfter() time.Duration { if s, err := t.Settings(); err == nil && strings.TrimSpace(s.DeliveredNotResolvedAfter) != "" { if d, err := time.ParseDuration(strings.TrimSpace(s.DeliveredNotResolvedAfter)); err == nil && d > 0 { return d } } return DefaultDeliveredNotResolvedAfter } // deliveredNotResolved says whether an issue is in fixing and every fix it names is delivered. func deliveredNotResolved(is *Issue) bool { return is.Status == StatusFixing && allDelivered(is) } // deliveredAt is when the last of an issue's delivered fixes was delivered, as the delivery said it. func deliveredAt(is *Issue) time.Time { var last time.Time for _, pr := range liveFixes(is) { if f := fixOf(is, pr); f != nil && contains(is.Delivered, pr) && f.Since.After(last) { last = f.Since } } return last } func notResolvedKey(n uint64) string { return "issue-tracker.delivered-not-resolved." + keyOf(n) } // sayNotResolved warns the operator's channel, once per spell, of an issue delivered and not resolved past the // bound, and answers whether it did; an issue no longer so has its warning cleared. func (t *Tracker) sayNotResolved(is *Issue, now time.Time) (bool, error) { if !deliveredNotResolved(is) { return false, t.clearNotResolved(is) } at := deliveredAt(is) if is.NotResolvedOpen || t.Notify == nil || now.Sub(at) < t.deliveredNotResolvedAfter() { return false, nil } worker := "nobody" if is.Worker != nil { worker = is.Worker.By } ago := now.Sub(at).Round(time.Minute) prs := strings.Join(liveFixes(is), ", ") if err := t.Notify(notResolvedKey(is.Number), "warning", fmt.Sprintf("issue %d: fixed and delivered, not resolved; %s works on it", is.Number, worker), fmt.Sprintf("Fixed and delivered, not resolved (issue %d)", is.Number), fmt.Sprintf("Every fix of issue %d (%s) was delivered %s ago, at %s, and the issue is still in fixing; %s works "+ "on it. Its worker proves the acceptance live and resolves it. When the worker is gone, another agent "+ "takes it over with a move to fixing and resolves it; nothing is needed from you otherwise.", is.Number, prs, ago, at.Format(time.RFC3339), worker), "", false); err != nil { return false, err } _, err := t.change(is.Number, Act{Op: "warned", By: Seat}, func(i *Issue) error { i.NotResolvedOpen = true; return nil }) return err == nil, err } // clearNotResolved clears the warning once the issue is no longer delivered and in fixing: resolved, moved on, or // naming a fix not delivered yet. func (t *Tracker) clearNotResolved(is *Issue) error { if !is.NotResolvedOpen || deliveredNotResolved(is) || t.Notify == nil { return nil } if err := t.Notify(notResolvedKey(is.Number), "", "", "", "", fmt.Sprintf("issue %d left fixing (%s), or names a "+ "fix not delivered yet", is.Number, is.Status), true); err != nil { return err } _, err := t.change(is.Number, Act{Op: "warned", By: Seat}, func(i *Issue) error { i.NotResolvedOpen = false; return nil }) return err } // askUnheard asks mesh-delivery for a few fixes of issues not yet resolved that the tracker never heard of: a pull // request linked after its delivery moved hears no transition again, and OnMoved asks only at a move. func (t *Tracker) askUnheard() error { if t.Deliveries == nil && t.Pull == nil { return nil } all, err := t.allIssues() if err != nil { return err } // Issues not resolved first, then the others: a resolved issue's fix merged before the tracker heard deliveries is learned // too, so its delivery line stops saying nothing was heard (issue 1950). The asks are bounded, and the pull request asked // longest ago goes first, so ones that cannot be learned (still open, gone) never starve the rest. type cand struct { n uint64 pr string open bool at time.Time } var cs []cand t.askedMu.Lock() for n, is := range all { for _, pr := range unheardOf(is) { cs = append(cs, cand{n, pr, unresolved(is), t.askedAt[pr]}) } } t.askedMu.Unlock() sort.Slice(cs, func(i, j int) bool { a, b := cs[i], cs[j] if a.open != b.open { return a.open } if !a.at.Equal(b.at) { return a.at.Before(b.at) } if a.n != b.n { return a.n < b.n } return a.pr < b.pr }) for i, c := range cs { if i == MostSweepAsks { break } t.askedMu.Lock() if t.askedAt == nil { t.askedAt = map[string]time.Time{} } t.askedAt[c.pr] = t.now() t.askedMu.Unlock() kept, err := t.learnFix(c.n, c.pr) if err != nil { t.Logf("[mesh-issues] the delivery of %s (issue %d) could not be asked (the next sweep asks the others first): %v", c.pr, c.n, err) continue } if kept { if err := t.afterFix(c.n); err != nil { return err } } } return nil } // unheardOf is the pull requests an issue names that the tracker keeps no record for. func unheardOf(is *Issue) []string { var out []string for _, pr := range fixPullRequests(is) { if fixOf(is, pr) == nil { out = append(out, pr) } } return out } // openPullRequest says whether an issue names a pull request not merged yet, or one whose delivery the tracker // never heard of: work in review is work, and a fix unheard of is not taken for none. One closed without merging // is not open. func openPullRequest(is *Issue) bool { for _, pr := range liveFixes(is) { f := fixOf(is, pr) if f == nil || (!f.Merged && !contains(is.Delivered, pr)) { return true } } return false } // abandoned says whether an issue's worker went quiet with no pull request open: nothing done on it for the bound, // and nothing in review. func abandoned(is *Issue, now time.Time, after time.Duration) bool { return worked(is.Status) && is.Worker != nil && is.Worker.stale(now, after) && !openPullRequest(is) } // hasFlag says whether an issue carries a list flag. func hasFlag(is *Issue, flag string, now time.Time, staleAfter time.Duration) bool { switch flag { case FlagDeliveredNotResolved: return deliveredNotResolved(is) case FlagAbandoned: return abandoned(is, now, staleAfter) } return false }