Measure each fix from the move to fixing to delivered, and say one over its budget (issue 403)
mesh/merge-gate pass: builds mesh-delivery, mesh-issues → novox; no bus step; every machine composes with the change as it did without (4 of 4 compose)
mesh/repo-check pass: its merge-check.sh passed, each part in its toolchain: merge-check.sh (go), merge-check-typescript.sh (typescript)
mesh/delivery-group group fix/403-fix-time ready: every member ready, and composed together they pass
mesh/delivery superseded: a newer head of the same pull request

hq ADR 0313: the operator's five minutes start when an agent moves the issue
to fixing, not at the merge. mesh-delivery says on each transition, and in
checks, when the pull request opened, when it merged and the class of what it
moves. mesh-issues keeps the first move to fixing, the fix time with its four
phases once the last fix is delivered, serves fix-times, and says
issue-tracker.fix.<class>.over-budget on the operator's channel, cleared by the
next fix of that class within budget.
This commit is contained in:
jochen
2026-10-12 15:50:12 +02:00
parent c5e1bc7bb4
commit 43df7ebc5a
15 changed files with 1342 additions and 31 deletions
@@ -65,6 +65,11 @@ type ChecksOf struct {
Since string `json:"since"`
Waits string `json:"waits,omitempty"`
Group string `json:"group,omitempty"`
// Class, Opened and MergedAt are what the issue tracker measures a fix's time by (novox/hq ADR 0313): the
// class of what it moves, when its pull request was first heard, and when it merged.
Class string `json:"class,omitempty"`
Opened string `json:"opened,omitempty"`
MergedAt string `json:"merged_at,omitempty"`
}
// Checks is the `checks` verb: a repository and a pull request's number, or a repository and a commit.
@@ -104,7 +109,13 @@ func (h *Holder) Checks(repository string, number int, commit string) (*ChecksAn
}
if d != nil {
a.Delivery = &ChecksOf{ID: d.ID, State: d.State, Since: d.Since.UTC().Format(time.RFC3339), Waits: d.waits,
Group: d.Group}
Group: d.Group, Class: fixClass(&d.Delivery)}
if !d.opened.IsZero() {
a.Delivery.Opened = d.opened.UTC().Format(time.RFC3339)
}
if !d.MergedAt.IsZero() {
a.Delivery.MergedAt = d.MergedAt.UTC().Format(time.RFC3339)
}
if d.Check != nil && sameCommit(d.Commit, a.Commit) {
a.Verdict = d.Check
}
@@ -139,6 +150,7 @@ type checksDelivery struct {
Delivery
waits string
groupSaid string
opened time.Time
}
// deliveryOf is the delivery of a commit — its head, or the commit it merged as — else, for a pull request,
@@ -167,7 +179,7 @@ func (h *Holder) deliveryOf(repository string, number int, commit string) *check
if found == nil {
return nil
}
c := &checksDelivery{Delivery: *found, waits: h.waitsFor(found)}
c := &checksDelivery{Delivery: *found, waits: h.waitsFor(found), opened: h.openedOf(found)}
if found.Check != nil {
v := *found.Check
c.Check = &v
@@ -14,8 +14,10 @@ import (
// the forge's holder, asked through its tools). Owed, never lost: kept with the delivery before anything is
// tried, and tried again until done.
// transitionEvent is what `mesh-delivery.transition` says: no secret, no address.
func transitionEvent(d *Delivery, t Transition) map[string]any {
// transitionEvent is what `mesh-delivery.transition` says: no secret, no address. opened is when its pull
// request was first heard (zero when not known); with its class and its merge's moment it is what the issue
// tracker measures a fix's time by (novox/hq ADR 0313 §2, §4).
func transitionEvent(d *Delivery, t Transition, opened time.Time) map[string]any {
e := map[string]any{"id": d.ID, "repository": d.Repository, "commit": d.Commit, "from": t.From, "to": t.To,
"event": t.Event, "why": t.Why, "at": t.At}
if t.By != "" {
@@ -36,9 +38,57 @@ func transitionEvent(d *Delivery, t Transition) map[string]any {
if d.Plan != nil {
e["plan"] = d.Plan.Summary
}
if c := fixClass(d); c != "" {
e["class"] = c
}
if !opened.IsZero() {
e["opened"] = opened.UTC()
}
if !d.MergedAt.IsZero() {
e["merged_at"] = d.MergedAt.UTC()
}
return e
}
// fixClass is the class of what a delivery moves (novox/hq ADR 0313 §2): its plan's, as the planner said it;
// none for a plan that moves no module, from a planner that did not say a class; else its walk's class once its
// delivery time is known; empty when nothing says it.
func fixClass(d *Delivery) string {
switch {
case d.Plan != nil && d.Plan.Class != "":
return d.Plan.Class
case d.Plan != nil && len(d.Plan.Moved) == 0 && len(d.Plan.Dependents) == 0 && len(d.Plan.New) == 0:
return classNone
case d.Time != nil:
return d.Time.Class
}
return ""
}
// classNone is the class of a delivery that moves no module: nothing for a walk to move, and no fix budget.
const classNone = "none"
// openedOf is when a delivery's pull request was first heard: the earliest head of it this owner holds, never
// one made from the forge's word at its merge; zero when none is held or it has no pull request. Under the lock.
func (h *Holder) openedOf(d *Delivery) time.Time {
if d.Number <= 0 {
return time.Time{}
}
var first time.Time
for _, c := range h.deliveries {
if c.FromForge || c.Number != d.Number || !strings.EqualFold(c.Repository, d.Repository) || c.Created.IsZero() {
continue
}
if first.IsZero() || c.Created.Before(first) {
first = c.Created
}
}
if first.IsZero() && !d.FromForge {
first = d.Created
}
return first
}
// noteLine is one transition as the commit's note keeps it.
func noteLine(d *Delivery, t Transition) string {
by := ""
@@ -172,9 +172,10 @@ func (h *Holder) refuse(err error) {
// moved records what a delivery's transitions owe outside, and keeps it.
func (h *Holder) moved(d *Delivery, ts []Transition) {
now := h.Now()
opened := h.openedOf(d)
for _, t := range ts {
h.Logf("[mesh-delivery] %s: %s → %s (%s): %s", d.ID, orNothing(t.From), t.To, t.Event, t.Why)
d.Owed = append(d.Owed, Effect{Kind: EffectEmit, Line: string(mustJSON(transitionEvent(d, t))), Since: now})
d.Owed = append(d.Owed, Effect{Kind: EffectEmit, Line: string(mustJSON(transitionEvent(d, t, opened))), Since: now})
line := noteLine(d, t)
d.Owed = append(d.Owed, Effect{Kind: EffectNote, Commit: d.Commit, Line: line, Since: now})
if d.MergedAs != "" && !sameCommit(d.MergedAs, d.Commit) && afterTheTrunk(t.To) {
@@ -0,0 +1,95 @@
package main
import (
"testing"
"time"
)
// novox/hq issue 403, ADR 0313: the issue tracker measures a fix's time from the move to fixing to delivered and
// splits it at the pull request's opening and its merge. A transition says what it reads them by: when the pull
// request was first heard (its first head, not a later one), when it merged, and the class of what it moves —
// core, leaf, or none for a change that moves no module. `checks` says the same for a fix the tracker hears of late.
// transitionsTo are the transition events said for a delivery that moved to a state.
func transitionsTo(w *world, id string, to State) []map[string]any {
var out []map[string]any
for _, e := range w.said {
if e["event-kind"] == "transition" && e["id"] == id && e["to"] == string(to) {
out = append(out, e)
}
}
return out
}
func TestATransitionSaysWhenItsPullRequestOpenedMergedAndItsClass(t *testing.T) {
w := newWorld(t)
repo := "novox/mesh-catalog"
opened := w.now
first := pr(repo, 9, "head9first", "fix/403-x", "modules/app/a.go")
w.h.PullUpdated(first)
w.later(3 * time.Minute)
second := pr(repo, 9, "head9second", "fix/403-x", "modules/app/a.go")
w.h.PullUpdated(second)
c := verdictMoving(repo, 9, "head9second", "app")
c.Plan.Class = "core"
w.h.Checked(c)
w.later(2 * time.Minute)
mergedWhen := w.now
w.h.PullMerged(mergedAt(second, "merge9aaaaaa", mergedWhen))
walk := aWalk("plan-merge9", repo, "merge9aaaaaa", false)
walk.Delivery = &WalkDelivery{Merges: []WalkMerge{{Repository: repo, Commit: "merge9aaaaaa"}}}
w.h.WalkMoved(w.ctl.put(walk))
w.settleAll()
walk = w.ctl.walk(walk.ID)
walk.State, walk.Revision = walkDone, walk.Revision+1
walk.Modules = map[string]*WalkModule{"app": {State: "built"}}
w.h.WalkMoved(w.ctl.put(walk))
w.settleAll()
id := IDOf(repo, "head9second")
if s := w.state(id); s != Delivered {
t.Fatalf("the merge is %s, not delivered", s)
}
said := transitionsTo(w, id, Delivered)
if len(said) == 0 {
t.Fatalf("no transition to delivered was said: %v", w.said)
}
e := said[len(said)-1]
if e["opened"] != opened.UTC().Format(time.RFC3339) {
t.Errorf("opened %v, want the first head's %s (not the second head's)", e["opened"], opened.Format(time.RFC3339))
}
if e["merged_at"] != mergedWhen.UTC().Format(time.RFC3339) {
t.Errorf("merged_at %v, want %s", e["merged_at"], mergedWhen.Format(time.RFC3339))
}
if e["class"] != "core" {
t.Errorf("class %v, want the plan's core", e["class"])
}
a, err := w.h.Checks(repo, 9, "")
if err != nil {
t.Fatal(err)
}
if a.Delivery == nil || a.Delivery.Opened != opened.UTC().Format(time.RFC3339) || a.Delivery.Class != "core" ||
a.Delivery.MergedAt != mergedWhen.UTC().Format(time.RFC3339) {
t.Errorf("checks says %+v", a.Delivery)
}
}
// A plan that moves no module is class none, so the tracker gives its fix no budget; a plan from a controller that
// says no class falls back to its walk's class once known, and says nothing before.
func TestAFixClassIsNoneWhenItMovesNothing(t *testing.T) {
for _, c := range []struct {
why string
d Delivery
want string
}{
{"the planner's class", Delivery{Plan: &DeliveryPlan{Moved: []string{"app"}, Class: "leaf"}}, "leaf"},
{"a plan that moves nothing, no class said", Delivery{Plan: planMoving("novox/x")}, "none"},
{"no class said, its walk's known", Delivery{Plan: planMoving("novox/x", "app"), Time: &DeliveryTime{Class: "core"}}, "core"},
{"no class said, nothing known", Delivery{Plan: planMoving("novox/x", "app")}, ""},
} {
if got := fixClass(&c.d); got != c.want {
t.Errorf("%s: %q, want %q", c.why, got, c.want)
}
}
}
@@ -165,6 +165,9 @@ type DeliveryPlan struct {
Machines []MachinePlan `json:"machines,omitempty"`
Steps []string `json:"steps,omitempty"`
Summary string `json:"summary"`
// Class is the class of the slowest-budgeted module it moves, as the planner says it: core, leaf, or none when
// it moves no module (novox/hq ADR 0313 §2). Empty in a plan from a controller that did not say it.
Class string `json:"class,omitempty"`
}
// receiving are the machines the plan sends a module to, in the plan's order; none when the plan names no
+37
View File
@@ -28,6 +28,7 @@ what it refuses.
| `groom` | optionally `pass`, `confirm` | the triage over the backlog: without arguments a dry run in the background, answering its pass id; with `pass` its progress and findings; with `confirm` and `pass` exactly that pass's findings applied, each checked again; see below |
| `tracking` | optionally `conditions` (condition keys) | for every condition key an issue links (link kind `condition`), the issues that link it, those not closed first: status, severity, needs, worker with since, active and stale, and where the fix is (`fix`); the operator's channel says it in a condition's message (hq ADR 0285) |
| `reserve` | `kind` (`decision` or `research`), optionally `for` (one line: what the record is about) | the next number of that kind, spent forever; answers `kind` and `number` (and `prefix`, the number as the record's name starts with it); see below |
| `fix-times` | optionally `days` (1 to 90, default 7) | each class's fix budget, the median and worst fix time of the issues whose last fix was delivered in the window, within and over, and every issue over its budget with its longest phase; see *A fix's time* |
**The cycle's rules, at the write** (hq playbook 03, which `00-META/checks/cycle.py` enforced on the
folders until the import, and hq ADR 0284): a status is one of `open`,
@@ -142,6 +143,35 @@ acceptance live, so a fix whose own walk failed and that a later merge carried i
issue resolved or wontfix is never said stuck. A module that runs only
its bundle cannot raise a controller condition (hq ADR 0240 rule 8, ADR 0256), so the warning is the channel's, not the controller's.
## A fix's time (hq ADR 0313)
An issue's **fix time** runs from its first move to `fixing` (kept as `fixing-from`; a take-over or a later move
back to `fixing` does not restart it, and an issue moved before the moment was kept has it read from its log) to the
moment the last of its fixes is delivered. Once every fix it names is delivered, `get` carries it as `fix-time`:
`from`, `end`, `took`, the issue's `class`, its **fix budget** (`budget`: five minutes for a leaf module, ten for a
core one, counted from the move to `fixing`), `over`, the fix delivered last (`fix`), the `worker`, and four
`phases` read from that fix, which add up to the fix time:
| phase | ends at | read from |
|---|---|---|
| implementing | its pull request opened | mesh-delivery's `opened` (the pull request's first head heard) |
| review | the first approval at its head | the forge's reviews (`gitea_pull_reviews`) |
| checks | its merge, required checks and any wait for the merge included | mesh-delivery's `merged_at` |
| merge to running | delivered | the delivery's `delivered` (its delivery time) |
A phase whose end came before the end of the phase before it counts zero; a phase whose end is not known (no
approval at the head on the forge, a delivery that did not say when the pull request opened) is `unknown`, and its
time counts in the next phase, which says so. The class is that of the slowest-budgeted module the fixes move, as
mesh-delivery says it from the delivery plan (`class`: `core`, `leaf`, or `none` for a fix that moves no module);
an issue whose fixes move no module, or whose class is not said, is measured and shown with no fix budget.
**Over its budget is loud.** When the last fix is delivered and the fix time exceeds the class's fix budget, the
operator's channel is told (`seat:operator-channel.notify`, key `issue-tracker.fix.<class>.over-budget`), naming the
issue, its worker and its longest phase. Nothing is said while a fix is undelivered: a merged fix not delivered is
loud as a stuck fix. The warning clears when the next issue of that class is delivered within its budget; the
newest fix time of each class decides, and every sweep says or clears what a failed ask left. The delivery budget
(mesh-delivery's `times`, from the merge) stays a measure inside the fix budget.
## The import of hq's issues (slice 3)
`issue-tracker.import` reads every `04-ISSUES/NNN-slug/00-report.md` of hq **through the records module
@@ -364,6 +394,13 @@ reserve spending nothing, the first reserve refused while the knowledge base doe
set, a kind held none of being no answer, hq's folders never read, and a
reservation that cannot be kept leaving its number spent.
A fix's time is tested in `fixtime_test.go`: the start at the first move to `fixing` through a diagnosis, a
take-over and a move away and back; phases that add up to the fix time, one ended early counting zero and one not
known counted in the next; the class of the slowest fix, none and not known with no budget; no warning while
undelivered, the warning once delivered over budget, cleared by the next of its class within budget and not by
another class, and said at the next sweep when the channel did not take it; `fix-times` over a window, an issue
before it not counted; the start read from the log; and the first approval at the head.
The import is tested over `testdata/hq`, copies of seven real reports of hq chosen for their shapes (a
bracketed list with commas inside parentheses, quoted block items, an apostrophe in prose, a `resolved:`
date, a replay, an empty list, a wontfix): the frontmatter read as written, each issue created at its own
@@ -145,7 +145,7 @@ func (t *Tracker) askUnheard() error {
if since.IsZero() {
since = t.now()
}
if _, err := t.recordFix(pr, d.State, d.Why, d.Merged, since, n); err != nil {
if _, err := t.recordFix(pr, d.heard(since), n); err != nil {
return err
}
heard = true
+94 -14
View File
@@ -37,6 +37,34 @@ type Fix struct {
Since time.Time `json:"since"`
// StuckSaid says it was said stuck, on the issue and to the operator's channel.
StuckSaid bool `json:"stuck-said,omitempty"`
// Class is the class of what its delivery moves, as mesh-delivery says it: core, leaf, or none when it moves
// no module; Opened when its pull request was first heard (novox/hq ADR 0313 §2, §4).
Class string `json:"class,omitempty"`
Opened *time.Time `json:"opened,omitempty"`
}
// heardFix is what the delivery said of a pull request at one moment: its state, why, whether it is merged, and
// what a fix's time is read by — the class of what it moves, when its pull request opened and when it merged,
// each when said.
type heardFix struct {
State string
Why string
Merged bool
At time.Time
Class string
Opened *time.Time
MergedAt *time.Time
}
// news says whether what was heard adds anything to what a fix keeps.
func (h heardFix) news(f *Fix, kept string) bool {
if f.State != h.State || f.Why != kept || (h.Merged && !f.Merged) || (h.Class != "" && h.Class != f.Class) {
return true
}
if h.Opened != nil && (f.Opened == nil || !f.Opened.Equal(*h.Opened)) {
return true
}
return h.MergedAt != nil && (f.MergedAt == nil || !f.MergedAt.Equal(*h.MergedAt))
}
// reached says whether a delivery's state says the change reached the machines (mesh-delivery's Reached).
@@ -147,7 +175,8 @@ func DeliveryLine(is *Issue) string {
// recordFix keeps what the delivery said of a pull request on every issue whose fixed-by names it (only, when
// given, on that one issue), and answers the issues it changed.
func (t *Tracker) recordFix(pr, state, why string, merged bool, at time.Time, only uint64) ([]uint64, error) {
func (t *Tracker) recordFix(pr string, h heardFix, only uint64) ([]uint64, error) {
state, why, merged, at := h.State, h.Why, h.Merged, h.At
all, err := t.allIssues()
if err != nil {
return nil, err
@@ -158,7 +187,7 @@ func (t *Tracker) recordFix(pr, state, why string, merged bool, at time.Time, on
continue
}
kept := cut(why, MostReasonKept)
if f := fixOf(is, pr); f != nil && (at.Before(f.Since) || (f.State == state && f.Why == kept && f.Merged == (merged || f.Merged))) {
if f := fixOf(is, pr); f != nil && (at.Before(f.Since) || !h.news(f, kept)) {
// Older than what is kept, or nothing new: transitions can arrive out of order.
continue
}
@@ -179,6 +208,17 @@ func (t *Tracker) recordFix(pr, state, why string, merged bool, at time.Time, on
m := at
f.MergedAt = &m
}
if h.MergedAt != nil && (merged || f.Merged) {
m := h.MergedAt.UTC()
f.MergedAt = &m // the delivery's own moment of the merge, not when it was heard
}
if h.Class != "" {
f.Class = h.Class
}
if h.Opened != nil {
o := h.Opened.UTC()
f.Opened = &o
}
if reached(state) && !contains(i.Delivered, pr) {
i.Delivered = append(i.Delivered, pr)
}
@@ -207,6 +247,10 @@ func (t *Tracker) afterFix(n uint64) error {
t.Logf("[mesh-issues] the warning that %d is delivered and not resolved could not be cleared on the operator's channel (tried again at the next sweep): %v", n, err)
}
if allDelivered(is) {
// Its fix time, final once its last fix is delivered (novox/hq ADR 0313).
if err := t.timeFix(n); err != nil {
return err
}
if unresolved(is) {
prs := liveFixes(is)
sort.Strings(prs)
@@ -321,12 +365,15 @@ func prKey(repository string, number int) string {
// to is kept on each issue whose fixed-by names it.
func (t *Tracker) OnTransition(e stdio.Envelope) error {
var body struct {
Repository string `json:"repository"`
Number int `json:"number"`
To string `json:"to"`
Why string `json:"why"`
MergedAs string `json:"merged_as"`
At time.Time `json:"at"`
Repository string `json:"repository"`
Number int `json:"number"`
To string `json:"to"`
Why string `json:"why"`
MergedAs string `json:"merged_as"`
At time.Time `json:"at"`
Class string `json:"class"`
Opened *time.Time `json:"opened"`
MergedAt *time.Time `json:"merged_at"`
}
if err := json.Unmarshal(e.Body, &body); err != nil || body.To == "" || body.Number <= 0 {
return nil
@@ -336,7 +383,8 @@ func (t *Tracker) OnTransition(e stdio.Envelope) error {
at = t.now()
}
merged := body.MergedAs != ""
changed, err := t.recordFix(prKey(body.Repository, body.Number), body.To, body.Why, merged, at, 0)
changed, err := t.recordFix(prKey(body.Repository, body.Number), heardFix{State: body.To, Why: body.Why, Merged: merged,
At: at, Class: body.Class, Opened: nonZero(body.Opened), MergedAt: nonZero(body.MergedAt)}, 0)
if err != nil {
return err
}
@@ -356,6 +404,24 @@ type Delivery struct {
Since time.Time
// Merged is the pull request's own merged flag, as the forge says it.
Merged bool
// Class, Opened and MergedAt are what a fix's time is read by (novox/hq ADR 0313), when the delivery says them.
Class string
Opened *time.Time
MergedAt *time.Time
}
// heard is what a delivery answered, as a fix keeps it.
func (d Delivery) heard(since time.Time) heardFix {
return heardFix{State: d.State, Why: d.Why, Merged: d.Merged, At: since, Class: d.Class, Opened: d.Opened, MergedAt: d.MergedAt}
}
// nonZero is a moment said, or nil for one not said.
func nonZero(t *time.Time) *time.Time {
if t == nil || t.IsZero() {
return nil
}
u := t.UTC()
return &u
}
// DeliveryState asks mesh-delivery for one pull request's delivery.
@@ -370,9 +436,12 @@ func busDeliveryState(owner string) DeliveryState {
}
var out struct {
Delivery struct {
State string `json:"state"`
Waits string `json:"waits"`
Since time.Time `json:"since"`
State string `json:"state"`
Waits string `json:"waits"`
Since time.Time `json:"since"`
Class string `json:"class"`
Opened *time.Time `json:"opened"`
MergedAt *time.Time `json:"merged_at"`
} `json:"delivery"`
Pull struct {
Merged bool `json:"merged"`
@@ -381,7 +450,8 @@ func busDeliveryState(owner string) DeliveryState {
if err := json.Unmarshal(raw, &out); err != nil {
return Delivery{}, fmt.Errorf("mesh-delivery's checks answered no delivery: %.200s", raw)
}
return Delivery{State: out.Delivery.State, Why: out.Delivery.Waits, Since: out.Delivery.Since.UTC(), Merged: out.Pull.Merged}, nil
return Delivery{State: out.Delivery.State, Why: out.Delivery.Waits, Since: out.Delivery.Since.UTC(), Merged: out.Pull.Merged,
Class: out.Delivery.Class, Opened: nonZero(out.Delivery.Opened), MergedAt: nonZero(out.Delivery.MergedAt)}, nil
}
}
@@ -430,7 +500,7 @@ func (t *Tracker) OnMoved(e stdio.Envelope) error {
if since.IsZero() {
since = t.now()
}
if _, err := t.recordFix(pr, state, d.Why, d.Merged, since, body.Number); err != nil {
if _, err := t.recordFix(pr, d.heard(since), body.Number); err != nil {
return err
}
}
@@ -481,6 +551,13 @@ func (t *Tracker) Sweep() (int, error) {
bound, now := t.stuckAfter(), t.now()
said := 0
for n, is := range all {
// A fix time missed at its delivery (a failure, or a restart between) is kept now; one of an issue moved to
// fixing before the tracker kept the moment is kept only when its fix is heard of again.
if is.FixingFrom != nil && is.FixTime == nil && allDelivered(is) {
if err := t.timeFix(n); err != nil {
t.Logf("[mesh-issues] the fix time of %d could not be kept (tried again at the next sweep): %v", n, err)
}
}
ok, err := t.sayNotResolved(is, now)
if err != nil {
t.Logf("[mesh-issues] that %d is delivered and not resolved could not be said on the operator's channel (said at the next sweep): %v", n, err)
@@ -537,5 +614,8 @@ func (t *Tracker) Sweep() (int, error) {
said++
}
}
if err := t.judgeFixBudgets(); err != nil {
t.Logf("[mesh-issues] a fix over its budget could not be said or cleared on the operator's channel (tried again at the next sweep): %v", err)
}
return said, nil
}
@@ -0,0 +1,597 @@
package main
// A fix's time (novox/hq ADR 0313, issue 403). An issue's fix time runs from its first move to fixing to the moment
// the last of its fixes is delivered to every machine it should reach. It is held to the fix budget of its class —
// five minutes for a leaf module, ten for a core one — and split into four phases read from the fix delivered
// last: implementing (to its pull request opened), review (to an approval at its head), checks (to its merge) and
// merge to running (to delivered, its delivery time). A fix over its budget is said on the operator's channel once
// it is final, naming the issue, its worker and its longest phase; the warning clears when the next issue of that
// class is delivered within its budget. `fix-times` answers the last days' median and worst per class.
import (
"encoding/json"
"fmt"
"sort"
"strings"
"time"
stdio "git.novox.be/novox/mesh-sdk/go"
)
// The classes of a fix (ADR 0282 §1, ADR 0313 §2) and the class of a fix that moves no module.
const (
ClassCore = "core"
ClassLeaf = "leaf"
ClassNone = "none"
)
// FixBudgets are the fix budgets by class, counted from the move to fixing (ADR 0313 §2).
var FixBudgets = map[string]time.Duration{ClassLeaf: 5 * time.Minute, ClassCore: 10 * time.Minute}
// fixClasses in the order `fix-times` says them.
var fixClasses = []string{ClassLeaf, ClassCore}
// The phases of a fix time, in order (ADR 0313 §4).
const (
PhaseImplementing = "implementing"
PhaseReview = "review"
PhaseChecks = "checks"
PhaseRunning = "merge to running"
)
// FixTime is an issue's fix time, kept on it once the last of its fixes is delivered.
type FixTime struct {
// From is the first move to fixing; End when the fix delivered last was delivered.
From time.Time `json:"from"`
End time.Time `json:"end"`
TookMS int64 `json:"took_ms"`
Took string `json:"took"`
// Class is core, leaf or none; empty when a fix's class is not known. A fix of class none, or one not known,
// has no budget: its time is shown and never warned on.
Class string `json:"class,omitempty"`
Budget string `json:"budget,omitempty"`
BudgetMS int64 `json:"budget_ms,omitempty"`
Over bool `json:"over,omitempty"`
// Fix is the pull request delivered last, which the phases are read from; Worker who worked on the issue.
Fix string `json:"fix"`
Worker string `json:"worker,omitempty"`
Phases []FixPhase `json:"phases"`
Longest string `json:"longest,omitempty"`
Said string `json:"said"`
// Warned says the over-budget warning was said for it on the operator's channel and not cleared since.
Warned bool `json:"warned,omitempty"`
}
// FixPhase is one phase of a fix time: measured, or unknown when the moment that ends it is not known — its time is
// then counted in the phase after it, which says so.
type FixPhase struct {
Name string `json:"name"`
State string `json:"state"`
TookMS int64 `json:"took_ms"`
Took string `json:"took"`
Said string `json:"said,omitempty"`
}
// Approval is when a pull request was first approved at its head, as the forge says it; a test stubs the asking.
// found is false when no approval at its head is on the forge.
type Approval func(repository string, number int) (at time.Time, found bool, err error)
// classOfFixes is the class of an issue's fixes: that of the slowest-budgeted module they move — core when any
// moves a core module, leaf when any moves a leaf one, none when none moves a module — and empty, with why, when
// a fix's class is not known and none is core.
func classOfFixes(is *Issue, prs []string) (string, string) {
var unknown []string
leaf := false
for _, pr := range prs {
f := fixOf(is, pr)
switch {
case f == nil || f.Class == "":
unknown = append(unknown, pr)
case f.Class == ClassCore:
return ClassCore, ""
case f.Class == ClassLeaf:
leaf = true
}
}
if len(unknown) > 0 {
return "", "the delivery did not say what " + strings.Join(unknown, ", ") + " moves"
}
if leaf {
return ClassLeaf, ""
}
return ClassNone, ""
}
// fixPhases splits the span from start to the delivered moment at the moments that end each phase, in order. A
// phase whose end is not known is unknown, and its time counts in the next phase known; a phase whose end came
// before the end of the phase before it counts zero (ADR 0313 §4). The phases add up to the fix time.
func fixPhases(start time.Time, ends []struct {
name string
at *time.Time
why string
}) ([]FixPhase, string) {
var out []FixPhase
prev := start
var carried []string
for _, e := range ends {
if e.at == nil {
out = append(out, FixPhase{Name: e.name, State: "unknown", Took: "0s", Said: e.why})
carried = append(carried, e.name)
continue
}
took := time.Duration(0)
if e.at.After(prev) {
took = e.at.Sub(prev)
prev = *e.at
}
p := FixPhase{Name: e.name, State: "measured", TookMS: took.Milliseconds(), Took: took.Round(time.Second).String()}
if len(carried) > 0 {
p.Said = "counts " + strings.Join(carried, " and ") + " too: its end is not known"
carried = nil
}
out = append(out, p)
}
best := -1
for i, p := range out {
if p.State == "measured" && (best < 0 || p.TookMS > out[best].TookMS) {
best = i
}
}
longest := ""
if best >= 0 {
longest = out[best].Name + " " + out[best].Took
}
return out, longest
}
// startOfFixing is when an issue was first moved to fixing: the earliest of what it keeps and what its log says
// (an issue moved before the tracker kept the moment is read from its log, a value carried into the log at its
// worker's since). Zero when it never was.
func (t *Tracker) startOfFixing(is *Issue) time.Time {
var start time.Time
if is.FixingFrom != nil {
start = *is.FixingFrom
}
if t.Log == nil {
return start
}
entries, err := t.readKey(is.Number, 0)
if err != nil {
t.Logf("[mesh-issues] the log of %d could not be read for its first move to fixing (its kept moment is used): %v", is.Number, err)
return start
}
status := ""
for _, e := range entries {
var op Operation
if json.Unmarshal(e.Value, &op) != nil {
continue
}
at, became := op.At, ""
switch {
case op.Issue != nil:
became = op.Issue.Status
if op.Issue.Worker != nil && !op.Issue.Worker.Since.IsZero() {
at = op.Issue.Worker.Since
}
case op.Set["status"] != nil:
_ = json.Unmarshal(op.Set["status"], &became)
default:
continue
}
if became == StatusFixing && status != StatusFixing && !at.IsZero() {
if start.IsZero() || at.Before(start) {
start = at.UTC()
}
break
}
status = became
}
return start
}
// fixTimeOf is an issue's fix time once every fix it names is delivered; nil, with why, when it is not measured.
func (t *Tracker) fixTimeOf(is *Issue) (*FixTime, string) {
if !allDelivered(is) {
return nil, "not every fix is delivered"
}
start := t.startOfFixing(is)
if start.IsZero() {
return nil, "it was never moved to fixing"
}
prs := liveFixes(is)
var last *Fix
for _, pr := range prs {
if f := fixOf(is, pr); f != nil && (last == nil || f.Since.After(last.Since)) {
last = f
}
}
if last == nil {
return nil, "no fix is heard of"
}
end := last.Since.UTC()
took := time.Duration(0)
if end.After(start) {
took = end.Sub(start)
}
ft := &FixTime{From: start, End: end, TookMS: took.Milliseconds(), Took: took.Round(time.Second).String(), Fix: last.PullRequest}
if is.Worker != nil {
ft.Worker = is.Worker.By
} else if is.FixTime != nil {
ft.Worker = is.FixTime.Worker
}
class, why := classOfFixes(is, prs)
ft.Class = class
if b, ok := FixBudgets[class]; ok {
ft.Budget, ft.BudgetMS, ft.Over = b.String(), b.Milliseconds(), took > b
}
// The phases, from the fix delivered last.
type end_ = struct {
name string
at *time.Time
why string
}
var approved *time.Time
approvalWhy := "no approval at its head is on the forge: review and checks are counted together"
if t.Approval != nil {
repo, number, _ := strings.Cut(last.PullRequest, "#")
var n int
if _, err := fmt.Sscanf(number, "%d", &n); err == nil {
at, found, err := t.Approval(repo, n)
switch {
case err != nil:
approvalWhy = "the forge could not be read for its approval (" + cut(err.Error(), 200) + "): review and checks are counted together"
case found:
a := at.UTC()
approved = &a
}
}
} else {
approvalWhy = "the forge is not asked here: review and checks are counted together"
}
openedWhy := "the delivery did not say when its pull request opened: implementing is counted in review"
mergedWhy := "the delivery did not say when it merged: checks are counted in merge to running"
ft.Phases, ft.Longest = fixPhases(start, []end_{
{PhaseImplementing, last.Opened, openedWhy},
{PhaseReview, approved, approvalWhy},
{PhaseChecks, last.MergedAt, mergedWhy},
{PhaseRunning, &end, ""},
})
var said []string
said = append(said, fmt.Sprintf("fixed in %s, from the first move to fixing (%s) to %s delivered (%s)", ft.Took,
start.Format(time.RFC3339), last.PullRequest, end.Format(time.RFC3339)))
switch {
case class == ClassNone:
said = append(said, "its fixes move no module, so it has no fix budget")
case class == "":
said = append(said, "no fix budget: "+why)
case ft.Over:
said = append(said, fmt.Sprintf("over its %s fix budget of %s", class, ft.Budget))
default:
said = append(said, fmt.Sprintf("within its %s fix budget of %s", class, ft.Budget))
}
if ft.Longest != "" {
said = append(said, "longest: "+ft.Longest)
}
ft.Said = strings.Join(said, "; ")
return ft, ""
}
// sameFixTime says whether two fix times say the same, the warning's mark aside.
func sameFixTime(a, b *FixTime) bool {
if a == nil || b == nil {
return a == b
}
x, y := *a, *b
x.Warned, y.Warned = false, false
ra, _ := json.Marshal(x)
rb, _ := json.Marshal(y)
return string(ra) == string(rb)
}
// timeFix keeps an issue's fix time once every fix it names is delivered, and again when a later fix moves its
// end; then the warnings of its class are judged.
func (t *Tracker) timeFix(n uint64) error {
is, _, err := read(t.Store, n)
if err != nil {
return err
}
ft, _ := t.fixTimeOf(is)
if ft == nil || sameFixTime(ft, is.FixTime) {
return nil
}
if _, err := t.change(n, Act{Op: "fix-time", By: Seat}, func(i *Issue) error {
if i.FixTime != nil {
ft.Warned = i.FixTime.Warned
}
i.FixTime = ft
return nil
}); err != nil {
return err
}
if err := t.judgeFixBudgets(); err != nil {
t.Logf("[mesh-issues] a fix over its budget could not be said or cleared on the operator's channel (tried again at the next sweep): %v", err)
}
return nil
}
// fixOverKey is the warning a fix over its class's budget raises (ADR 0313 §6).
func fixOverKey(class string) string { return "issue-tracker.fix." + class + ".over-budget" }
// judgeFixBudgets says or clears each class's warning by the newest fix time of the class (ADR 0313 §6): the newest
// over its budget and not yet said is said; the newest within its budget clears what was said for the class. Run
// after every fix time kept and at every sweep, so a warning that failed is said, or cleared, at the next.
func (t *Tracker) judgeFixBudgets() error {
if t.Notify == nil {
return nil
}
all, err := t.allIssues()
if err != nil {
return err
}
var firstErr error
for _, class := range fixClasses {
var newest *Issue
var warned []uint64
for _, is := range all {
ft := is.FixTime
if ft == nil || ft.Class != class {
continue
}
if ft.Warned {
warned = append(warned, is.Number)
}
if newest == nil || ft.End.After(newest.FixTime.End) || (ft.End.Equal(newest.FixTime.End) && is.Number > newest.Number) {
newest = is
}
}
if newest == nil {
continue
}
ft := newest.FixTime
if ft.Over {
if ft.Warned {
continue
}
worker := ft.Worker
if worker == "" {
worker = "nobody named"
}
longest := ft.Longest
if longest == "" {
longest = "not known"
}
if err := t.Notify(fixOverKey(class), "warning",
fmt.Sprintf("issue %d: fixed in %s, over its %s fix budget of %s (longest: %s)", newest.Number, ft.Took, class, ft.Budget, longest),
fmt.Sprintf("A fix took longer than its budget (issue %d)", newest.Number),
fmt.Sprintf("Issue %d was fixed in %s, from %s's move to fixing to its last fix delivered, over the %s fix "+
"budget of %s. Its longest phase was %s. Nothing is needed from you: it says where fixes lose time. "+
"It clears when the next %s fix is delivered within its budget.", newest.Number, ft.Took, worker, class,
ft.Budget, longest, class), "", false); err != nil {
if firstErr == nil {
firstErr = err
}
continue
}
if _, err := t.change(newest.Number, Act{Op: "warned", By: Seat}, func(i *Issue) error {
if i.FixTime != nil {
i.FixTime.Warned = true
}
return nil
}); err != nil && firstErr == nil {
firstErr = err
}
continue
}
if len(warned) == 0 || ft.BudgetMS == 0 {
continue
}
if err := t.Notify(fixOverKey(class), "", "", "", "", fmt.Sprintf("issue %d's fix was delivered within its %s fix budget (%s of %s)",
newest.Number, class, ft.Took, ft.Budget), true); err != nil {
if firstErr == nil {
firstErr = err
}
continue
}
sort.Slice(warned, func(i, j int) bool { return warned[i] < warned[j] })
for _, n := range warned {
if _, err := t.change(n, Act{Op: "warned", By: Seat}, func(i *Issue) error {
if i.FixTime != nil {
i.FixTime.Warned = false
}
return nil
}); err != nil && firstErr == nil {
firstErr = err
}
}
}
return firstErr
}
// FixTimesClass is one class's line of `fix-times`.
type FixTimesClass struct {
Class string `json:"class"`
Budget string `json:"budget"`
BudgetMS int64 `json:"budget_ms"`
Count int `json:"count"`
Within int `json:"within"`
Over int `json:"over"`
Median string `json:"median,omitempty"`
MedianMS int64 `json:"median_ms,omitempty"`
Worst string `json:"worst,omitempty"`
WorstMS int64 `json:"worst_ms,omitempty"`
// WorstIssue is the issue whose fix took the worst time.
WorstIssue uint64 `json:"worst_issue,omitempty"`
}
// FixTimesRow is an issue as `fix-times` names it.
type FixTimesRow struct {
Number uint64 `json:"number"`
Title string `json:"title"`
Class string `json:"class,omitempty"`
Took string `json:"took"`
TookMS int64 `json:"took_ms"`
Budget string `json:"budget,omitempty"`
Longest string `json:"longest,omitempty"`
Worker string `json:"worker,omitempty"`
Delivered string `json:"delivered"`
}
// FixTimes is the answer of `fix-times`.
type FixTimes struct {
Days int `json:"days"`
Since string `json:"since"`
Line string `json:"line"`
Classes []FixTimesClass `json:"classes"`
// Over is every issue over its budget in the window, newest first, with its longest phase.
Over []FixTimesRow `json:"over,omitempty"`
// Unbudgeted are the issues fixed in the window with no fix budget: their fixes move no module, or what they
// move is not known.
Unbudgeted []FixTimesRow `json:"unbudgeted,omitempty"`
}
// MostFixTimesDays is the longest window `fix-times` reads.
const MostFixTimesDays = 90
// FixTimes is the `fix-times` verb: each class's fix budget, the last days' median and worst fix time, and every
// issue over its budget with its longest phase (ADR 0313 §5, §7). An issue counts in the window its last fix was
// delivered in.
func (t *Tracker) FixTimes(days int) (*FixTimes, error) {
if days <= 0 {
days = 7
}
if days > MostFixTimesDays {
return nil, fmt.Errorf("days %d is more than the %d fix-times reads", days, MostFixTimesDays)
}
all, err := t.allIssues()
if err != nil {
return nil, err
}
since := t.now().Add(-time.Duration(days) * 24 * time.Hour)
a := &FixTimes{Days: days, Since: since.Format(time.RFC3339)}
byClass := map[string][]*Issue{}
row := func(is *Issue) FixTimesRow {
ft := is.FixTime
return FixTimesRow{Number: is.Number, Title: is.Title, Class: ft.Class, Took: ft.Took, TookMS: ft.TookMS, Budget: ft.Budget,
Longest: ft.Longest, Worker: ft.Worker, Delivered: ft.End.Format(time.RFC3339)}
}
for _, is := range all {
ft := is.FixTime
if ft == nil || ft.End.Before(since) {
continue
}
if _, ok := FixBudgets[ft.Class]; !ok {
a.Unbudgeted = append(a.Unbudgeted, row(is))
continue
}
byClass[ft.Class] = append(byClass[ft.Class], is)
if ft.Over {
a.Over = append(a.Over, row(is))
}
}
newestFirst := func(rows []FixTimesRow) {
sort.Slice(rows, func(i, j int) bool {
if rows[i].Delivered != rows[j].Delivered {
return rows[i].Delivered > rows[j].Delivered
}
return rows[i].Number > rows[j].Number
})
}
newestFirst(a.Over)
newestFirst(a.Unbudgeted)
var line []string
for _, class := range fixClasses {
b := FixBudgets[class]
c := FixTimesClass{Class: class, Budget: b.String(), BudgetMS: b.Milliseconds()}
is := byClass[class]
sort.Slice(is, func(i, j int) bool {
if is[i].FixTime.TookMS != is[j].FixTime.TookMS {
return is[i].FixTime.TookMS < is[j].FixTime.TookMS
}
return is[i].Number < is[j].Number
})
c.Count = len(is)
for _, i := range is {
if i.FixTime.Over {
c.Over++
} else {
c.Within++
}
}
if n := len(is); n > 0 {
ms := is[n/2].FixTime.TookMS
if n%2 == 0 {
ms = (is[n/2-1].FixTime.TookMS + is[n/2].FixTime.TookMS) / 2
}
med := time.Duration(ms) * time.Millisecond
c.MedianMS, c.Median = ms, med.Round(time.Second).String()
worst := is[n-1]
c.WorstMS, c.Worst, c.WorstIssue = worst.FixTime.TookMS, worst.FixTime.Took, worst.Number
line = append(line, fmt.Sprintf("%s: median %s, worst %s (issue %d), %d of %d within %s", class, c.Median,
c.Worst, c.WorstIssue, c.Within, c.Count, c.Budget))
} else {
line = append(line, fmt.Sprintf("%s: no issue fixed", class))
}
a.Classes = append(a.Classes, c)
}
a.Line = fmt.Sprintf("fix time over the last %d day(s), from the move to fixing to delivered — %s", days, strings.Join(line, "; "))
if len(a.Unbudgeted) > 0 {
a.Line += fmt.Sprintf("; %d fixed with no fix budget", len(a.Unbudgeted))
}
return a, nil
}
// firstApproval is when a pull request was first approved at its head among the forge's reviews: an approval
// dismissed, or of an older head, does not count.
func firstApproval(raw json.RawMessage) (time.Time, bool, error) {
var a struct {
Reviews []struct {
State string `json:"state"`
AtHead bool `json:"at_head"`
Submitted string `json:"submitted"`
Stale bool `json:"stale"`
Dismissed bool `json:"dismissed"`
} `json:"reviews"`
}
var s string
if json.Unmarshal(raw, &s) == nil {
raw = json.RawMessage(s)
}
var wrapped struct {
Content []struct {
Text string `json:"text"`
} `json:"content"`
}
if json.Unmarshal(raw, &wrapped) == nil && len(wrapped.Content) > 0 {
raw = json.RawMessage(wrapped.Content[0].Text)
}
if err := json.Unmarshal(raw, &a); err != nil || a.Reviews == nil {
return time.Time{}, false, fmt.Errorf("the forge's gitea_pull_reviews answered no reviews: %.200s", raw)
}
var first time.Time
for _, r := range a.Reviews {
if r.State != "APPROVED" || !r.AtHead || r.Stale || r.Dismissed {
continue
}
at, err := time.Parse(time.RFC3339, r.Submitted)
if err != nil {
continue
}
if first.IsZero() || at.Before(first) {
first = at
}
}
return first, !first.IsZero(), nil
}
// busApproval asks the forge's reviews of a pull request over the bus (gitea_pull_reviews).
func busApproval(owner func() string) Approval {
return func(repository string, number int) (time.Time, bool, error) {
raw, err := stdio.Ask("gitea.gitea_pull_reviews", map[string]any{"owner": owner(), "repo": repository, "number": number})
if err != nil {
return time.Time{}, false, err
}
return firstApproval(raw)
}
}
@@ -0,0 +1,392 @@
package main
import (
"encoding/json"
"strings"
"testing"
"time"
stdio "git.novox.be/novox/mesh-sdk/go"
)
// novox/hq ADR 0313 (issue 403): an issue's fix time runs from its first move to fixing to its last fix
// delivered; it is held to its class's fix budget, split into four phases that add up to it, shown in get and
// fix-times, and a fix over its budget is said on the operator's channel once final, cleared at the next of its
// class delivered within budget.
// fixClock is a harness whose clock a test moves.
type fixClock struct {
*harness
at time.Time
// approvals is when each pull request was approved at its head, as the forge would say it.
approvals map[int]time.Time
}
func newFixClock(t *testing.T) *fixClock {
c := &fixClock{harness: newHarness(t), at: time.Date(2026, 10, 12, 9, 0, 0, 0, time.UTC), approvals: map[int]time.Time{}}
c.Tracker.Now = func() time.Time { return c.at }
return c
}
func (c *fixClock) approval(repository string, number int) (time.Time, bool, error) {
a, ok := c.approvals[number]
return a, ok, nil
}
func (c *fixClock) later(d time.Duration) time.Time { c.at = c.at.Add(d); return c.at }
// delivery says one transition of a pull request as mesh-delivery says it, with what a fix's time is read by.
func (c *fixClock) delivery(t *testing.T, number int, to, class string, opened, mergedAt *time.Time) {
t.Helper()
body := map[string]any{"repository": "novox/mesh-catalog", "number": number, "to": to, "at": c.at}
if mergedAt != nil {
body["merged_as"] = "m" + strings.Repeat("a", 11)
body["merged_at"] = *mergedAt
}
if class != "" {
body["class"] = class
}
if opened != nil {
body["opened"] = *opened
}
raw, _ := json.Marshal(body)
if err := c.OnTransition(stdio.Envelope{Key: "mesh-delivery.transition", Body: raw}); err != nil {
t.Fatal(err)
}
}
func at(t time.Time) *time.Time { return &t }
func phaseTook(t *testing.T, ft *FixTime) map[string]time.Duration {
t.Helper()
out := map[string]time.Duration{}
var sum int64
for _, p := range ft.Phases {
out[p.Name] = time.Duration(p.TookMS) * time.Millisecond
sum += p.TookMS
}
if sum != ft.TookMS {
t.Fatalf("the phases add up to %dms, the fix time is %dms: %+v", sum, ft.TookMS, ft.Phases)
}
return out
}
// fixOne takes an issue through a fix: opened, moved to fixing, a pull request opened, approved, merged and
// delivered after the given spans; its fix time as kept.
func (c *fixClock) fixOne(t *testing.T, title string, pr int, class string, implementing, review, checks, running time.Duration) uint64 {
t.Helper()
n := c.open(t, title)
c.fixing(t, n)
if _, err := c.Move(MoveAsk{Number: n, Status: StatusFixing, By: "fix@g14", FixedBy: []string{"mesh-catalog#" + itoa(pr)}}); err != nil {
t.Fatal(err)
}
opened := c.later(implementing)
c.delivery(t, pr, "proposed", "", &opened, nil)
approved := c.later(review)
c.approvals[pr] = approved
merged := c.later(checks)
c.delivery(t, pr, "published", class, &opened, &merged)
c.later(running)
c.delivery(t, pr, "delivered", class, &opened, &merged)
return n
}
func itoa(n int) string { b, _ := json.Marshal(n); return string(b) }
func TestAFixTimeRunsFromTheFirstMoveToFixingAndAddsUpItsPhases(t *testing.T) {
c := newFixClock(t)
c.Tracker.Approval = c.approval
note := &notified{}
c.Notify = note.notify
n := c.open(t, "a fix measured")
c.later(time.Minute)
if _, err := c.Move(MoveAsk{Number: n, Status: StatusDiagnosing, By: "fix@g14"}); err != nil {
t.Fatal(err)
}
c.later(10 * time.Minute) // finding the cause is not the fix's time
start := c.at
if _, err := c.Move(MoveAsk{Number: n, Status: StatusFixing, By: "fix@g14", FixedBy: []string{"mesh-catalog#31"}}); err != nil {
t.Fatal(err)
}
c.later(30 * time.Second)
// A take-over, and a move away and back, restart nothing.
if _, err := c.Move(MoveAsk{Number: n, Status: StatusFixing, By: "other@shanks", TakeOver: "the first worker's session ended"}); err != nil {
t.Fatal(err)
}
if _, err := c.Move(MoveAsk{Number: n, Status: StatusOpen}); err != nil {
t.Fatal(err)
}
if _, err := c.Move(MoveAsk{Number: n, Status: StatusFixing, By: "other@shanks"}); err != nil {
t.Fatal(err)
}
opened := c.later(90 * time.Second) // implementing: 2m
c.delivery(t, 31, "proposed", "", &opened, nil)
c.approvals[31] = c.later(time.Minute) // review: 1m
merged := c.later(3 * time.Minute) // checks: 3m
c.delivery(t, 31, "published", "leaf", &opened, &merged)
if is := get(t, c.harness, n); is.FixTime != nil {
t.Fatalf("a fix merged and not delivered has a fix time: %+v", is.FixTime)
}
c.later(2 * time.Minute) // merge to running: 2m
c.delivery(t, 31, "delivered", "leaf", &opened, &merged)
v, err := c.View(n)
if err != nil {
t.Fatal(err)
}
ft := v.FixTime
if ft == nil {
t.Fatalf("no fix time once the fix is delivered: %+v", v.Issue)
}
if !ft.From.Equal(start) || ft.TookMS != (8*time.Minute).Milliseconds() {
t.Fatalf("fix time from %s took %s, want from %s (the first move to fixing) and 8m0s", ft.From, ft.Took, start)
}
got := phaseTook(t, ft)
if got[PhaseImplementing] != 2*time.Minute || got[PhaseReview] != time.Minute || got[PhaseChecks] != 3*time.Minute ||
got[PhaseRunning] != 2*time.Minute {
t.Fatalf("phases %v", got)
}
if ft.Class != ClassLeaf || !ft.Over || ft.Budget != "5m0s" || ft.Longest != "checks 3m0s" || ft.Worker != "other@shanks" {
t.Fatalf("class %q over %v budget %q longest %q worker %q", ft.Class, ft.Over, ft.Budget, ft.Longest, ft.Worker)
}
// Over its budget: said once, naming the issue, its worker and its longest phase.
if len(note.said) != 1 || !strings.HasPrefix(note.said[0], "issue-tracker.fix.leaf.over-budget warning issue "+itoa(int(n))) ||
!strings.Contains(note.said[0], "checks 3m0s") {
t.Fatalf("said %q", note.said)
}
// Heard again: nothing new is kept or said.
c.delivery(t, 31, "delivered", "leaf", &opened, &merged)
if len(note.said) != 1 {
t.Fatalf("said again: %q", note.said)
}
}
// A phase whose end is not known is unknown and counted in the next; one that ended before the phase before it
// counts zero (a pull request opened before the move to fixing).
func TestAPhaseNotKnownIsCountedInTheNextAndOneEndedEarlyCountsZero(t *testing.T) {
c := newFixClock(t)
n := c.open(t, "a fix whose pull request came first")
opened := c.at
c.later(time.Minute)
c.fixing(t, n)
if _, err := c.Move(MoveAsk{Number: n, Status: StatusFixing, By: "fix@g14", FixedBy: []string{"mesh-catalog#32"}}); err != nil {
t.Fatal(err)
}
merged := c.later(4 * time.Minute)
c.later(time.Minute)
c.delivery(t, 32, "delivered", "leaf", &opened, &merged)
ft := get(t, c.harness, n).FixTime
if ft == nil {
t.Fatal("no fix time")
}
got := phaseTook(t, ft)
if got[PhaseImplementing] != 0 || got[PhaseChecks] != 4*time.Minute || got[PhaseRunning] != time.Minute {
t.Fatalf("phases %v", got)
}
for _, p := range ft.Phases {
if p.Name == PhaseReview && (p.State != "unknown" || p.Said == "") {
t.Fatalf("review with no approval asked: %+v", p)
}
if p.Name == PhaseChecks && !strings.Contains(p.Said, "review") {
t.Fatalf("checks does not say it counts review: %+v", p)
}
}
if ft.Over {
t.Fatalf("5m of a leaf's 5m is within it: %+v", ft)
}
}
// The class is the slowest-budgeted of the fixes'; a fix that moves no module has no budget and is never warned
// on; one whose class is not said has none either.
func TestAnIssuesClassIsItsSlowestFixAndOneMovingNothingHasNoBudget(t *testing.T) {
c := newFixClock(t)
note := &notified{}
c.Notify = note.notify
n := c.open(t, "two fixes, one core")
c.fixing(t, n)
if _, err := c.Move(MoveAsk{Number: n, Status: StatusFixing, By: "fix@g14", FixedBy: []string{"mesh-catalog#41", "mesh-catalog#42"}}); err != nil {
t.Fatal(err)
}
merged := c.later(time.Minute)
c.delivery(t, 41, "delivered", "leaf", nil, &merged)
if get(t, c.harness, n).FixTime != nil {
t.Fatal("a fix time before the last fix is delivered")
}
c.later(7 * time.Minute)
c.delivery(t, 42, "delivered", "core", nil, &merged)
ft := get(t, c.harness, n).FixTime
if ft == nil || ft.Class != ClassCore || ft.Over || ft.Fix != "mesh-catalog#42" {
t.Fatalf("8m of a core fix: %+v", ft)
}
none := c.fixOne(t, "a fix in hq alone", 43, ClassNone, time.Minute, time.Minute, 10*time.Minute, time.Second)
ft = get(t, c.harness, none).FixTime
if ft == nil || ft.Class != ClassNone || ft.Over || ft.BudgetMS != 0 || !strings.Contains(ft.Said, "no fix budget") {
t.Fatalf("a fix moving nothing: %+v", ft)
}
unknown := c.fixOne(t, "a fix whose class is not said", 44, "", time.Minute, time.Minute, 10*time.Minute, time.Second)
ft = get(t, c.harness, unknown).FixTime
if ft == nil || ft.Class != "" || ft.Over || !strings.Contains(ft.Said, "no fix budget") {
t.Fatalf("a fix of no known class: %+v", ft)
}
if len(note.said) != 0 {
t.Fatalf("warned on a fix with no budget: %q", note.said)
}
}
// Nothing is said while a fix is undelivered, however long; once delivered over budget it is said; the next leaf
// fix delivered within budget clears it.
func TestAFixOverItsBudgetIsSaidOnceFinalAndClearedByTheNextWithin(t *testing.T) {
c := newFixClock(t)
c.Tracker.Approval = c.approval
note := &notified{}
c.Notify = note.notify
n := c.open(t, "a slow fix")
c.fixing(t, n)
if _, err := c.Move(MoveAsk{Number: n, Status: StatusFixing, By: "fix@g14", FixedBy: []string{"mesh-catalog#51"}}); err != nil {
t.Fatal(err)
}
merged := c.later(20 * time.Minute)
c.delivery(t, 51, "published", "leaf", nil, &merged)
if _, err := c.Sweep(); err != nil {
t.Fatal(err)
}
if len(note.said) != 0 {
t.Fatalf("said while undelivered: %q", note.said)
}
c.later(time.Minute)
c.delivery(t, 51, "delivered", "leaf", nil, &merged)
if len(note.said) != 1 || !strings.HasPrefix(note.said[0], fixOverKey(ClassLeaf)+" warning") {
t.Fatalf("said %q", note.said)
}
if !get(t, c.harness, n).FixTime.Warned {
t.Fatal("not marked said")
}
// A core fix within its budget clears nothing of the leaf's.
c.fixOne(t, "a quick core fix", 52, ClassCore, time.Minute, time.Minute, time.Minute, time.Minute)
if len(note.clear) != 0 {
t.Fatalf("cleared by another class: %q", note.clear)
}
c.fixOne(t, "a quick leaf fix", 53, ClassLeaf, time.Minute, 30*time.Second, time.Minute, time.Minute)
if len(note.clear) != 1 || note.clear[0] != fixOverKey(ClassLeaf) {
t.Fatalf("cleared %q", note.clear)
}
if get(t, c.harness, n).FixTime.Warned {
t.Fatal("still marked said after the clear")
}
if _, err := c.Sweep(); err != nil {
t.Fatal(err)
}
if len(note.said) != 1 || len(note.clear) != 1 {
t.Fatalf("the sweep said again: %q, cleared %q", note.said, note.clear)
}
}
// A warning the channel did not take is said at the next sweep.
func TestAWarningNotTakenIsSaidAtTheNextSweep(t *testing.T) {
c := newFixClock(t)
note := &notified{err: errFake("the channel is away")}
c.Notify = note.notify
n := c.fixOne(t, "slow", 61, ClassLeaf, 5*time.Minute, time.Minute, time.Minute, time.Minute)
if get(t, c.harness, n).FixTime == nil {
t.Fatal("a fix time is kept although the warning was not taken")
}
note.err = nil
if _, err := c.Sweep(); err != nil {
t.Fatal(err)
}
if len(note.said) != 1 || !get(t, c.harness, n).FixTime.Warned {
t.Fatalf("said %q", note.said)
}
}
// fix-times answers each class's budget, the window's median and worst, and every issue over its budget with its
// longest phase; an issue delivered before the window does not count, so a lever is judged by the medians before
// and after it.
func TestFixTimesAnswersTheWindowsMedianAndWorst(t *testing.T) {
c := newFixClock(t)
old := c.fixOne(t, "long ago", 70, ClassLeaf, 30*time.Minute, time.Minute, time.Minute, time.Minute)
c.later(8 * 24 * time.Hour)
a := c.fixOne(t, "four minutes", 71, ClassLeaf, time.Minute, time.Minute, time.Minute, time.Minute)
b := c.fixOne(t, "six minutes", 72, ClassLeaf, 3*time.Minute, time.Minute, time.Minute, time.Minute)
c.fixOne(t, "eight minutes", 73, ClassLeaf, 5*time.Minute, time.Minute, time.Minute, time.Minute)
c.fixOne(t, "a core fix", 74, ClassCore, time.Minute, time.Minute, time.Minute, time.Minute)
c.fixOne(t, "hq alone", 75, ClassNone, time.Minute, time.Minute, time.Minute, time.Minute)
out, err := c.call(t, "fix-times", map[string]any{})
if err != nil {
t.Fatal(err)
}
ft := out.(*FixTimes)
leaf, core := ft.Classes[0], ft.Classes[1]
if ft.Days != 7 || leaf.Class != ClassLeaf || leaf.Count != 3 || leaf.Median != "6m0s" || leaf.Worst != "8m0s" ||
leaf.Within != 1 || leaf.Over != 2 || leaf.Budget != "5m0s" {
t.Fatalf("leaf %+v (issue %d, delivered before the window, must not count)", leaf, old)
}
if core.Count != 1 || core.Median != "4m0s" || core.Within != 1 {
t.Fatalf("core %+v", core)
}
if len(ft.Over) != 2 || ft.Over[0].Title != "eight minutes" || ft.Over[0].Longest != "implementing 5m0s" || ft.Over[1].Number != b {
t.Fatalf("over %+v", ft.Over)
}
if len(ft.Unbudgeted) != 1 || ft.Unbudgeted[0].Title != "hq alone" {
t.Fatalf("unbudgeted %+v", ft.Unbudgeted)
}
if !strings.Contains(ft.Line, "leaf: median 6m0s, worst 8m0s") {
t.Fatalf("line %q", ft.Line)
}
// Wider, the old one counts and moves the median.
out, _ = c.call(t, "fix-times", map[string]any{"days": 30})
if l := out.(*FixTimes).Classes[0]; l.Count != 4 || l.Median != "7m0s" {
t.Fatalf("over 30 days: %+v", l)
}
if _, err := c.call(t, "fix-times", map[string]any{"days": 91}); err == nil {
t.Fatal("days 91 not refused")
}
_ = a
}
// An issue moved to fixing before the tracker kept the moment has its start read from its log.
func TestTheStartOfAnIssueMovedBeforeItWasKeptIsReadFromItsLog(t *testing.T) {
c := newFixClock(t)
n := c.open(t, "moved to fixing long ago")
c.later(time.Minute)
start := c.at
c.fixing(t, n)
c.later(time.Hour)
is := get(t, c.harness, n)
if is.FixingFrom == nil || !is.FixingFrom.Equal(start) {
t.Fatalf("fixing-from %v, want %s", is.FixingFrom, start)
}
is.FixingFrom = nil
if got := c.startOfFixing(is); !got.Equal(start) {
t.Fatalf("from the log: %s, want %s", got, start)
}
}
// The forge's reviews: the first approval at the head, never one dismissed, stale or of an older head.
func TestTheReviewEndsAtTheFirstApprovalAtItsHead(t *testing.T) {
raw := json.RawMessage(`{"reviews":[
{"state":"APPROVED","at_head":false,"submitted":"2026-10-12T09:01:00Z"},
{"state":"REQUEST_CHANGES","at_head":true,"submitted":"2026-10-12T09:02:00Z"},
{"state":"APPROVED","at_head":true,"dismissed":true,"submitted":"2026-10-12T09:03:00Z"},
{"state":"APPROVED","at_head":true,"submitted":"2026-10-12T09:05:00Z"},
{"state":"APPROVED","at_head":true,"submitted":"2026-10-12T09:04:00Z"}]}`)
at, found, err := firstApproval(raw)
if err != nil || !found || at.Format(time.RFC3339) != "2026-10-12T09:04:00Z" {
t.Fatalf("%s %v %v", at, found, err)
}
if _, found, err := firstApproval(json.RawMessage(`{"reviews":[]}`)); err != nil || found {
t.Fatalf("no reviews: %v %v", found, err)
}
if _, _, err := firstApproval(json.RawMessage(`"nothing"`)); err == nil {
t.Fatal("an answer with no reviews read as none")
}
}
type errFake string
func (e errFake) Error() string { return string(e) }
@@ -112,6 +112,12 @@ type Issue struct {
Triage *Verdict `json:"triage,omitempty"`
// Worker is who works on it, while it is diagnosing or fixing (ADR 0284); nil at every other status.
Worker *Worker `json:"worker,omitempty"`
// FixingFrom is when it was first moved to fixing: where its fix time starts (novox/hq ADR 0313 §1). A
// take-over or a later move back to fixing keeps it.
FixingFrom *time.Time `json:"fixing-from,omitempty"`
// FixTime is its fix time, phase by phase, once the last of its fixes is delivered (ADR 0313 §4): measured,
// never set by a verb.
FixTime *FixTime `json:"fix-time,omitempty"`
// Opening is the id of the call that opened it, so a write whose answer was lost is known as its own.
Opening string `json:"opening"`
// Imported says that the issue, or its status, was taken from its folder in hq by the import of
+33 -7
View File
@@ -22,7 +22,7 @@ import (
const Seat = "issue-tracker"
// Verbs are the seat's verbs, as the manifest declares and claims them.
var Verbs = []string{"open", "get", "list", "move", "note", "link", "import", "groom", "set", "tracking", "reserve"}
var Verbs = []string{"open", "get", "list", "move", "note", "link", "import", "groom", "set", "tracking", "reserve", "fix-times"}
func logf(format string, a ...any) { fmt.Fprintf(os.Stderr, format+"\n", a...) }
@@ -95,13 +95,16 @@ func main() {
CallerOf: stdio.CallerOf,
}
t.Notify = busNotifier()
t.Deliveries = func(repository, number string) (Delivery, error) {
owner := "novox"
forgeOwner := func() string {
if s, err := t.Settings(); err == nil && strings.TrimSpace(s.ForgeOwner) != "" {
owner = strings.TrimSpace(s.ForgeOwner)
return strings.TrimSpace(s.ForgeOwner)
}
return busDeliveryState(owner)(repository, number)
return "novox"
}
t.Deliveries = func(repository, number string) (Delivery, error) {
return busDeliveryState(forgeOwner())(repository, number)
}
t.Approval = busApproval(forgeOwner)
if len(os.Args) > 1 && os.Args[1] == "dry-run" {
os.Exit(dryRunAtTheTerminal(t, os.Args[2:]))
}
@@ -381,8 +384,10 @@ func Tools(t *Tracker) []stdio.Tool {
Description: "One issue whole, by its number: title, status, severity, what it needs, report, who opened it " +
"and when, the owners it is located in, who works on it and since when (worker, and worker-stale when " +
"nobody did anything on it for a day), what fixed it and whether that is delivered, its notes and its " +
"links; the issues it waits for, those that depend on it (needed-by), and whether it is ready; and " +
"whether its fix is delivered, as the delivery last said it (merged, held and why, delivered).",
"links; the issues it waits for, those that depend on it (needed-by), and whether it is ready; " +
"whether its fix is delivered, as the delivery last said it (merged, held and why, delivered); and, once " +
"the last of its fixes is delivered, its fix time (fix-time) from its first move to fixing, its class and " +
"fix budget, and its phases: implementing, review, checks, merge to running.",
Input: schema([]string{"number"}, map[string]any{"number": number("the issue's number")}),
Run: func(m map[string]any) (any, error) {
a, err := readArgs("get", m, "number")
@@ -710,5 +715,26 @@ func Tools(t *Tracker) []stdio.Tool {
}
return t.reserve(t.callerOf(m), ask)
}},
{Name: Seat + ".fix-times",
Description: "How long fixes take, from the move to fixing to the last fix delivered (the fix time): each class's fix budget (leaf 5m, core 10m), the median and worst fix time of the issues whose last " +
"fix was delivered in the last days (7 unless days says otherwise), how many were within and over, and every " +
"issue over its budget with its longest phase (implementing, review, checks, merge to running), newest first; " +
"issues whose fixes move no module, or whose class is not known, are listed apart with no budget. One issue's " +
"fix time, phase by phase, is in its get. A lever meant to shorten fixes is judged by comparing this median " +
"over the seven days before it was delivered with the seven days after. Refused: days above 90.",
Input: schema(nil, map[string]any{
"days": map[string]any{"type": "integer", "minimum": 1, "maximum": MostFixTimesDays, "description": "the window, in days back from now (default 7)"},
}),
Run: func(m map[string]any) (any, error) {
a, err := readArgs("fix-times", m, "days")
if err != nil {
return nil, err
}
days, err := a.number("days", false)
if err != nil {
return nil, err
}
return t.FixTimes(int(days))
}},
}
}
@@ -106,7 +106,7 @@ func TestItDeclaresAndHoldsTheIssueTrackerSeatOnceForTheMesh(t *testing.T) {
// the import and a reservation read the record through the records module (issue 424).
if !reflect.DeepEqual(m.Consumes, []string{"mesh-issues." + EventOpened, "mesh-issues." + EventMoved, "mesh-delivery.transition"}) ||
!reflect.DeepEqual(m.Invokes, []string{"claude-code.claude_code_judge", "seat:mesh-delivery.checks", "seat:operator-channel.notify",
"records.records_status", "records.records_list", "records.records_read", "mesh-knowledge.knowledge_highest"}) {
"gitea.gitea_pull_reviews", "records.records_status", "records.records_list", "records.records_read", "mesh-knowledge.knowledge_highest"}) {
t.Fatalf("consumes %v, invokes %v", m.Consumes, m.Invokes)
}
if len(m.Build.Artifacts) != 1 || m.Build.Artifacts[0].From != "cmd/mesh-issues" ||
+10 -1
View File
@@ -36,8 +36,12 @@ type Tracker struct {
Judge Judgement
// Deliveries answers a pull request's delivery (ADR 0279 §3, ADR 0281); nil asks nothing.
Deliveries DeliveryState
// Notify tells the operator's channel of a stuck fix (ADR 0281); nil says nothing there.
// Notify tells the operator's channel of a stuck fix (ADR 0281) and of a fix over its budget (ADR 0313); nil
// says nothing there.
Notify Notifier
// Approval answers when a pull request was first approved at its head (ADR 0313 §4); nil asks nothing, and a
// fix's review is counted with its checks.
Approval Approval
// CallerOf is who called, for the arguments one call was handed (stdio.CallerOf); nil proves nobody.
CallerOf func(map[string]any) string
// Log is the tracker's record, the log `changes` (novox/hq ADR 0300); the bucket `issues` is its fold.
@@ -456,6 +460,11 @@ func (t *Tracker) move(by string, a MoveAsk) (*Moved, error) {
return err
}
is.Status = to
if to == StatusFixing && from != StatusFixing && is.FixingFrom == nil {
// The first move to fixing starts the fix time; none after it restarts it (ADR 0313 §1).
start := at
is.FixingFrom = &start
}
if (to == StatusResolved || to == StatusWontfix) && from != to {
is.Closed = &at
} else if to != StatusResolved && to != StatusWontfix {
+5 -2
View File
@@ -17,7 +17,8 @@
"groom",
"set",
"tracking",
"reserve"
"reserve",
"fix-times"
]
}
],
@@ -36,7 +37,8 @@
"groom",
"set",
"tracking",
"reserve"
"reserve",
"fix-times"
]
}
],
@@ -61,6 +63,7 @@
"claude-code.claude_code_judge",
"seat:mesh-delivery.checks",
"seat:operator-channel.notify",
"gitea.gitea_pull_reviews",
"records.records_status",
"records.records_list",
"records.records_read",