From 41f18b24527fbeb069cd0368652e32f23aa91303 Mon Sep 17 00:00:00 2001 From: mesh-tools Date: Tue, 20 Oct 2026 06:41:48 +0000 Subject: [PATCH] Keep today's module mesh-delivery in the catalogue, unchanged: its removal follows the handover as a pull request of its own --- modules/mesh-delivery/README.md | 19 + .../cmd/mesh-delivery/adr350_test.go | 50 + .../cmd/mesh-delivery/answers.go | 672 ++++ .../cmd/mesh-delivery/approval.go | 557 ++++ .../cmd/mesh-delivery/approval_test.go | 354 +++ .../cmd/mesh-delivery/backup_test.go | 154 + .../cmd/mesh-delivery/bus_test.go | 677 ++++ .../cmd/mesh-delivery/checkask_test.go | 163 + .../mesh-delivery/cmd/mesh-delivery/checks.go | 297 ++ .../cmd/mesh-delivery/checks_test.go | 242 ++ .../cmd/mesh-delivery/covered_test.go | 681 +++++ .../cmd/mesh-delivery/effects.go | 453 +++ .../mesh-delivery/cmd/mesh-delivery/events.go | 110 + .../cmd/mesh-delivery/events_test.go | 266 ++ .../cmd/mesh-delivery/fakes_test.go | 733 +++++ .../cmd/mesh-delivery/group_merge.go | 229 ++ .../cmd/mesh-delivery/group_test.go | 443 +++ .../cmd/mesh-delivery/history_test.go | 233 ++ .../mesh-delivery/cmd/mesh-delivery/holder.go | 2717 +++++++++++++++++ .../cmd/mesh-delivery/holder_test.go | 631 ++++ .../cmd/mesh-delivery/issue1079_test.go | 101 + .../cmd/mesh-delivery/issue1083_test.go | 89 + .../cmd/mesh-delivery/issue1107_test.go | 61 + .../cmd/mesh-delivery/issue1114_test.go | 90 + .../cmd/mesh-delivery/issue1275_test.go | 413 +++ .../cmd/mesh-delivery/issue326_test.go | 89 + .../cmd/mesh-delivery/issue341_test.go | 412 +++ .../cmd/mesh-delivery/issue362_test.go | 434 +++ .../cmd/mesh-delivery/issue403_test.go | 95 + .../cmd/mesh-delivery/issue407_test.go | 105 + .../cmd/mesh-delivery/issue438_test.go | 183 ++ .../cmd/mesh-delivery/issue441_test.go | 251 ++ .../cmd/mesh-delivery/issue442_test.go | 200 ++ .../cmd/mesh-delivery/issue443_test.go | 66 + .../cmd/mesh-delivery/issue451_test.go | 48 + .../cmd/mesh-delivery/issue459_test.go | 338 ++ .../cmd/mesh-delivery/issue474_test.go | 211 ++ .../cmd/mesh-delivery/issue480_test.go | 134 + .../cmd/mesh-delivery/issue529_test.go | 142 + .../cmd/mesh-delivery/issue611_test.go | 150 + .../cmd/mesh-delivery/issue747_test.go | 258 ++ .../cmd/mesh-delivery/issue753_test.go | 272 ++ .../cmd/mesh-delivery/issue783_test.go | 151 + .../cmd/mesh-delivery/issue902_test.go | 66 + .../mesh-delivery/cmd/mesh-delivery/main.go | 422 +++ .../cmd/mesh-delivery/manifest_test.go | 177 ++ .../mesh-delivery/cmd/mesh-delivery/model.go | 1030 +++++++ .../cmd/mesh-delivery/operator_test.go | 104 + .../mesh-delivery/cmd/mesh-delivery/ports.go | 487 +++ .../cmd/mesh-delivery/pullkey.go | 115 + .../cmd/mesh-delivery/pullkey_test.go | 174 ++ .../mesh-delivery/cmd/mesh-delivery/react.go | 119 + .../mesh-delivery/cmd/mesh-delivery/ready.go | 528 ++++ .../cmd/mesh-delivery/ready_test.go | 239 ++ .../cmd/mesh-delivery/recheck_test.go | 175 ++ .../cmd/mesh-delivery/replay362_test.go | 127 + .../mesh-delivery/cmd/mesh-delivery/table.go | 735 +++++ .../cmd/mesh-delivery/table_test.go | 260 ++ .../cmd/mesh-delivery/timers_source_test.go | 85 + .../cmd/mesh-delivery/timers_test.go | 212 ++ .../mesh-delivery/cmd/mesh-delivery/times.go | 373 +++ .../cmd/mesh-delivery/times_test.go | 261 ++ .../cmd/mesh-delivery/unchecked.go | 305 ++ .../mesh-delivery/unchecked_events_test.go | 95 + .../cmd/mesh-delivery/unchecked_test.go | 80 + .../mesh-delivery/cmd/mesh-delivery/verbs.go | 851 ++++++ .../cmd/mesh-delivery/waiting_checks.go | 67 + modules/mesh-delivery/go.mod | 22 + modules/mesh-delivery/go.sum | 27 + modules/mesh-delivery/module.json | 95 + 70 files changed, 21205 insertions(+) create mode 100644 modules/mesh-delivery/README.md create mode 100644 modules/mesh-delivery/cmd/mesh-delivery/adr350_test.go create mode 100644 modules/mesh-delivery/cmd/mesh-delivery/answers.go create mode 100644 modules/mesh-delivery/cmd/mesh-delivery/approval.go create mode 100644 modules/mesh-delivery/cmd/mesh-delivery/approval_test.go create mode 100644 modules/mesh-delivery/cmd/mesh-delivery/backup_test.go create mode 100644 modules/mesh-delivery/cmd/mesh-delivery/bus_test.go create mode 100644 modules/mesh-delivery/cmd/mesh-delivery/checkask_test.go create mode 100644 modules/mesh-delivery/cmd/mesh-delivery/checks.go create mode 100644 modules/mesh-delivery/cmd/mesh-delivery/checks_test.go create mode 100644 modules/mesh-delivery/cmd/mesh-delivery/covered_test.go create mode 100644 modules/mesh-delivery/cmd/mesh-delivery/effects.go create mode 100644 modules/mesh-delivery/cmd/mesh-delivery/events.go create mode 100644 modules/mesh-delivery/cmd/mesh-delivery/events_test.go create mode 100644 modules/mesh-delivery/cmd/mesh-delivery/fakes_test.go create mode 100644 modules/mesh-delivery/cmd/mesh-delivery/group_merge.go create mode 100644 modules/mesh-delivery/cmd/mesh-delivery/group_test.go create mode 100644 modules/mesh-delivery/cmd/mesh-delivery/history_test.go create mode 100644 modules/mesh-delivery/cmd/mesh-delivery/holder.go create mode 100644 modules/mesh-delivery/cmd/mesh-delivery/holder_test.go create mode 100644 modules/mesh-delivery/cmd/mesh-delivery/issue1079_test.go create mode 100644 modules/mesh-delivery/cmd/mesh-delivery/issue1083_test.go create mode 100644 modules/mesh-delivery/cmd/mesh-delivery/issue1107_test.go create mode 100644 modules/mesh-delivery/cmd/mesh-delivery/issue1114_test.go create mode 100644 modules/mesh-delivery/cmd/mesh-delivery/issue1275_test.go create mode 100644 modules/mesh-delivery/cmd/mesh-delivery/issue326_test.go create mode 100644 modules/mesh-delivery/cmd/mesh-delivery/issue341_test.go create mode 100644 modules/mesh-delivery/cmd/mesh-delivery/issue362_test.go create mode 100644 modules/mesh-delivery/cmd/mesh-delivery/issue403_test.go create mode 100644 modules/mesh-delivery/cmd/mesh-delivery/issue407_test.go create mode 100644 modules/mesh-delivery/cmd/mesh-delivery/issue438_test.go create mode 100644 modules/mesh-delivery/cmd/mesh-delivery/issue441_test.go create mode 100644 modules/mesh-delivery/cmd/mesh-delivery/issue442_test.go create mode 100644 modules/mesh-delivery/cmd/mesh-delivery/issue443_test.go create mode 100644 modules/mesh-delivery/cmd/mesh-delivery/issue451_test.go create mode 100644 modules/mesh-delivery/cmd/mesh-delivery/issue459_test.go create mode 100644 modules/mesh-delivery/cmd/mesh-delivery/issue474_test.go create mode 100644 modules/mesh-delivery/cmd/mesh-delivery/issue480_test.go create mode 100644 modules/mesh-delivery/cmd/mesh-delivery/issue529_test.go create mode 100644 modules/mesh-delivery/cmd/mesh-delivery/issue611_test.go create mode 100644 modules/mesh-delivery/cmd/mesh-delivery/issue747_test.go create mode 100644 modules/mesh-delivery/cmd/mesh-delivery/issue753_test.go create mode 100644 modules/mesh-delivery/cmd/mesh-delivery/issue783_test.go create mode 100644 modules/mesh-delivery/cmd/mesh-delivery/issue902_test.go create mode 100644 modules/mesh-delivery/cmd/mesh-delivery/main.go create mode 100644 modules/mesh-delivery/cmd/mesh-delivery/manifest_test.go create mode 100644 modules/mesh-delivery/cmd/mesh-delivery/model.go create mode 100644 modules/mesh-delivery/cmd/mesh-delivery/operator_test.go create mode 100644 modules/mesh-delivery/cmd/mesh-delivery/ports.go create mode 100644 modules/mesh-delivery/cmd/mesh-delivery/pullkey.go create mode 100644 modules/mesh-delivery/cmd/mesh-delivery/pullkey_test.go create mode 100644 modules/mesh-delivery/cmd/mesh-delivery/react.go create mode 100644 modules/mesh-delivery/cmd/mesh-delivery/ready.go create mode 100644 modules/mesh-delivery/cmd/mesh-delivery/ready_test.go create mode 100644 modules/mesh-delivery/cmd/mesh-delivery/recheck_test.go create mode 100644 modules/mesh-delivery/cmd/mesh-delivery/replay362_test.go create mode 100644 modules/mesh-delivery/cmd/mesh-delivery/table.go create mode 100644 modules/mesh-delivery/cmd/mesh-delivery/table_test.go create mode 100644 modules/mesh-delivery/cmd/mesh-delivery/timers_source_test.go create mode 100644 modules/mesh-delivery/cmd/mesh-delivery/timers_test.go create mode 100644 modules/mesh-delivery/cmd/mesh-delivery/times.go create mode 100644 modules/mesh-delivery/cmd/mesh-delivery/times_test.go create mode 100644 modules/mesh-delivery/cmd/mesh-delivery/unchecked.go create mode 100644 modules/mesh-delivery/cmd/mesh-delivery/unchecked_events_test.go create mode 100644 modules/mesh-delivery/cmd/mesh-delivery/unchecked_test.go create mode 100644 modules/mesh-delivery/cmd/mesh-delivery/verbs.go create mode 100644 modules/mesh-delivery/cmd/mesh-delivery/waiting_checks.go create mode 100644 modules/mesh-delivery/go.mod create mode 100644 modules/mesh-delivery/go.sum create mode 100644 modules/mesh-delivery/module.json diff --git a/modules/mesh-delivery/README.md b/modules/mesh-delivery/README.md new file mode 100644 index 00000000..c2b64c5b --- /dev/null +++ b/modules/mesh-delivery/README.md @@ -0,0 +1,19 @@ +# mesh-delivery + +The `mesh-delivery` module of the mesh catalogue: claims the seat `mesh-delivery`. + +## Files it does not know + +Each directory or archive this module declares says how it treats a file it did not write (ADR 0422, ADR 0416). + +- `${dir:state}`: everything in it is the program's; the mesh moves nothing out (ADR 0422 §2). + +## A backup that failed before the send + +The controller backs up the data of the modules a delivery moves before it asks or sends anything (ADR 0404). When its +walk's record, in `plan-moved`, names a backup that failed or stayed silent and nothing of the plan was sent, the +delivery goes from delivering to held (the observed row `backup failed`); the held reason names the machine, the +module and the backup's words, and the pull request's comment shows it. The hold does not lift by itself. `release` +of such a delivery asks the controller (`delivery-backups-again`) to take the backups again and the delivery goes on +delivering; a record from before the controller cleared the failure does not hold it again. A held delivery already +ends as covered or delivered when a later merge covers it. diff --git a/modules/mesh-delivery/cmd/mesh-delivery/adr350_test.go b/modules/mesh-delivery/cmd/mesh-delivery/adr350_test.go new file mode 100644 index 00000000..05075466 --- /dev/null +++ b/modules/mesh-delivery/cmd/mesh-delivery/adr350_test.go @@ -0,0 +1,50 @@ +package main + +import ( + "reflect" + "strings" + "testing" + "time" +) + +// One build per change (novox/hq ADR 0350): a pull request closed unmerged has its check builds let go of, and a +// walk's record says per module whether its build reused a check's build or built, and why. + +// A pull request closed unmerged asks the controller to let go of what its checks built for its merge (§5); one +// the controller cannot answer is said, never fatal. +func TestAPullRequestClosedUnmergedLetsGoOfItsCheckBuilds(t *testing.T) { + w := newWorldOver(t, newMemStore(), newFakeController(), newFakeForge(), atCC) + cc := pr(cat, 101, headCC, "feat/claude-code", "modules/claude-code/module.json") + w.h.PullUpdated(cc) + w.h.PullClosed(cc) + if want := []string{"novox/mesh-catalog#101"}; !reflect.DeepEqual(w.ctl.forgot, want) { + t.Fatalf("closed unmerged, the controller was asked to let go of %v", w.ctl.forgot) + } + if s := w.state(IDOf(cat, headCC)); s != Stopped { + t.Fatalf("closed unmerged it is %s", s) + } + + w.ctl.down = true + du := pr(cat, 102, headDu, "feat/dunst", "modules/dunst/module.json") + w.h.PullUpdated(du) + w.h.PullClosed(du) + if s := w.state(IDOf(cat, headDu)); s != Stopped { + t.Fatalf("with the controller down, a closed delivery is %s", s) + } +} + +// The walk's record says per module what its build did, and the delivery keeps it beside its walk (§4). +func TestAWalksRecordSaysWhetherEachModuleWasReusedOrBuilt(t *testing.T) { + walk := catWalk("plan-1", mergeCC, atCC.Add(time.Minute), "claude-code", "dunst") + walk.Modules["claude-code"].Made = "reused from the check of novox/mesh-catalog#101 at head cccccccccccc " + + "(merged into trunk 553b71910000): tools sha256:aa" + walk.Modules["dunst"].Made = "built: no check of novox/mesh-catalog#101 built it" + seen := walk.Seen() + if !strings.HasPrefix(seen.Built["claude-code"], "reused from the check of novox/mesh-catalog#101") || + seen.Built["dunst"] != "built: no check of novox/mesh-catalog#101 built it" { + t.Fatalf("the delivery keeps %v", seen.Built) + } + if catWalk("plan-2", mergeCC, atCC, "dunst").Seen().Built != nil { + t.Fatal("a walk from a controller that says nothing of its builds reads as if it said something") + } +} diff --git a/modules/mesh-delivery/cmd/mesh-delivery/answers.go b/modules/mesh-delivery/cmd/mesh-delivery/answers.go new file mode 100644 index 00000000..9fb1a14d --- /dev/null +++ b/modules/mesh-delivery/cmd/mesh-delivery/answers.go @@ -0,0 +1,672 @@ +package main + +import ( + "fmt" + "sort" + "strings" + "time" +) + +// Which merges a walk answers (novox/hq issue 362, ADR 0276). +// +// On 2026-10-10 two merges of the catalogue 18 seconds apart were walked by one walk at the later merge's +// commit, which superseded the first. This owner found the walk a delivery follows by the walk's id or by the +// commit it was merged as, final or not: the surviving walk matched the dunst delivery, which the supersession +// had already ended, so nothing gave it its word, and the claude-code delivery, merged as another commit, +// matched no walk and was held for a person after ten minutes, for a reason that named neither. +// +// A walk answers every merge it carries: the merges it names (`delivery.merges`, said by a controller that +// batches merges), the merge of its own commit, a merge whose walk it took over, and a merge of its repository +// and branch the forge says its commit contains. A delivery follows the newest walk that answers it, and only a +// delivery not yet ended is ever tied to a walk by what it answers: an ended delivery is moved only by the +// record of its own walk, which no longer moves it. + +// seenWalk is a walk's last record and when it was heard. +type seenWalk struct { + walk Walk + at time.Time +} + +// keptWalkFor is how long a walk no longer open is kept for a merge heard after it ended; a batch is kept as +// long, past its window's latest close. +const keptWalkFor = 10 * time.Minute + +// answer is a delivery a walk answers and the commit of its repository the walk carries for it. +type answer struct { + d *Delivery + carried string +} + +// walkAnswers is whether a walk answers a delivery's merge, and the commit it carries for it. +func walkAnswers(w Walk, d *Delivery) (string, bool) { + if w.Release != nil || w.IsBatch() || d.MergedAs == "" { + return "", false + } + if c, ok := w.carriedFor(d.Repository, d.MergedAs); ok { + return c, true + } + if d.Walk != nil && d.Walk.TakenOverBy == w.ID && namesNoMerges(w) && strings.EqualFold(w.Repository, d.Repository) && + sameBranch(w.Branch, d.Base) { + // The walk that took its walk over, from a controller that names no merges: it answers every merge the one + // it took over answered, by its commit of the same repository and branch. A walk that names its merges + // answers only those (carriedFor, above). + return w.Commit, true + } + return "", false +} + +// containedAnswer is, for a controller that does not name the merges a walk answers, whether an open walk +// answers a ready merged delivery by containment: of its repository and branch, made for a merge after it, the +// forge says its commit contains this merge, and it moves every module this merge's plan moves. A later walk that +// moves only what its own merge moved does not deliver this one, however much of the trunk it contains. +func (h *Holder) containedAnswer(w Walk, d *Delivery) (string, bool) { + if !h.mayContain(w, d) || !d.ContainedIn[w.Commit] { + return "", false + } + return w.Commit, true +} + +// mayContain is whether an open walk is one to ask the forge about for a ready merged delivery (containedAnswer). +func (h *Holder) mayContain(w Walk, d *Delivery) bool { + if w.Release != nil || w.IsBatch() || !w.Open() || !namesNoMerges(w) || d.State != Ready || d.MergedAs == "" || d.Walk != nil || + d.Plan == nil || len(d.Plan.Moved) == 0 || !strings.EqualFold(w.Repository, d.Repository) || + !sameBranch(w.Branch, d.Base) || sameCommit(w.Commit, d.MergedAs) { + return false + } + for _, m := range d.Plan.Moved { + if _, moves := w.Modules[m]; !moves { + return false + } + } + // Made for a merge after this one: the walk's own merge, where this owner holds it, or else the walk itself. + after := w.Created + for _, o := range h.deliveries { + if strings.EqualFold(o.Repository, w.Repository) && sameCommit(o.MergedAs, w.Commit) && !o.MergedAt.IsZero() { + after = o.MergedAt + } + } + return after.After(d.MergedAt) +} + +// namesNoMerges is whether a walk is from a controller that does not name the merges it answers. +func namesNoMerges(w Walk) bool { return w.Delivery == nil || len(w.Delivery.Merges) == 0 } + +func sameBranch(a, b string) bool { return a == "" || b == "" || strings.EqualFold(a, b) } + +// follows is whether a delivery takes a walk that answers it as the one it follows: it has none, it is that walk, +// that walk took its own over, or that walk is newer than the one it follows. A walk record kept before its +// creation was read is followed past only once it is superseded or failed. +func follows(d *Delivery, w Walk) bool { + switch { + case d.Walk == nil || d.Walk.ID == w.ID || d.Walk.TakenOverBy == w.ID: + return true + case d.Walk.Created.IsZero(): + return d.Walk.State == walkSuperseded || d.Walk.State == walkFailed + } + return w.Created.After(d.Walk.Created) +} + +// deliveriesWalking are the deliveries a walk record moves: every delivery that already follows it — ended or +// not, its own record — and every delivery not ended that it answers and that follows it now. In id order. +func (h *Holder) deliveriesWalking(w Walk) []answer { + var out []answer + for _, d := range h.deliveries { + if d.Walk != nil && d.Walk.ID == w.ID { + carried := d.AnsweredBy + if c, ok := walkAnswers(w, d); ok { + carried = c + } + out = append(out, answer{d: d, carried: carried}) + continue + } + if d.State.Final() || !follows(d, w) { + continue + } + if c, ok := walkAnswers(w, d); ok { + out = append(out, answer{d: d, carried: c}) + } + } + sort.Slice(out, func(i, j int) bool { return out[i].d.ID < out[j].d.ID }) + return out +} + +// accounted is whether any delivery, ended or not, follows the walk or is answered by it: a walk known for +// what it answers is never made a delivery of its own. +func (h *Holder) accounted(w Walk) bool { + for _, d := range h.deliveries { + if d.Walk != nil && d.Walk.ID == w.ID { + return true + } + if _, ok := walkAnswers(w, d); ok { + return true + } + } + return false +} + +// held is whether a live delivery follows the walk, or a live delivery it answers follows a newer one. A walk whose +// word only stops that lapsed give (lapsedOn, novox/hq issue 753) is not held: it stays under the orphans' timer, so +// past the grace without its word it is listed by `stalled`, saying why it was not given. +func (h *Holder) heldWalk(w Walk) bool { + for _, d := range h.deliveries { + if d.State.Final() { + continue + } + if d.Walk != nil && d.Walk.ID == w.ID { + return true + } + if _, ok := walkAnswers(w, d); ok { + return true + } + } + return false +} + +// pickWalk is the newest walk heard that answers a merged delivery with no walk yet; nil for none. +func (h *Holder) pickWalk(d *Delivery) (Walk, string, bool) { + var best Walk + var carried string + found := false + for _, s := range h.walks { + if !follows(d, s.walk) { + continue + } + c, ok := walkAnswers(s.walk, d) + if !ok { + c, ok = h.containedAnswer(s.walk, d) + } + if !ok { + continue + } + if !found || s.walk.Created.After(best.Created) || s.walk.Created.Equal(best.Created) && s.walk.ID > best.ID { + best, carried, found = s.walk, c, true + } + } + return best, carried, found +} + +// linkAnswering ties a merged delivery with no walk to the newest walk heard that answers it; under the lock. +func (h *Holder) linkAnswering(d *Delivery) bool { + if d.State.Final() || d.MergedAs == "" || d.Walk != nil { + return false + } + w, carried, ok := h.pickWalk(d) + if !ok { + return false + } + h.linkWalk(d, w, carried) + delete(h.unmatched, w.ID) + return true +} + +// containsAsks are the forge's asks whether an open walk of a merged delivery's repository and branch, at +// another commit, contains its merge: for a controller that does not name the merges a walk answers (issue 362), +// asked once per pair, the answer kept on the delivery. +func (h *Holder) containsAsks(d *Delivery, now time.Time) []func() { + var asks []func() + for _, s := range h.walks { + w := s.walk + if !h.mayContain(w, d) { + continue + } + if _, known := d.ContainedIn[w.Commit]; known { + continue + } + if a := h.askContains(d, w.Commit, now); a != nil { + asks = append(asks, a) + } + } + return asks +} + +// batchOf is the batch a merged delivery's merge is in, when the controller said one; false for none. +func (h *Holder) batchOf(d *Delivery) (Walk, bool) { + if d.MergedAs == "" { + return Walk{}, false + } + for _, s := range h.walks { + if !s.walk.IsBatch() || s.walk.Delivery == nil { + continue + } + for _, m := range s.walk.Delivery.Merges { + if strings.EqualFold(m.Repository, d.Repository) && sameCommit(m.Commit, d.MergedAs) { + return s.walk, true + } + } + } + return Walk{}, false +} + +// batchSaid is what a batch holding a delivery's merge waits for, in words. +func batchSaid(b Walk, now time.Time) string { + said := "its batch " + b.ID + ": " + b.State + bt := b.Delivery.Batch + switch { + case b.State == walkQueued && bt != nil && bt.Behind != "": + said += " behind " + bt.Behind + case b.State == walkAssembling && bt != nil && bt.ClosesAt != nil: + left := bt.ClosesAt.Sub(now).Round(time.Second) + if left < 0 { + left = 0 + } + said += fmt.Sprintf(", %s left", left) + if bt.AtMost != nil { + said += " (at the latest " + bt.AtMost.Local().Format("15:04:05") + ")" + } + } + return said + "; plan not yet calculated" +} + +// unwalkedWhy is why a merged delivery no walk answers is held: the open walks of its repository and that none +// answers its merge, or that the controller holds none — what happened, not what might have. +func (h *Holder) unwalkedWhy(d *Delivery) string { + var open []string + for _, s := range h.walks { + w := s.walk + if w.Release == nil && !w.IsBatch() && w.Open() && strings.EqualFold(w.Repository, d.Repository) { + open = append(open, fmt.Sprintf("%s at %s", w.ID, shortOf(w.Commit, 8))) + } + } + sort.Strings(open) + head := fmt.Sprintf("merged as %s, and no walk answers its merge within %s: ", shortOf(d.MergedAs, 8), unwalked) + if len(open) == 0 { + return head + "the controller holds no open walk of " + d.Repository + " — its merge was not heard, or " + + "nothing the mesh holds follows " + orNone(d.Base) + ": a person decides" + } + return head + "the open walk(s) of " + d.Repository + " — " + strings.Join(open, ", ") + " — name no merge of " + + "it and the forge did not say they contain it: a person decides" +} + +func orNone(s string) string { + if s == "" { + return "its branch" + } + return s +} + +// orphanLines are the waiting walks no live delivery holds, past the grace a merge is given to be heard, as +// stalled lines in the state Unheld: nothing would ever give such a walk its word (issue 362). Under the lock. +func (h *Holder) orphanLines(now time.Time) []StalledLine { + var out []StalledLine + for id, first := range h.orphans { + s, known := h.walks[id] + if !known || now.Sub(first) < matchGrace { + continue + } + w := s.walk + if len(h.lapsedOn(w)) > 0 { + // Owed its word by stops that lapsed, and not given it past the grace (review of issue 753): said, why. + says := fmt.Sprintf("the walk %s of %s at %s waits for this owner's word, which the stopped deliveries whose "+ + "merge it answers owe it (novox/hq issue 753), and it has not started", w.ID, w.Repository, shortOf(w.Commit, 8)) + if why := h.lapsedSaid(w); why != "" { + says += ": " + why + } + out = append(out, StalledLine{ID: IDOf(w.Repository, w.Commit), State: Published, Walk: w.ID, + For: now.Sub(first).Round(time.Second).String(), Bound: matchGrace.String(), + H2: "none: its word is given at its turn — the operator's past the bound", Says: says}) + continue + } + out = append(out, StalledLine{ID: IDOf(w.Repository, w.Commit), State: Unheld, Walk: w.ID, + For: now.Sub(first).Round(time.Second).String(), Bound: matchGrace.String(), + H2: "none: no delivery holds the walk, so nothing gives it its word — the operator's", + Says: fmt.Sprintf("the walk %s of %s at %s waits for this owner's word, and no delivery that has not ended "+ + "follows it or is answered by it: `mesh-delivery.show` for the deliveries of %s, and `plans go %s --why …` "+ + "starts it by hand", w.ID, w.Repository, shortOf(w.Commit, 8), w.Repository, w.ID)}) + out[len(out)-1].Says += h.endedSaid(w) + if hold := h.coverHeld(id); hold != "" { + out[len(out)-1].Says += "; it is held back meanwhile: " + hold + } + } + return out +} + +// Unheld is a waiting walk no live delivery holds: not a state of the table — only what a stalled line says. +const Unheld State = "unheld" + +// seeOrphans keeps when each waiting walk no live delivery holds was first seen so, and forgets the rest; a +// walk still unmatched waits for its merge or its adoption instead. Under the lock. +func (h *Holder) seeOrphans(now time.Time) { + for id, s := range h.walks { + w := s.walk + _, unmatched := h.unmatched[id] + if unmatched || w.Release != nil || w.IsBatch() || !w.Open() || !w.Seen().Waits || h.heldWalk(w) { + delete(h.orphans, id) + continue + } + if _, was := h.orphans[id]; !was { + h.orphans[id] = now + h.Logf("[mesh-delivery] the walk %s waits for this owner's word and no live delivery holds it: said once "+ + "it is %s so", id, matchGrace) + } + } + for id := range h.orphans { + if _, known := h.walks[id]; !known { + delete(h.orphans, id) + } + } +} + +// forgetWalks drops a walk record no longer open after keptWalkFor, and a batch heard of that long past its +// window's latest close. Under the lock. +func (h *Holder) forgetWalks(now time.Time) { + for id, s := range h.walks { + w := s.walk + switch { + case w.IsBatch(): + end := s.at + if w.Delivery != nil && w.Delivery.Batch != nil && w.Delivery.Batch.AtMost != nil && w.Delivery.Batch.AtMost.After(end) { + end = *w.Delivery.Batch.AtMost + } + if now.Sub(end) > 3*time.Hour { + delete(h.walks, id) + } + case !w.Open() && now.Sub(s.at) > keptWalkFor: + delete(h.walks, id) + delete(h.deliverErr, id) + } + } +} + +// --- the word, once per walk ------------------------------------------------------------------------------ + +// turnWant is a walk whose turn came for some of the deliveries it carries, and why. +type turnWant struct { + ids []string + whys []string +} + +// endedSaid names the ended deliveries a waiting walk answers or that follow it, with their states, so a walk no live +// delivery holds says what it carries instead of only that nothing holds it (novox/hq issue 753); empty for none. +func (h *Holder) endedSaid(w Walk) string { + var ended []string + for _, d := range h.deliveries { + if !d.State.Final() { + continue + } + _, answers := walkAnswers(w, d) + if answers || d.Walk != nil && d.Walk.ID == w.ID { + ended = append(ended, fmt.Sprintf("%s (%s)", d.ID, d.State)) + } + } + if len(ended) == 0 { + return "" + } + sort.Strings(ended) + return "; every delivery it carries ended: " + strings.Join(ended, ", ") +} + +// lapsedOn are the deliveries, by id, whose word an open walk waiting for this owner's word takes from a stop that +// lapsed (novox/hq issue 753): each was stopped by its group for a member delivered since, so its own change waits for +// a later walk of its repository that contains its merge (issue 747), and this walk answers that merge — it names it, +// or is made at it — and has not left it out. Such a walk is the one that delivery waits for: it gives it its word, so +// a walk whose every other delivery ended is never left without one. A walk at the merge of a delivery stopped for +// itself, or by a person, gets nothing from it. Under the lock. +// +// No member is ordered before another (novox/hq ADR 0418 §2): each such delivery gives its word at once. +func (h *Holder) lapsedOn(w Walk) []string { + if w.Release != nil || w.IsBatch() || !w.Open() || !w.Seen().Waits { + return nil + } + var out []string + for _, d := range h.deliveries { + if h.lapsedAnswered(w, d) { + out = append(out, d.ID) + } + } + sort.Strings(out) + return out +} + +// lapsedAnswered is whether a delivery's stop lapsed and the walk answers its merge, not left out. +func (h *Holder) lapsedAnswered(w Walk, d *Delivery) bool { + if d.MergedAs == "" || !h.stopLapsed(d) { + return false + } + if _, ok := walkAnswers(w, d); !ok { + return false + } + _, left := w.leftOut(d.Repository, d.MergedAs) + return !left +} + +// lapsedSaid is why a walk whose word stops that lapsed give still waits: the members of their groups not yet +// delivered before them, what the last look held it for, or the controller's refusal of the word. Under the lock. +func (h *Holder) lapsedSaid(w Walk) string { + var parts []string + if why := h.waitsWhy[w.ID]; why != "" { + parts = append(parts, "not let go yet: "+why) + } + if err := h.deliverErr[w.ID]; err != "" { + parts = append(parts, "the controller refused its word: "+err) + } + for _, id := range h.lapsedOn(w) { + if d := h.deliveries[id]; d != nil && strings.Contains(d.LetGoWhy, lapsedWord(w.ID)) { + parts = append(parts, fmt.Sprintf("its word was given for %s at %s, and the controller has not started it", + id, d.LetGo.Local().Format("15:04:05"))) + } + } + return strings.Join(parts, "; ") +} + +// lapsedOwed is whether a walk's word is owed by stops that lapsed. +func (h *Holder) lapsedOwed(w Walk) bool { + return len(h.lapsedOn(w)) > 0 +} + +// lapsedTurns registers, for every open walk waiting for this owner's word, the turn of each delivery whose stop +// lapsed and whose merge it answers (lapsedOn, novox/hq issue 753). The word is then given as for any walk: once, +// when every live delivery it carries is at its turn too and it carries no stopped change still held back (words). +// Given once per walk: the walk's id is in the delivery's LetGoWhy once its word was given. Under the lock. +func (h *Holder) lapsedTurns() { + for id, s := range h.walks { + for _, did := range h.lapsedOn(s.walk) { + d := h.deliveries[did] + if strings.Contains(d.LetGoWhy, lapsedWord(id)) { + continue + } + if h.turning == nil { + h.turning = map[string]*turnWant{} + } + want := h.turning[id] + if want == nil { + want = &turnWant{} + h.turning[id] = want + } + if !containsString(want.ids, d.ID) { + want.ids = append(want.ids, d.ID) + want.whys = append(want.whys, fmt.Sprintf("%s: %s its merge %s, stopped by its group for %s, whose change "+ + "was delivered since; it waits for a later walk of %s that contains its merge, and this is that "+ + "walk (novox/hq issue 753)", d.ID, lapsedWord(id), shortOf(d.MergedAs, 8), d.stoppedFor(), d.Repository)) + } + } + } +} + +// lapsedWord is how the word a lapsed stop gives a walk names that walk. +func lapsedWord(walk string) string { return "the walk " + walk + " answers" } + +// words are the asks of the word of every walk whose turn came for every live delivery it carries: one ask per +// walk, however many deliveries it carries, and none while one of them is not at its turn (ADR 0276 decision 5). +// Under the lock. +func (h *Holder) words() []func() { + walks := make([]string, 0, len(h.turning)) + for walk := range h.turning { + walks = append(walks, walk) + } + sort.Strings(walks) + var asks []func() + h.waitsWhy = map[string]string{} + for _, walk := range walks { + want := h.turning[walk] + var waiting []string + for _, d := range h.deliveries { + if d.State.Final() || d.Walk == nil || d.Walk.ID != walk || containsString(want.ids, d.ID) { + continue + } + waiting = append(waiting, fmt.Sprintf("%s (%s)", d.ID, d.State)) + } + if len(waiting) > 0 { + sort.Strings(waiting) + h.waitsWhy[walk] = strings.Join(waiting, ", ") + " carried by it not at its turn" + h.Logf("[mesh-delivery] the walk %s is not let go yet: %s", walk, h.waitsWhy[walk]) + continue + } + ids, why := append([]string(nil), want.ids...), strings.Join(want.whys, "; ") + // **Never the word for a walk that carries a stopped change** (ADR 0299, review of #221): it is asked to + // leave the merge out first (leaveStopped). + if stopped, _ := h.carriesStopped(walk); len(stopped) > 0 { + h.waitsWhy[walk] = "it carries " + strings.Join(stopped, ", ") + ", whose delivery was stopped: it is " + + "asked to leave that merge out first" + if hold := h.coverHeld(walk); hold != "" { + h.waitsWhy[walk] = hold + } + h.Logf("[mesh-delivery] the walk %s is not let go: it carries %s, whose delivery was stopped", walk, + strings.Join(stopped, ", ")) + continue + } + if h.asking[walk] { + continue // its word is being asked + } + h.asking[walk] = true + walk := walk + asks = append(asks, func() { + err := h.Controller.Deliver(walk, why) + h.mu.Lock() + defer h.mu.Unlock() + delete(h.asking, walk) + if err != nil { + if h.deliverErr == nil { + h.deliverErr = map[string]string{} + } + h.deliverErr[walk] = err.Error() // said by `stalled` past the grace (review of novox/hq issue 753) + h.Logf("[mesh-delivery] the walk %s could not be let go yet: %v", walk, err) + h.again("a walk's word the controller refused or did not answer") + return + } + delete(h.deliverErr, walk) + for _, id := range ids { + d := h.deliveries[id] + if d == nil { + continue + } + d.LetGo, d.LetGoWhy = h.Now(), why + if d.Walk != nil && d.Walk.ID == walk { + d.Walk.Waits, d.Walk.GoBy = false, byOwner + } + h.settleAndKeep(d) + } + }) + } + h.turning = nil + return asks +} + +// endedMovingNothing is whether a delivery ended delivered with nothing for a walk to move: merged, its plan +// moving no module the mesh holds, and no walk followed. +func endedMovingNothing(d *Delivery) bool { + return d.State == Delivered && d.MergedAs != "" && d.Walk == nil && d.MovesNothing() +} + +// endedAsNothingToMove are the deliveries a walk answers that ended delivered as nothing for a walk to move (novox/hq +// issues 362, 529: the nzbgeek case); empty when it answers any other. Such a walk is held by nothing: said, never +// let go by this owner. Under the lock. +func (h *Holder) endedAsNothingToMove(w Walk) []string { + var out []string + for _, d := range h.deliveries { + _, answers := walkAnswers(w, d) + if !answers && (d.Walk == nil || d.Walk.ID != w.ID) { + continue + } + if !endedMovingNothing(d) { + return nil + } + out = append(out, d.ID) + } + sort.Strings(out) + return out +} + +// WalksWaiting is `stalled` asked with `walks` (novox/hq issue 529): for each walk waiting for this owner's word, +// what holds it, in the shape of a stalled line naming the walk — each held delivery it carries (State held, its +// reason and the verb that releases it), else why its word was not given at the last look (State published), else +// that no live delivery holds it (State unheld). A walk whose word is being given is not listed. The controller's +// `plans` says these instead of pointing at a hand start; the self-check never raises a condition for them. +func (h *Holder) WalksWaiting() []StalledLine { + h.mu.Lock() + defer h.mu.Unlock() + now := h.Now() + var out []StalledLine + for id, s := range h.walks { + w := s.walk + if w.Release != nil || w.IsBatch() || !w.Open() || !w.Seen().Waits || h.asking[id] { + continue + } + waited := now.Sub(w.Created).Round(time.Second).String() + var held []StalledLine + live := false + for _, d := range h.deliveries { + if d.State.Final() || d.Walk == nil || d.Walk.ID != id { + continue + } + live = true + if d.State == Held { + held = append(held, StalledLine{ID: d.ID, Number: d.Number, State: Held, Walk: id, For: waited, + Bound: "0s", H2: "none: held for a person — the operator's", Says: HeldLine(d)}) + } + } + switch { + case len(held) > 0: + out = append(out, held...) + case h.coverHeld(id) != "": + out = append(out, StalledLine{ID: IDOf(w.Repository, w.Commit), State: Published, Walk: id, For: waited, + Bound: coverAskBound.String(), H2: "none: it is asked to leave the merge out past the bound", + Says: "held back: " + h.coverHeld(id)}) + case h.waitsWhy[id] != "": + out = append(out, StalledLine{ID: IDOf(w.Repository, w.Commit), State: Published, Walk: id, For: waited, + Bound: "0s", H2: "none: it goes at its turn", Says: "not let go yet: " + h.waitsWhy[id]}) + case !live && h.lapsedOwed(w): + // Its word comes from the stopped deliveries whose merge it answers (novox/hq issue 753): said, never + // unheld, while the controller has not taken it, with what holds it. + lapsed := h.lapsedOn(w) + says := "its word is owed by the stopped deliveries whose merge it answers, each stopped for a member " + + "delivered since (novox/hq issue 753)" + if len(lapsed) > 0 { + says = "its word is given for " + strings.Join(lapsed, ", ") + ", stopped for a member delivered since, " + + "whose merge it answers (novox/hq issue 753)" + } + if why := h.lapsedSaid(w); why != "" { + says += "; " + why + } + out = append(out, StalledLine{ID: IDOf(w.Repository, w.Commit), State: Published, Walk: id, For: waited, + Bound: "0s", H2: "none: it goes at its turn", Says: says}) + case !live: + says := "no delivery that has not ended follows it or is answered by it, so nothing gives it its word: " + + "`mesh-delivery.show` for the deliveries of " + w.Repository + if ended := h.endedAsNothingToMove(w); len(ended) > 0 { + // The nzbgeek case (novox/hq issues 362, 529): its merges' deliveries ended as nothing for a walk to + // move, yet the walk moves modules — nothing judged those, so nothing here lets it go. + says = "held by nothing: its merges' deliveries ended as nothing-to-move (" + strings.Join(ended, ", ") + + "), yet the walk moves " + strings.Join(walkModules(w), ", ") + " — nothing gives it its word" + } + out = append(out, StalledLine{ID: IDOf(w.Repository, w.Commit), State: Unheld, Walk: id, For: waited, + Bound: "0s", H2: "none: no delivery holds the walk — the operator's", Says: says + h.endedSaid(w)}) + } + } + sort.Slice(out, func(i, j int) bool { + if out[i].Walk != out[j].Walk { + return out[i].Walk < out[j].Walk + } + return out[i].ID < out[j].ID + }) + return out +} + +// walkModules are the modules a walk's record names, sorted. +func walkModules(w Walk) []string { + out := make([]string, 0, len(w.Modules)) + for m := range w.Modules { + out = append(out, m) + } + sort.Strings(out) + return out +} diff --git a/modules/mesh-delivery/cmd/mesh-delivery/approval.go b/modules/mesh-delivery/cmd/mesh-delivery/approval.go new file mode 100644 index 00000000..104f270a --- /dev/null +++ b/modules/mesh-delivery/cmd/mesh-delivery/approval.go @@ -0,0 +1,557 @@ +package main + +import ( + "fmt" + "regexp" + "sort" + "strconv" + "strings" + "time" + + "git.novox.be/novox/mesh-sdk/go/asks" +) + +// A merge into main waits for the operator's Approve (novox/hq ADR 0398 §3, issue 948, ADR 0421). +// +// This owner asks the operator on the router (the operator channel, as the knowledge base asks for a record) as soon as +// a pull request's checks pass: Approve or Decline, each option binding the act "merge these heads". It merges only after +// an Approve whose warrant holds for that ask — the same asker, the same ask's digest, an option it offered, in time, and +// the act that option bound. +// +// - Asked once per head. A group is asked once, for all its members' heads together, by the first member that reaches +// this point; the Approve covers every member at those heads. +// - A push after the ask ends it, said once on the pull request: the new head is asked on its own. +// - A Decline merges nothing and is said once on the pull request. +// - An ask that ended without a choice (expired, cancelled, refused by the router) is asked again when the router says so. +// - The ask, its state and the warrant's words are kept on the delivery's ReadyRun and shown by `show`. +// +// The terminal Approve (ADR 0398 §3) is issue 953; today the proving channel is the operator's phone. + +// Module is this module's name: the one the bus grants its asks under. +const Module = "mesh-delivery" + +// The two answers, both at the level approve. +const ( + OptionApprove = "approve" + OptionDecline = "decline" +) + +// AskLasts is how long the operator has to answer: a day less a margin, under the router's bound for an approval. +const AskLasts = asks.ApproveLasts - 10*time.Minute + +// Operator is the operator channel as this owner reaches it: asker.Client in the running module. +type Operator interface { + // Ask checks, keeps and publishes an ask under this module's name. + Ask(a asks.Ask) error + // Cancel takes an ask back. + Cancel(id string) error + // Where is an ask's state as the router keeps it, and its warrant once there is one. + Where(id string) (string, *asks.Warrant, error) +} + +// headsString is the heads an act names, as an act's value: "=" for each, sorted, joined by commas. +func headsString(heads map[string]string) string { + var out []string + for id, sha := range heads { + out = append(out, id+"="+sha) + } + sort.Strings(out) + return strings.Join(out, ",") +} + +// mergeAct is what an answer binds: this ask, the answer, and the heads it covers. +func mergeAct(ask, option string, heads map[string]string) asks.Act { + return asks.Act{"verb": Module + ".merge", "ask": ask, "option": option, "heads": headsString(heads)} +} + +var unshownPath = regexp.MustCompile(`[/\\@~$]|\d+\.\d+\.\d+|[A-Za-z0-9+/=_]{24,}|(?i)\b[a-z0-9-]+\.[a-z]{2,12}\b`) + +// phone is a line the operator's phone may be shown: what the router refuses in an ask's words (a path, an address, a +// host-like name, a long unbroken word) is left out, and the line is cut to n characters. +func phone(s string, n int) string { + s = strings.Join(strings.Fields(unshownPath.ReplaceAllString(s, "…")), " ") + if r := []rune(s); len(r) > n { + return string(r[:n-1]) + "…" + } + return s +} + +var issueNamed = regexp.MustCompile(`(?i)\bissues?\s*#?(\d+)`) + +// IssuesIn reads the issues a pull request's title and body name ("issue 948"), ascending, once each. +func IssuesIn(text string) []int { + seen := map[int]bool{} + var out []int + for _, m := range issueNamed.FindAllStringSubmatch(text, -1) { + if n, err := strconv.Atoi(m[1]); err == nil && !seen[n] { + seen[n] = true + out = append(out, n) + } + } + sort.Ints(out) + return out +} + +func issuesSaid(ds []*Delivery) string { + seen := map[int]bool{} + var out []string + for _, d := range ds { + for _, n := range d.Issues { + if !seen[n] { + seen[n] = true + out = append(out, "issue "+strconv.Itoa(n)) + } + } + } + if len(out) == 0 { + return "none named" + } + return strings.Join(out, ", ") +} + +func repoName(repository string) string { + _, name, ok := strings.Cut(repository, "/") + if !ok { + return repository + } + return name +} + +func pullSaid(d *Delivery) string { return fmt.Sprintf("%s#%d", repoName(d.Repository), d.Number) } + +// composeMergeAsk is the ask for a delivery, or for a group's members together. +func composeMergeAsk(id string, d *Delivery, group string, members []*Delivery, heads map[string]string, now time.Time) asks.Ask { + var a asks.Ask + a.ID, a.Who, a.Expires = id, asks.Operator, now.Add(AskLasts) + a.OnExpiry = "nothing is merged; the ask is made again" + a.About = "delivery." + phone(repoName(d.Repository), 40) + "." + strconv.Itoa(d.Number) + a.About = strings.ReplaceAll(a.About, " ", "-") + var reviewers, reaches, details []string + unreviewed := "" + for _, m := range members { + words, ok := reviewWords(m) + if !ok && unreviewed == "" { + unreviewed = pullSaid(m) + " is " + words + } + reviewers = append(reviewers, fmt.Sprintf("%s: %s", pullSaid(m), words)) + plan := "no delivery plan yet" + if m.Plan != nil && m.Plan.Summary != "" { + plan = m.Plan.Summary + } + reaches = append(reaches, plan) + details = append(details, fmt.Sprintf("%s %q at %s: issues: %s; review: %s; reaches: %s", + pullSaid(m), m.Title, shortOf(m.Commit, 12), issuesSaid([]*Delivery{m}), words, plan)) + } + if group == "" { + a.Headline = phone("Merge into main: "+pullSaid(d)+" "+d.Title, asks.HeadlineLength) + summary := fmt.Sprintf("%s %s passed its checks. Approve merges it into main at %s; Decline leaves it unmerged.", + pullSaid(d), d.Title, shortOf(d.Commit, 12)) + if unreviewed != "" { + summary = strings.ToUpper(unreviewed[:1]) + unreviewed[1:] + ". " + summary + } + a.Explanation = strings.Join([]string{ + phone(summary, asks.SummaryLength), + phone("Issues: "+issuesSaid(members), asks.PointLength), + phone("Review: "+reviewers[0], asks.PointLength), + phone("Reaches: "+reaches[0], asks.PointLength), + }, "\n") + } else { + var names []string + for _, m := range members { + names = append(names, pullSaid(m)) + } + a.Headline = phone("Merge into main: group "+group+" ("+strings.Join(names, ", ")+")", asks.HeadlineLength) + summary := fmt.Sprintf("The group %s, %d pull requests, passed its checks. Approve merges all of them into main in one pass, at the heads named; Decline leaves them unmerged.", + group, len(members)) + if unreviewed != "" { + summary = strings.ToUpper(unreviewed[:1]) + unreviewed[1:] + ". " + summary + } + a.Explanation = strings.Join([]string{ + phone(summary, asks.SummaryLength), + phone("Issues: "+issuesSaid(members), asks.PointLength), + phone("Review: "+strings.Join(reviewers, "; "), asks.PointLength), + phone("Reaches: "+strings.Join(reaches, "; "), asks.PointLength), + }, "\n") + } + a.Details = phone(strings.Join(details, "\n"), 1500) + if len(details) > 0 { // keep the lines apart: phone joined them + a.Details = strings.Join(func() []string { + var out []string + for _, l := range details { + out = append(out, phone(l, 400)) + } + return out + }(), "\n") + } + for _, o := range []struct{ id, label, does string }{ + {OptionApprove, "Approve", "the pull request is merged into main at the head named (a group: all its members, in one pass)"}, + {OptionDecline, "Decline", "nothing is merged"}, + } { + binds, _ := asks.ActDigest(mergeAct(id, o.id, heads)) + a.Options = append(a.Options, asks.Option{ID: o.id, Label: o.label, Does: o.does, Level: asks.Approve, Binds: binds}) + } + return a +} + +// askMembers are the deliveries one ask covers: the group's members not yet merged, by id, or the delivery. +func (h *Holder) askMembers(d *Delivery) []*Delivery { + g := h.groups[d.Group] + if d.Group == "" || g == nil { + return []*Delivery{d} + } + var out []*Delivery + seen := map[string]bool{} + ids := append([]string(nil), g.Members...) + sort.Strings(ids) + for _, id := range ids { + if o := h.deliveries[id]; o != nil && !seen[id] && o.MergedAs == "" && !o.State.Final() { + seen[id] = true + out = append(out, o) + } + } + if len(out) == 0 { + return []*Delivery{d} + } + return out +} + +// covering is the ready run holding an ask (pending or approved) that covers d at its head: its own, or a group +// sibling's, whose ask named d's head too. +func (h *Holder) covering(d *Delivery) *ReadyRun { + has := func(o *Delivery) *ReadyRun { + if o != nil && o.Ready != nil && (o.Ready.AskID != "" || o.Ready.Approved != nil) && o.Ready.Heads[d.ID] == d.Commit { + return o.Ready + } + return nil + } + if r := has(d); r != nil { + return r + } + if g := h.groups[d.Group]; d.Group != "" && g != nil { + for _, id := range g.Members { + if r := has(h.deliveries[id]); r != nil { + return r + } + } + } + return nil +} + +func headsEqual(a, b map[string]string) bool { + if len(a) != len(b) { + return false + } + for k, v := range a { + if b[k] != v { + return false + } + } + return true +} + +// headsHold is whether an ask's heads are still the heads of d's group: every member now at the head asked, and every +// head asked that is no member now a member already merged. +func (h *Holder) headsHold(d *Delivery, c *ReadyRun) bool { + now := map[string]bool{} + for _, m := range h.askMembers(d) { + now[m.ID] = true + if c.Heads[m.ID] != m.Commit { + return false + } + } + for id := range c.Heads { + if !now[id] { + if o := h.deliveries[id]; o == nil || o.MergedAs == "" { + return false + } + } + } + return true +} + +// endStaleAsks ends every ask whose heads are no longer its group's: a push to any member ends the approval (ADR 0398 +// §3), and says so once on each pull request it covered (ADR 0421 §6). An ask for a head a newer one replaced ends too. +// A Decline stands, and is not ended. Under the lock; the cancels run with the look's asks. +func (h *Holder) endStaleAsks() { + for _, d := range h.deliveries { + r := d.Ready + if r == nil || r.Merged != nil || d.MergedAs != "" || r.Declined != nil || (r.AskID == "" && r.Approved == nil) || len(r.Heads) == 0 { + continue + } + switch { + case d.State.Final(): + h.cancels = append(h.cancels, h.dropAsk(r.AskID, "its head "+shortOf(d.Commit, 12)+" is no longer the pull request's head, or the pull request closed")) + case !h.headsHold(d, r): + h.cancels = append(h.cancels, h.dropAsk(r.AskID, "a head it covers moved on: the heads it covers are no longer the group's heads")) + } + } +} + +// approval is what a delivery that may merge waits on for the operator: empty once its head is approved, and the ask +// to make when none is made yet. Under the lock. +func (h *Holder) approval(d *Delivery, now time.Time) (string, func()) { + r := d.Ready + c := h.covering(d) + if c != nil { + if c.Approved != nil { + r.Approved, r.Warrant, r.AskID, r.Ask, r.Heads = c.Approved, c.Warrant, c.AskID, c.Ask, c.Heads + return "", nil + } + return "the operator's Approve is asked (" + c.AskID + ")", nil + } + if r.Refused != "" { + return "the operator's answer was refused here: " + r.Refused, nil + } + if h.Operator == nil { + return "the operator cannot be asked from here", nil + } + members := h.askMembers(d) + group := "" + if len(members) > 1 { + group = d.Group + } + heads := map[string]string{} + for _, m := range members { + heads[m.ID] = m.Commit + } + r.Asks++ + id := fmt.Sprintf("merge-%d-%s-%d", d.Number, shortOf(d.Commit, 12), r.Asks) + ask := composeMergeAsk(id, d, group, members, heads, now) + taken := []*Delivery{} + var reads []memberRead + for _, m := range members { + h.runOf(m, now) + m.Ready.AskID, m.Ready.Ask, m.Ready.Heads, m.Ready.AskedAt, m.Ready.Asks = id, &ask, heads, now, max(m.Ready.Asks, r.Asks) + h.keep(m) + taken = append(taken, m) + reads = append(reads, readOf(m, m.Commit)) + } + h.Logf("[mesh-delivery] %s passed its checks: the operator is asked to Approve its merge into main (%s, ask %d)", + d.ID, id, r.Asks) + gid := "" + if group != "" { + gid = d.Group + } + return "the operator's Approve is asked (" + id + ")", func() { + // A group is not asked while a member's required status is not a success at its head (issue 1275): the group + // says which, and is asked once every member may merge. + var holdMember, holdSays string + var readErr error + for _, m := range reads { + if gid == "" { + break + } + says, _, err := h.notASuccess(m) + if err != nil { + readErr = err + break + } + if says != "" { + holdMember, holdSays = m.id, says + break + } + } + undo := func() { + for _, m := range taken { + if m.Ready != nil && m.Ready.AskID == id { + m.Ready.AskID, m.Ready.Ask, m.Ready.Heads = "", nil, nil + h.keep(m) + } + } + } + if readErr != nil || holdSays != "" { + h.mu.Lock() + defer h.mu.Unlock() + undo() + if g := h.groups[gid]; g != nil && holdSays != "" { + if g.Hold == nil || g.Hold.Kind != "unasked" || g.Hold.Says != holdSays || g.Hold.Member != holdMember { + h.Logf("[mesh-delivery] group %s is not asked for its Approve: %s waits on a required status: %s", gid, holdMember, holdSays) + g.Hold = &GroupHold{Kind: "unasked", Member: holdMember, Says: holdSays, Since: h.Now()} + h.keepGroup(g) + } + } + if readErr != nil { + h.Logf("[mesh-delivery] group %s: the required statuses cannot be read, so the operator is not asked yet: %v", gid, readErr) + } + h.again("a group's Approve held for a required status") + return + } + if g := func() *Group { h.mu.Lock(); defer h.mu.Unlock(); return h.groups[gid] }(); g != nil && g.Hold != nil && g.Hold.Kind == "unasked" { + h.mu.Lock() + g.Hold = nil + h.keepGroup(g) + h.mu.Unlock() + } + if err := h.Operator.Ask(ask); err != nil { + h.mu.Lock() + defer h.mu.Unlock() + undo() + h.again("the operator's Approve, which could not be asked") + h.Logf("[mesh-delivery] the operator could not be asked about %s (%v); asked again in %s", d.ID, err, askAgainAfter) + } + } +} + +// dropAsk forgets a pending ask on every delivery it covers and takes it back from the router; under the lock. What it +// says is said once on each pull request it covered (ADR 0421 §6): the Approve is ended, and why. +func (h *Holder) dropAsk(id, why string) func() { + var said []func() + seen := map[string]bool{} + var covered []*Delivery + for _, d := range h.deliveries { + if d.Ready != nil && d.Ready.AskID == id { + covered = append(covered, d) + } + } + sort.Slice(covered, func(i, j int) bool { return covered[i].ID < covered[j].ID }) + for _, d := range covered { + d.Ready.AskID, d.Ready.Ask, d.Ready.Heads = "", nil, nil + if d.Ready.Merged == nil { + d.Ready.Approved, d.Ready.Warrant = nil, "" + } + h.keep(d) + if k := fmt.Sprintf("%s#%d", d.Repository, d.Number); !seen[k] && d.Number != 0 { + seen[k] = true + said = append(said, h.sayOn(d, "mesh-delivery: the operator's Approve of this pull request's merge into main is ended: "+why+ + ". Nothing is merged on it. The operator is asked again once the checks pass at the new head.")) + } + } + h.Logf("[mesh-delivery] the operator's ask %s is cancelled: %s", id, why) + return func() { + for _, s := range said { + s() + } + if h.Operator == nil { + return + } + if err := h.Operator.Cancel(id); err != nil { + h.Logf("[mesh-delivery] the operator's ask %s could not be cancelled (%v)", id, err) + } + } +} + +// withdrawAsk takes back an ask nobody answered, on every delivery it covers, so the next look asks again with what is +// known now (issue 1079). Unlike dropAsk it ends no Approve and says nothing on a pull request: there was none. Under the +// lock; the router's cancel runs with the look's asks. +func (h *Holder) withdrawAsk(id, why string) func() { + for _, d := range h.deliveries { + if d.Ready != nil && d.Ready.AskID == id && d.Ready.Approved == nil { + d.Ready.AskID, d.Ready.Ask, d.Ready.Heads = "", nil, nil + h.keep(d) + } + } + h.Logf("[mesh-delivery] the operator's ask %s is withdrawn to be asked again: %s", id, why) + return func() { + if h.Operator == nil { + return + } + if err := h.Operator.Cancel(id); err != nil { + h.Logf("[mesh-delivery] the operator's ask %s could not be withdrawn (%v)", id, err) + } + } +} + +// pendingAsks are the asks waiting for an answer, by id; under the lock. +func (h *Holder) pendingAsks() []string { + seen := map[string]bool{} + var out []string + for _, d := range h.deliveries { + if r := d.Ready; r != nil && r.AskID != "" && r.Approved == nil && r.Declined == nil && !seen[r.AskID] { + seen[r.AskID] = true + out = append(out, r.AskID) + } + } + sort.Strings(out) + return out +} + +// hearAsks reads the router's record of every pending ask and applies the warrant of each that closed. +func (h *Holder) hearAsks() { + if h.Operator == nil { + return + } + h.mu.Lock() + ids := h.pendingAsks() + h.mu.Unlock() + for _, id := range ids { + state, warrant, err := h.Operator.Where(id) + if err != nil { + h.Logf("[mesh-delivery] the router's record of the ask %s cannot be read: %v", id, err) + continue + } + if state == "" || state == "open" || state == "held" { + continue + } + h.mu.Lock() + said := h.decided(id, state, warrant) + h.mu.Unlock() + for _, s := range said { + s() + } + } +} + +// decided applies one closed ask's warrant to the deliveries it covers; under the lock. It returns what to say on the +// pull requests. +func (h *Holder) decided(id, state string, w *asks.Warrant) []func() { + var covered []*Delivery + for _, d := range h.deliveries { + if d.Ready != nil && d.Ready.AskID == id { + covered = append(covered, d) + } + } + if len(covered) == 0 { + return nil + } + sort.Slice(covered, func(i, j int) bool { return covered[i].ID < covered[j].ID }) + first, now := covered[0].Ready, h.Now() + end := func(why string) { + for _, d := range covered { + d.Ready.AskID, d.Ready.Ask, d.Ready.Heads = "", nil, nil + h.keep(d) + } + h.Logf("[mesh-delivery] the operator's ask %s ended without a choice (%s); asked again at once", id, why) + } + if w == nil || w.Outcome != asks.OutcomeChosen { + why := "the router says " + state + if w != nil { + why = "ended " + string(w.Outcome) + if w.Words != "" { + why += " (" + w.Words + ")" + } + } + end(why) + return nil + } + option, err := w.For(Module, *first.Ask) + if err == nil { + err = option.Performs(mergeAct(id, option.ID, first.Heads)) + } + if err != nil { + h.Logf("[mesh-delivery] REFUSED the answer to the ask %s: %v", id, err) + for _, d := range covered { + d.Ready.AskID, d.Ready.Ask, d.Ready.Heads = "", nil, nil + d.Ready.Refused = "was answered, and the answer does not hold here: " + clip(oneLine(err.Error()), 300) + h.keep(d) + } + return nil + } + var said []func() + for _, d := range covered { + r := d.Ready + r.Warrant = w.Says() + h.mergeAgainNow(d) // the operator chose: the merge is asked at once + switch option.ID { + case OptionApprove: + r.Approved = &now + h.Logf("[mesh-delivery] %s: the operator approved its merge into main (%s)", d.ID, r.Warrant) + case OptionDecline: + r.Declined = &now + h.Logf("[mesh-delivery] %s: the operator declined its merge into main (%s)", d.ID, r.Warrant) + said = append(said, h.sayOn(d, fmt.Sprintf("mesh-delivery: **not merged.** The merge of this pull request into main at %s was "+ + "declined: %s. A new head asks the operator again.", d.Commit, r.Warrant))) + } + h.keep(d) + } + return said +} diff --git a/modules/mesh-delivery/cmd/mesh-delivery/approval_test.go b/modules/mesh-delivery/cmd/mesh-delivery/approval_test.go new file mode 100644 index 00000000..d440d1af --- /dev/null +++ b/modules/mesh-delivery/cmd/mesh-delivery/approval_test.go @@ -0,0 +1,354 @@ +package main + +import ( + "strings" + "testing" + "time" + + "git.novox.be/novox/mesh-sdk/go/asks" +) + +// **A merge into main waits for the operator's Approve** (novox/hq ADR 0398 §3, issue 948): a marked, green delivery +// into main is asked of the operator, and merged only after an Approve whose warrant holds for that ask. + +// markedGreen is a pull request into main, announced, judged green and marked ready at its head; the operator is not +// answering by himself (w.op.auto is off). +func markedGreen(t *testing.T) *world { + w := newWorld(t) + w.op.auto = false + p := pr("novox/app", 7, readyHead, "feat/x", "src/a.go") + p.Title = "Fix the apostrophe (issue 948, issue 12)" + p.Body = "Reviewed-by: reviewer@shanks at " + readyHead + w.h.PullUpdated(p) + w.h.Checked(judgedAt("novox/app", 7, readyHead, "pass", "t1t1t1t1t1t1")) + w.forge.open = []OpenPull{openPull("novox/app", 7, readyHead)} + w.h.SeeOpenPulls() + w.settleAll() + return w +} + +func TestAMarkedGreenDeliveryIntoMainIsAskedAndNotMergedUntilApproved(t *testing.T) { + w := markedGreen(t) + id := IDOf("novox/app", readyHead) + if len(w.forge.merges) != 0 { + t.Fatalf("merged before the operator's Approve: %v", w.forge.merges) + } + open := w.op.open() + if len(open) != 1 { + t.Fatalf("the operator is asked %v; wanted exactly one ask", open) + } + a := w.op.ask(open[0]) + if !strings.HasPrefix(a.Headline, "Merge into main: app#7") || a.Who != asks.Operator || + len(a.Options) != 2 || a.Options[0].ID != OptionApprove || a.Options[1].ID != OptionDecline { + t.Fatalf("the ask is not the merge ask: %+v", a) + } + for _, want := range []string{"issue 948", "issue 12", "reviewed by reviewer on shanks", "builds app"} { + if !strings.Contains(a.Explanation, want) { + t.Fatalf("the ask does not say %q: %s", want, a.Explanation) + } + } + if err := asks.Glance(a.Explanation); err != nil { + t.Fatalf("the ask does not read at a glance: %v", err) + } + // Ticks and minutes pass without an answer: not merged, not asked again. + for range 3 { + w.later(2 * time.Minute) + w.settleAll() + } + if len(w.forge.merges) != 0 || w.op.count() != 1 { + t.Fatalf("while waiting: merges %v, asks %d", w.forge.merges, w.op.count()) + } + if d := w.delivery(id); d.Ready.AskID != open[0] || !strings.Contains(d.Ready.Waits, "Approve") { + t.Fatalf("the ask is not kept on the delivery: %+v", d.Ready) + } + w.op.answer(open[0], OptionApprove) + w.settleAll() + if len(w.forge.merges) != 1 || w.forge.merges[0] != "novox/app#7@"+readyHead { + t.Fatalf("after the Approve it was merged %v; wanted once, at the marked head", w.forge.merges) + } + d := w.delivery(id) + if d.Ready.Approved == nil || !strings.Contains(d.Ready.Warrant, "chose Approve") { + t.Fatalf("the warrant is not kept: %+v", d.Ready) + } + w.h.PullMerged(merged("novox/app", 7, readyHead, "m7m7m7m7m7m7")) + w.h.WalkMoved(w.ctl.put(aWalk("plan-7", "novox/app", "m7m7m7m7m7m7", true))) + w.settleAll() + var last Transition + for _, tr := range w.delivery(id).Transitions { + if tr.Event == EvMerged { + last = tr + } + } + if !strings.Contains(last.By, "approved: the operator, via telegram") { + t.Fatalf("the merge's transition does not name the operator's Approve: %+v", last) + } +} + +func TestAnAnswerThatIsNotForThisAskMergesNothing(t *testing.T) { + w := markedGreen(t) + id := w.op.open()[0] + w.op.answer(id, OptionApprove) + w.op.mu.Lock() + w.op.warrant[id].AskDigest = "sha256:not-the-ask-that-was-shown" + w.op.mu.Unlock() + w.settleAll() + if len(w.forge.merges) != 0 { + t.Fatalf("a warrant for another ask merged: %v", w.forge.merges) + } + d := w.delivery(IDOf("novox/app", readyHead)) + if !strings.Contains(d.Ready.Refused, "digest") || d.Ready.Approved != nil { + t.Fatalf("the refused answer is not kept: %+v", d.Ready) + } + if lines := w.h.Stalled(); len(lines) != 1 || !strings.Contains(lines[0].Says, "answer") { + t.Fatalf("a refused answer is not said loudly: %+v", lines) + } +} + +func TestADeclinedMergeIsNotMergedAndSaidOnce(t *testing.T) { + w := markedGreen(t) + w.op.answer(w.op.open()[0], OptionDecline) + for range 3 { + w.later(2 * time.Minute) + w.settleAll() + } + if len(w.forge.merges) != 0 || w.op.count() != 1 { + t.Fatalf("declined, yet merges %v and asks %d", w.forge.merges, w.op.count()) + } + if len(w.forge.comments) != 1 || !strings.Contains(w.forge.comments[0], "declined") || + !strings.Contains(w.forge.comments[0], "chose Decline") { + t.Fatalf("the Decline is not said once on the pull request, with its warrant: %v", w.forge.comments) + } +} + +func TestAPushAfterTheAskCancelsItAndTheNewHeadIsAskedOnItsOwnWhenItsChecksPass(t *testing.T) { + w := markedGreen(t) + old := w.op.open()[0] + w.h.PullUpdated(pr("novox/app", 7, pushed, "feat/x", "src/a.go")) + w.settleAll() + if len(w.op.cancelled) != 1 || w.op.cancelled[0] != old || len(w.op.open()) != 0 { + t.Fatalf("the ask was not cancelled: cancelled %v, open %v", w.op.cancelled, w.op.open()) + } + if len(w.forge.merges) != 0 || w.op.count() != 1 { + t.Fatalf("merges %v, asks %d", w.forge.merges, w.op.count()) + } + if d := w.delivery(IDOf("novox/app", readyHead)); d.Ready.AskID != "" { + t.Fatalf("the cancelled ask is still kept as waiting: %+v", d.Ready) + } + // The new head is asked afresh once its checks pass. + w.h.Checked(judgedAt("novox/app", 7, pushed, "pass", "t1t1t1t1t1t1")) + w.settleAll() + if open := w.op.open(); len(open) != 1 || open[0] == old { + t.Fatalf("the new head was not asked afresh: %v", open) + } +} + +func TestAnExpiredAskIsAskedAgainOnce(t *testing.T) { + w := markedGreen(t) + first := w.op.open()[0] + w.op.end(first) + w.settleAll() + if w.op.count() != 2 || len(w.op.open()) != 1 || w.op.open()[0] == first { + t.Fatalf("an expired ask was not asked again once: %d asks, open %v", w.op.count(), w.op.open()) + } + for range 3 { + w.later(2 * time.Minute) + w.settleAll() + } + if w.op.count() != 2 || len(w.forge.merges) != 0 { + t.Fatalf("asked more than once again: %d asks, merges %v", w.op.count(), w.forge.merges) + } +} + +func TestAGroupIsAskedOnceForAllItsHeadsAndMergedInOnePassOnOneApprove(t *testing.T) { + w := newWorld(t) + w.op.auto = false + const ctlHead, catHead = "c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1", "d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2" + w.h.PullUpdated(pr("novox/mesh-controller", 1, ctlHead, "feat/x", "internal/a.go")) + w.h.PullUpdated(pr("novox/mesh-catalog", 2, catHead, "feat/x", "modules/app/module.json")) + w.h.Checked(judgedAt("novox/mesh-controller", 1, ctlHead, "pass", "t1t1t1t1t1t1")) + w.h.Checked(judgedAt("novox/mesh-catalog", 2, catHead, "pass", "t3t3t3t3t3t3")) + w.forge.open = []OpenPull{openPull("novox/mesh-controller", 1, ctlHead), + openPull("novox/mesh-catalog", 2, catHead)} + w.h.SeeOpenPulls() + w.settleAll() + w.h.Checked(CheckedEvent{Group: "feat/x", ID: "check-1", Verdict: "pass", Summary: "both compose"}) + w.settleAll() + if open := w.op.open(); len(open) != 1 || len(w.forge.merges) != 0 { + t.Fatalf("a group is asked once before any merge: open %v, merges %v", w.op.open(), w.forge.merges) + } + a := w.op.ask(w.op.open()[0]) + if !strings.Contains(a.Headline, "group feat") || !strings.Contains(a.Details, "mesh-controller#1") || + !strings.Contains(a.Details, "mesh-catalog#2") { + t.Fatalf("the ask does not name the group and its members: %+v", a) + } + w.op.answer(a.ID, OptionApprove) + w.settleAll() + if len(w.forge.merges) != 2 || w.op.count() != 1 { + t.Fatalf("both members were not merged in one pass on the one Approve: merges %v, asks %d", w.forge.merges, w.op.count()) + } +} + +// A push to a group member without a mark ends the group's ask (ADR 0398 §3, issue 948; review round 1): the Approve +// that follows would be for heads the group no longer has. +func TestAPushToAGroupMemberEndsTheGroupsAskAndApproveAndSaysSoOnEveryMember(t *testing.T) { + w := newWorld(t) + w.op.auto = false + const ctlHead, catHead, catPushed = "c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1", "d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2", "e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3" + w.h.PullUpdated(pr("novox/mesh-controller", 1, ctlHead, "feat/x", "internal/a.go")) + w.h.PullUpdated(pr("novox/mesh-catalog", 2, catHead, "feat/x", "modules/app/module.json")) + w.h.Checked(judgedAt("novox/mesh-controller", 1, ctlHead, "pass", "t1t1t1t1t1t1")) + w.h.Checked(judgedAt("novox/mesh-catalog", 2, catHead, "pass", "t3t3t3t3t3t3")) + w.forge.open = []OpenPull{openPull("novox/mesh-controller", 1, ctlHead), + openPull("novox/mesh-catalog", 2, catHead)} + w.h.SeeOpenPulls() + w.settleAll() + w.h.Checked(CheckedEvent{Group: "feat/x", ID: "check-1", Verdict: "pass", Summary: "both compose"}) + w.settleAll() + first := w.op.open() + if len(first) != 1 { + t.Fatalf("asked %v", first) + } + // The catalog member is pushed; its new head has no mark yet. + w.h.PullUpdated(pr("novox/mesh-catalog", 2, catPushed, "feat/x", "modules/app/module.json")) + w.forge.open = []OpenPull{openPull("novox/mesh-controller", 1, ctlHead), + {Owner: "novox", Repo: "mesh-catalog", Number: 2, Base: "main", HeadSHA: catPushed}} + w.h.SeeOpenPulls() + w.settleAll() + if len(w.op.cancelled) != 1 || w.op.cancelled[0] != first[0] { + t.Fatalf("the group's ask was not cancelled by the push: %v", w.op.cancelled) + } + w.op.answer(first[0], OptionApprove) // a late answer to the cancelled ask + w.h.Checked(judgedAt("novox/mesh-catalog", 2, catPushed, "pass", "t3t3t3t3t3t3")) + w.h.Checked(CheckedEvent{Group: "feat/x", ID: "check-2", Verdict: "pass", Summary: "both compose again"}) + w.settleAll() + if len(w.forge.merges) != 0 { + t.Fatalf("merged on an Approve of heads the group no longer has: %v", w.forge.merges) + } +} + +// mergedSaid is the module events `merged` the world has said. +func mergedSaid(w *world) []map[string]any { + w.mu.Lock() + defer w.mu.Unlock() + var out []map[string]any + for _, m := range w.said { + if m["event-kind"] == "merged" { + out = append(out, m) + } + } + return out +} + +// Issue 1025: a merge on the operator's Approve is announced once, with the warrant's ask, option and heads. +func TestIssue1025AMergeOnTheApproveIsAnnouncedWithItsWarrant(t *testing.T) { + w := markedGreen(t) + id := IDOf("novox/app", readyHead) + open := w.op.open() + if len(open) != 1 { + t.Fatalf("asks %v", open) + } + if got := mergedSaid(w); len(got) != 0 { + t.Fatalf("announced before the merge: %v", got) + } + d := w.delivery(id) + wantHeads := map[string]string{} + for k, v := range d.Ready.Heads { + wantHeads[k] = v + } + w.op.answer(open[0], OptionApprove) + w.settleAll() + got := mergedSaid(w) + if len(got) != 1 { + t.Fatalf("announced %d times; wanted once: %v", len(got), got) + } + m := got[0] + if m["owner"] != "novox" || m["repo"] != "app" || m["number"] != float64(7) || m["head_sha"] != readyHead || + m["ask_id"] != open[0] || m["option"] != OptionApprove { + t.Fatalf("the event does not name the pull request and its warrant: %v", m) + } + hs, _ := m["heads"].(map[string]any) + if len(hs) == 0 || len(hs) != len(wantHeads) { + t.Fatalf("heads %v; wanted %v", hs, wantHeads) + } + for k, v := range wantHeads { + if hs[k] != v { + t.Fatalf("heads %v; wanted %v", hs, wantHeads) + } + } + if g, ok := m["group"]; ok && g != "" { + t.Fatalf("a lone delivery names group %v", g) + } + w.later(2 * time.Minute) + w.settleAll() + if n := len(mergedSaid(w)); n != 1 { + t.Fatalf("announced %d times after more ticks", n) + } +} + +// Issue 1025: a declined merge announces nothing. +func TestIssue1025ADeclinedMergeAnnouncesNothing(t *testing.T) { + w := markedGreen(t) + open := w.op.open() + w.op.answer(open[0], OptionDecline) + w.settleAll() + if got := mergedSaid(w); len(got) != 0 || len(w.forge.merges) != 0 { + t.Fatalf("declined, yet merges %v, announced %v", w.forge.merges, got) + } +} + +// Issue 1025: a merge of a delivery in a group carries the group id and the heads the group's Approve was bound to. +func TestIssue1025AMergeInAGroupCarriesTheGroupAndItsHeads(t *testing.T) { + w := newWorld(t) + w.op.auto = false + const ctlHead, catHead = "c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1", "d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2" + ctl, cat := IDOf("novox/mesh-controller", ctlHead), IDOf("novox/mesh-catalog", catHead) + w.h.PullUpdated(pr("novox/mesh-controller", 1, ctlHead, "feat/x", "internal/a.go")) + w.h.PullUpdated(pr("novox/mesh-catalog", 2, catHead, "feat/x", "modules/app/module.json")) + w.h.Checked(judgedAt("novox/mesh-controller", 1, ctlHead, "pass", "t1t1t1t1t1t1")) + w.h.Checked(judgedAt("novox/mesh-catalog", 2, catHead, "pass", "t3t3t3t3t3t3")) + w.forge.open = []OpenPull{openPull("novox/mesh-controller", 1, ctlHead), + openPull("novox/mesh-catalog", 2, catHead)} + w.h.SeeOpenPulls() + w.settleAll() + w.h.Checked(CheckedEvent{Group: "feat/x", ID: "check-1", Verdict: "pass", Summary: "both compose"}) + w.settleAll() + open := w.op.open() + if len(open) != 1 { + t.Fatalf("asks %v", open) + } + w.op.answer(open[0], OptionApprove) + w.settleAll() + got := mergedSaid(w) + if len(got) != 2 { + t.Fatalf("announced %d; wanted each member's merge once, from the one pass: %v", len(got), got) + } + for _, m := range got { + hs, _ := m["heads"].(map[string]any) + if m["group"] != "feat/x" || m["ask_id"] != open[0] || m["option"] != OptionApprove || + hs[ctl] != ctlHead || hs[cat] != catHead || len(hs) != 2 { + t.Fatalf("the group's merge event: %v", m) + } + } +} + +// Issue 1025: the announcement is owed and kept, so it is said after a restart, and once. +func TestIssue1025AnOwedMergedIsSaidAfterARestartAndOnce(t *testing.T) { + w := newWorld(t) + p := pr("novox/app", 7, readyHead, "feat/x", "src/a.go") + w.h.PullUpdated(p) + id := IDOf("novox/app", readyHead) + w.h.mu.Lock() + d := w.h.deliveries[id] + d.Owed = append(d.Owed, Effect{Kind: EffectEmit, Event: "merged", Line: `{"owner":"novox","repo":"app","number":7,"head_sha":"` + readyHead + `"}`, Since: w.now}) + if err := w.store.PutDelivery(d); err != nil { + t.Fatal(err) + } + w.h.mu.Unlock() + again := newWorldOver(t, w.store, w.ctl, w.forge, w.now.Add(time.Minute)) + again.settleAll() + again.later(time.Hour) + again.settleAll() + if n := len(mergedSaid(again)); n != 1 { + t.Fatalf("said %d times after the restart; wanted once", n) + } +} diff --git a/modules/mesh-delivery/cmd/mesh-delivery/backup_test.go b/modules/mesh-delivery/cmd/mesh-delivery/backup_test.go new file mode 100644 index 00000000..1b802538 --- /dev/null +++ b/modules/mesh-delivery/cmd/mesh-delivery/backup_test.go @@ -0,0 +1,154 @@ +package main + +import ( + "strings" + "testing" +) + +// A failed backup before the send holds the delivery, and a release asks the backups again (novox/hq ADR 0404 +// clause 4, issue 1068). The walk's backups field is the controller's contract. + +func failedBackup() *WalkBackups { + return &WalkBackups{State: "failed", Asked: []WalkBackupAsked{ + {Machine: "ace", Module: "mail", State: "done"}, + {Machine: "ace", Module: "pg", State: "failed", Words: "no space left on device"}}} +} + +// aDeliveringWalk is a delivery whose walk was let go and runs, nothing sent yet. +func aDeliveringWalk(t *testing.T) (*world, string, string) { + t.Helper() + w := newWorld(t) + id := IDOf("novox/app", head) + w.h.PullUpdated(pr("novox/app", 7, head, "feat/x", "src/a.go")) + w.h.Checked(verdict("novox/app", 7, head, "pass")) + w.h.PullMerged(merged("novox/app", 7, head, merge)) + w.h.WalkMoved(w.ctl.put(aWalk("plan-1", "novox/app", merge, true))) + w.settleAll() + if s := w.state(id); s != Delivering { + t.Fatalf("setup: %s, want delivering", s) + } + return w, id, "plan-1" +} + +func TestABackupFailedTakesADeliveringDeliveryToHeldOnlyWhileNothingWasSent(t *testing.T) { + w, id, plan := aDeliveringWalk(t) + sent := w.ctl.walk(plan) + sent.Backups = failedBackup() + sent.Modules["app"].SentAt = &w.now + w.h.WalkMoved(w.ctl.put(sent)) + w.settleAll() + if s := w.state(id); s == Held { + t.Fatal("held for a failed backup after a module of the plan was sent") + } + w, id, plan = aDeliveringWalk(t) + held := w.ctl.walk(plan) + held.Backups = failedBackup() + w.h.WalkMoved(w.ctl.put(held)) + w.settleAll() + d := w.delivery(id) + if d.State != Held { + t.Fatalf("with a failed backup and nothing sent it is %s", d.State) + } + for _, want := range []string{"ace", "pg", "no space left on device"} { + if !strings.Contains(d.HeldWhy, want) { + t.Errorf("the held reason %q does not name %q", d.HeldWhy, want) + } + } + // It does not lift by itself: the same facts read again leave it held. + w.h.WalkMoved(w.ctl.put(w.ctl.walk(plan))) + w.settleAll() + if s := w.state(id); s != Held { + t.Fatalf("read again it is %s, want it still held", s) + } +} + +func TestAReleaseOfADeliveryHeldForAFailedBackupAsksTheControllerAgainAndGoesToDelivering(t *testing.T) { + w, id, plan := aDeliveringWalk(t) + held := w.ctl.walk(plan) + held.Backups = failedBackup() + w.h.WalkMoved(w.ctl.put(held)) + w.settleAll() + if s := w.state(id); s != Held { + t.Fatalf("setup: %s, want held", s) + } + lettingGo := len(w.ctl.delivers) + if _, err := w.h.Release(id, "freed some space", "jochen"); err != nil { + t.Fatal(err) + } + if got := w.ctl.backupsAgain; len(got) != 1 || got[0] != plan { + t.Fatalf("the controller was asked to take the backups again %v, want once for %s", got, plan) + } + if len(w.ctl.delivers) != lettingGo { + t.Errorf("a started walk was let go again: %v", w.ctl.delivers) + } + if s := w.state(id); s != Delivering { + t.Fatalf("released, it is %s", s) + } + // The record the controller cleared is read: it stays delivering. + w.h.WalkMoved(w.ctl.walk(plan)) + w.settleAll() + if s := w.state(id); s != Delivering { + t.Fatalf("with the backups cleared it is %s", s) + } + // Another failure, in a later record, holds it again. + again := w.ctl.walk(plan) + again.Backups = failedBackup() + w.h.WalkMoved(w.ctl.put(again)) + w.settleAll() + if s := w.state(id); s != Held { + t.Fatalf("a failed backup in a later record left it %s", s) + } +} + +func TestARecordThatStillHoldsTheFailureAfterAReleaseDoesNotHoldItAgain(t *testing.T) { + w, id, plan := aDeliveringWalk(t) + held := w.ctl.walk(plan) + held.Backups = failedBackup() + stale := w.ctl.put(held) + w.h.WalkMoved(stale) + w.settleAll() + if _, err := w.h.Release(id, "freed some space", "jochen"); err != nil { + t.Fatal(err) + } + w.h.WalkMoved(stale) // the record as it was before the controller cleared it + w.settleAll() + if s := w.state(id); s != Delivering { + t.Fatalf("a record from before the release held it again: %s", s) + } +} + +func TestADeliveryHeldForAFailedBackupIsNotLetGoByItsWalkAlone(t *testing.T) { + // The controller's own path starts a walk with nobody's word: the held-to-delivering row for that must not + // undo the backup hold. + d := &Delivery{ID: "novox/app@c0", State: Held, MergedAs: "aaaa", Walk: &WalkSeen{ID: "plan-1", State: walkBuilding, Revision: 3, + BackupsFailed: []WalkBackupFailure{{Machine: "ace", Module: "pg", Words: "no space left on device"}}}} + if _, err := Apply(d, EvGo, false, Facts{}); err == nil { + t.Fatal("a walk started on the controller's own path lifted the backup hold") + } +} + +func TestAReleaseWhoseBackupsTheControllerRefusedToAskAgainLeavesTheDeliveryHeld(t *testing.T) { + w, id, plan := aDeliveringWalk(t) + held := w.ctl.walk(plan) + held.Backups = failedBackup() + w.h.WalkMoved(w.ctl.put(held)) + w.settleAll() + w.ctl.refuseBackupsAgain = true + _, err := w.h.Release(id, "freed some space", "jochen") + if err == nil || !strings.Contains(err.Error(), "held again") { + t.Fatalf("the release of a refused word answered %v, want an error saying it is held again", err) + } + w.settleAll() + d := w.delivery(id) + if d.State != Held || d.BackupsAgainAfter != 0 || !strings.Contains(d.HeldWhy, "no space left on device") { + t.Fatalf("after the refusal it is %s (again-after %d, why %q), want held for the backup", d.State, d.BackupsAgainAfter, d.HeldWhy) + } + // The controller takes the word the next time: the delivery goes on. + w.ctl.refuseBackupsAgain = false + if _, err := w.h.Release(id, "freed some space", "jochen"); err != nil { + t.Fatal(err) + } + if s := w.state(id); s != Delivering { + t.Fatalf("released again, it is %s", s) + } +} diff --git a/modules/mesh-delivery/cmd/mesh-delivery/bus_test.go b/modules/mesh-delivery/cmd/mesh-delivery/bus_test.go new file mode 100644 index 00000000..363da871 --- /dev/null +++ b/modules/mesh-delivery/cmd/mesh-delivery/bus_test.go @@ -0,0 +1,677 @@ +//go:build integration + +package main + +import ( + "bufio" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "os/exec" + "strings" + "sync" + "testing" + "time" + + stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/asks" + "github.com/nats-io/nats-server/v2/server" + "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" +) + +// **The acceptance of acting on events** (novox/hq ADR 0406, design 47 milestone 2, ADR 0391 phase 3, issue 975; ADR +// 0421, issue 1022): a pull request whose verdict passes is asked of the operator within seconds of the verdict's event, +// with no label, comment or mark written by anyone, and the merge is asked within five seconds of the Approve's event — +// with no tick, no poll and no reconcile to find it. +// Proved over a real bus: a JetStream server is linked into the test and started for it alone; the module is this +// test binary started again as the bundle, speaking MCP on its standard streams to a runtime played here, which does +// what the tool runner does for a bundle — its state on the buckets the controller would make from the manifest, its +// events delivered from the bus's streams, the controller's verbs and the forge's tools answered by the test (the +// pattern of mesh-issues' bus_test). + +const bundleVar = "MESH_DELIVERY_TEST_BUNDLE" + +func TestMain(m *testing.M) { + if os.Getenv(bundleVar) == "1" { + main() + os.Exit(0) + } + os.Exit(m.Run()) +} + +type testBus struct { + nc *nats.Conn + js jetstream.JetStream + kvs map[string]jetstream.KeyValue +} + +func newTestBus(t *testing.T) *testBus { + t.Helper() + opts := &server.Options{Host: "127.0.0.1", Port: server.RANDOM_PORT, JetStream: true, StoreDir: t.TempDir(), + NoLog: true, NoSigs: true} + s, err := server.NewServer(opts) + if err != nil { + t.Fatalf("a bus for this test could not be made: %v", err) + } + go s.Start() + if !s.ReadyForConnections(30 * time.Second) { + s.Shutdown() + t.Fatal("a bus for this test did not come up within 30s") + } + t.Cleanup(func() { s.Shutdown(); s.WaitForShutdown() }) + nc, err := nats.Connect(s.ClientURL()) + if err != nil { + t.Fatal(err) + } + t.Cleanup(nc.Close) + js, err := jetstream.New(nc) + if err != nil { + t.Fatal(err) + } + b := &testBus{nc: nc, js: js, kvs: map[string]jetstream.KeyValue{}} + // The controller's job: a bucket per state the manifest declares. + m, _ := readManifest(t) + for _, name := range m.State { + kv, err := js.CreateKeyValue(context.Background(), jetstream.KeyValueConfig{Bucket: m.Module + "_" + name, History: 1}) + if err != nil { + t.Fatal(err) + } + b.kvs[name] = kv + } + // The streams the events are on (mesh.mod..event., mesh.seat..event.). + if _, err := js.CreateStream(context.Background(), jetstream.StreamConfig{Name: "EVENTS", + Subjects: []string{"mesh.mod.gitea.event.>", "mesh.seat." + ControllerSeat + ".event.>", "mesh.seat." + asks.Seat + ".event.>"}}); err != nil { + t.Fatal(err) + } + return b +} + +// runtime is the half of the conversation the tool runner plays for one running bundle. +type runtime struct { + t *testing.T + b *testBus + cmd *exec.Cmd + in io.WriteCloser + mu sync.Mutex + next int + wait map[string]chan json.RawMessage + + // what the bundle subscribed to, and asked of the mesh. + subs []string + asked []askedOfMesh + notify chan struct{} + // the asks the operator was asked, and the controller's walks and the forge's open pull requests as the test plays + // them. + operatorAsks []asks.Ask + // operatorCancels are the ids of the asks taken back from the router, in order (issue 1079). + operatorCancels []string + openPulls []OpenPull +} + +type askedOfMesh struct { + Key string + Body map[string]any + At time.Time +} + +func (b *testBus) start(t *testing.T) *runtime { + t.Helper() + cmd := exec.Command(os.Args[0], "-test.run=^$") + cmd.Env = append(os.Environ(), bundleVar+"=1", "MESH_SERVED_MODULE=mesh-delivery", "MESH_MODULE=mesh-delivery", "MESH_NODE=test") + in, err := cmd.StdinPipe() + if err != nil { + t.Fatal(err) + } + out, err := cmd.StdoutPipe() + if err != nil { + t.Fatal(err) + } + cmd.Stderr = testLog{t} + if err := cmd.Start(); err != nil { + t.Fatal(err) + } + r := &runtime{t: t, b: b, cmd: cmd, in: in, wait: map[string]chan json.RawMessage{}, notify: make(chan struct{}, 64)} + go r.read(out) + t.Cleanup(func() { _ = in.Close(); _ = cmd.Wait() }) + return r +} + +type testLog struct{ t *testing.T } + +func (w testLog) Write(p []byte) (int, error) { + defer func() { _ = recover() }() // the bundle may speak after the test ended + w.t.Logf("bundle: %s", strings.TrimRight(string(p), "\n")) + return len(p), nil +} + +type rpcMessage struct { + ID json.RawMessage `json:"id,omitempty"` + Method string `json:"method,omitempty"` + Params json.RawMessage `json:"params,omitempty"` + Result json.RawMessage `json:"result,omitempty"` + Error *struct { + Code int `json:"code"` + Message string `json:"message"` + } `json:"error,omitempty"` +} + +func (r *runtime) say(m map[string]any) { + m["jsonrpc"] = "2.0" + raw, _ := json.Marshal(m) + r.mu.Lock() + defer r.mu.Unlock() + _, _ = r.in.Write(append(raw, '\n')) +} + +func (r *runtime) read(out io.Reader) { + sc := bufio.NewScanner(out) + sc.Buffer(make([]byte, 1<<20), 1<<24) + for sc.Scan() { + var m rpcMessage + if json.Unmarshal(sc.Bytes(), &m) != nil { + continue + } + if m.Method == "" { + var id string + _ = json.Unmarshal(m.ID, &id) + r.mu.Lock() + ch := r.wait[id] + delete(r.wait, id) + r.mu.Unlock() + if ch != nil { + ch <- sc.Bytes() + } + continue + } + go r.answer(m) + } +} + +func (r *runtime) ping() { + select { + case r.notify <- struct{}{}: + default: + } +} + +// asksOf are the asks of the mesh made so far under a key. +func (r *runtime) asksOf(key string) []askedOfMesh { + r.mu.Lock() + defer r.mu.Unlock() + var out []askedOfMesh + for _, a := range r.asked { + if a.Key == key { + out = append(out, a) + } + } + return out +} + +// until waits for what the test needs to be true, woken by every message of the bundle. +func (r *runtime) until(what string, within time.Duration, ok func() bool) { + r.t.Helper() + deadline := time.After(within) + for { + if ok() { + return + } + select { + case <-r.notify: + case <-time.After(50 * time.Millisecond): + case <-deadline: + if ok() { + return + } + r.t.Fatalf("within %s: %s did not happen", within, what) + } + } +} + +// toolAnswer is a tool's or a verb's answer as the runtime hands it over. +func toolAnswer(answer any) any { + return map[string]any{"ok": true, "output": "", "answer": answer} +} + +func (r *runtime) answer(m rpcMessage) { + var p struct { + State string `json:"state"` + Key string `json:"key"` + Value json.RawMessage `json:"value"` + Revision uint64 `json:"revision"` + Pattern string `json:"pattern"` + Subject string `json:"subject"` + Body json.RawMessage `json:"body"` + } + _ = json.Unmarshal(m.Params, &p) + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + defer r.ping() + result, err := func() (any, error) { + switch m.Method { + case "mesh/subscribe": + r.mu.Lock() + r.subs = append(r.subs, p.Pattern) + r.mu.Unlock() + return map[string]any{}, nil + case "mesh/publish": + return map[string]any{}, nil // the module's own events: not needed here + case "mesh/seat.publish": + // The ask for the operator, published under the module's name. + if strings.Contains(p.Subject, ".accept.ask.") { + var a asks.Ask + if err := json.Unmarshal(p.Body, &a); err != nil { + return nil, err + } + r.mu.Lock() + r.operatorAsks = append(r.operatorAsks, a) + r.mu.Unlock() + } + if strings.Contains(p.Subject, ".accept.cancel.") { + var c struct { + ID string `json:"id"` + } + if err := json.Unmarshal(p.Body, &c); err != nil { + return nil, err + } + r.mu.Lock() + r.operatorCancels = append(r.operatorCancels, c.ID) + r.mu.Unlock() + } + return map[string]any{}, nil + case "mesh/seat.record": + return nil, nil + case "mesh/ask": + var a struct { + Key string `json:"key"` + Body map[string]any `json:"body"` + } + if err := json.Unmarshal(m.Params, &a); err != nil { + return nil, err + } + r.mu.Lock() + r.asked = append(r.asked, askedOfMesh{Key: a.Key, Body: a.Body, At: time.Now()}) + pulls := append([]OpenPull{}, r.openPulls...) + r.mu.Unlock() + switch a.Key { + case "seat:" + ControllerSeat + ".delivery-walks": + return toolAnswer(map[string]any{"held": true, "walks": []any{}}), nil + case "seat:" + ControllerSeat + ".delivery-plan": + return toolAnswer(map[string]any{"plan": DeliveryPlan{Repository: "novox/app", Moved: []string{"app"}, + Summary: "builds app, sent to the node"}}), nil + case "seat:" + ControllerSeat + ".delivery-check": + return toolAnswer(map[string]any{"asked": "check-1"}), nil + case "gitea.gitea_open_pulls": + return toolAnswer(map[string]any{"pulls": pulls}), nil + case "gitea.gitea_commit_statuses": + // Green: both merge checks a success at the head asked, as the gitea module answers. + sha, _ := a.Body["sha"].(string) + if sha == "" { + sha = "0000000" + } + return toolAnswer(map[string]any{"commit": sha, "statuses": []any{ + map[string]any{"context": "mesh/merge-gate", "state": "success"}, + map[string]any{"context": "mesh/repo-check", "state": "success"}}, + "merge": map[string]any{"branch": "main", "protected": true, "required": []string{"mesh/merge-gate", "mesh/repo-check"}, + "blocking": []any{}, "mergeable": true, "says": "main requires mesh/merge-gate, mesh/repo-check"}}), nil + case "gitea.gitea_note_append", "gitea.gitea_delivery_view", "gitea.gitea_commit_status", "gitea.gitea_merge_pull_request", + "gitea.gitea_add_comment": + return toolAnswer(map[string]any{}), nil + } + return nil, fmt.Errorf("the runtime answers no ask %s in this test", a.Key) + } + kv := r.b.kvs[p.State] + if kv == nil { + return nil, fmt.Errorf("the state %q is not declared", p.State) + } + switch m.Method { + case "mesh/state.get": + e, err := kv.Get(ctx, p.Key) + if errors.Is(err, jetstream.ErrKeyNotFound) { + return nil, nil + } + if err != nil { + return nil, err + } + return map[string]any{"key": p.Key, "value": json.RawMessage(e.Value()), "revision": e.Revision()}, nil + case "mesh/state.create": + rev, err := kv.Create(ctx, p.Key, p.Value) + return map[string]any{"revision": rev}, err + case "mesh/state.update": + rev, err := kv.Update(ctx, p.Key, p.Value, p.Revision) + return map[string]any{"revision": rev}, err + case "mesh/state.put": + rev, err := kv.Put(ctx, p.Key, p.Value) + return map[string]any{"revision": rev}, err + case "mesh/state.delete": + return map[string]any{}, kv.Delete(ctx, p.Key) + case "mesh/state.keys": + keys, err := kv.Keys(ctx) + if errors.Is(err, jetstream.ErrNoKeysFound) { + return []string{}, nil + } + return keys, err + } + return nil, fmt.Errorf("the runtime offers no %s to this bundle", m.Method) + }() + if err != nil { + code := -32000 + var api *jetstream.APIError + if errors.Is(err, jetstream.ErrKeyExists) || errors.As(err, &api) && api.ErrorCode == jetstream.JSErrCodeStreamWrongLastSequence { + code = -32010 + } + r.say(map[string]any{"id": m.ID, "error": map[string]any{"code": code, "message": err.Error()}}) + return + } + r.say(map[string]any{"id": m.ID, "result": result}) +} + +// deliverEvents is the runtime reading the events streams and handing each event to the bundle, once, as +// `mesh/event`, the way the tool runner does for a module's consumer. +func (r *runtime) deliverEvents(ctx context.Context) { + stream, err := r.b.js.Stream(ctx, "EVENTS") + if err != nil { + r.t.Fatal(err) + } + cons, err := stream.OrderedConsumer(ctx, jetstream.OrderedConsumerConfig{}) + if err != nil { + r.t.Fatal(err) + } + cc, err := cons.Consume(func(msg jetstream.Msg) { + key := keyOfSubject(msg.Subject()) + r.mu.Lock() + id := fmt.Sprintf("event-%d", r.next) + r.next++ + r.mu.Unlock() + r.say(map[string]any{"id": id, "method": "mesh/event", "params": map[string]any{"envelope": stdio.Envelope{ + Key: key, Node: "test", Body: msg.Data(), Headers: map[string]string{"x-source": strings.SplitN(key, ".", 2)[0]}}}}) + }) + if err != nil { + r.t.Fatal(err) + } + r.t.Cleanup(cc.Stop) +} + +// keyOfSubject is an event's key, as the runtime gives it to a module: mesh.mod..event. is +// ., and mesh.seat..event. is .. +func keyOfSubject(subject string) string { + parts := strings.Split(subject, ".") + if len(parts) < 5 || parts[3] != "event" { + return subject + } + return parts[2] + "." + strings.Join(parts[4:], ".") +} + +func (b *testBus) publish(t *testing.T, subject string, body any) time.Time { + t.Helper() + raw, err := json.Marshal(body) + if err != nil { + t.Fatal(err) + } + if _, err := b.js.Publish(context.Background(), subject, raw); err != nil { + t.Fatal(err) + } + return time.Now() +} + +func TestAPassingVerdictAsksTheApproveAndItsApproveMergesWithinSecondsOverARealBus(t *testing.T) { + b := newTestBus(t) + rt := b.start(t) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + rt.deliverEvents(ctx) + // The module has started when it hears what it follows, and the operator's answers. + rt.until("the module to subscribe to the events it follows", 20*time.Second, func() bool { + rt.mu.Lock() + defer rt.mu.Unlock() + return len(rt.subs) >= len(followed)+1 + }) + for _, k := range followed { + if strings.Contains(k, "labeled") || strings.Contains(k, "commented") { + t.Fatalf("the module follows %s: a label or a comment starts nothing here (ADR 0421 §4)", k) + } + } + + // A pull request into main is announced, and the controller's verdict on its head passes. + const head = "1111111111111111111111111111111111111111" + announced := pr("novox/app", 7, head, "fix/x", "src/a.go") + announced.BaseSHA = "t1t1t1t1t1t1" + announced.Title = "Fix the apostrophe (issue 975)" + announced.Body = "The description, with no review line yet." + b.publish(t, "mesh.mod.gitea.event.pull.updated", announced) + passed := b.publish(t, "mesh.seat."+ControllerSeat+".event.checked", judgedAt("novox/app", 7, head, "pass", "t1t1t1t1t1t1")) + + // The operator is asked, within seconds of the verdict's event, with no label, comment or mark written by anyone. + rt.until("the operator to be asked to Approve the merge", 5*time.Second, func() bool { + rt.mu.Lock() + defer rt.mu.Unlock() + return len(rt.operatorAsks) == 1 + }) + rt.mu.Lock() + ask := rt.operatorAsks[0] + asked := time.Now() + rt.mu.Unlock() + if !strings.Contains(ask.Explanation, "not reviewed at this head") { + t.Fatalf("the first ask does not say the record is missing: %s", ask.Explanation) + } + t.Logf("the operator was asked %s after the verdict's event", asked.Sub(passed)) + // The description is edited to add the review record (issue 1079): the forge module's edit event carries it. The unanswered + // ask is withdrawn from the router and asked again, once, with the record read; nothing else is asked of the mesh. + first := ask + b.publish(t, "mesh.mod.gitea.event.forge.pull-request.edited", map[string]any{"owner": "novox", "repo": "app", "number": 7, + "title": announced.Title, "head_sha": head, "base": "main", "body": "Reviewed-by: lead@shanks at " + head}) + rt.until("the first ask to be withdrawn and the operator asked again", 5*time.Second, func() bool { + rt.mu.Lock() + defer rt.mu.Unlock() + return len(rt.operatorCancels) >= 1 && len(rt.operatorAsks) >= 2 + }) + rt.mu.Lock() + if len(rt.operatorCancels) != 1 || rt.operatorCancels[0] != first.ID || len(rt.operatorAsks) != 2 { + t.Fatalf("withdrawn %v, asked %d; wanted the first ask %s withdrawn once and exactly one new ask", rt.operatorCancels, len(rt.operatorAsks), first.ID) + } + ask = rt.operatorAsks[1] + rt.mu.Unlock() + if ask.ID == first.ID || !strings.Contains(ask.Explanation, "reviewed by lead on shanks") || strings.Contains(ask.Explanation, "not reviewed") { + t.Fatalf("the new ask is not a fresh one naming the reviewer: %s %s", ask.ID, ask.Explanation) + } + if len(rt.asksOf("gitea.gitea_merge_pull_request")) != 0 { + t.Fatalf("merged before the Approve") + } + // The pull request's key holds its head, the trunk commit it was asked against and the verdict. + rt.until("the pull request's key to hold head, trunk and verdict", 5*time.Second, func() bool { + e, err := b.kvs["pulls"].Get(context.Background(), "novox.app.7") + var k PullKey + return err == nil && json.Unmarshal(e.Value(), &k) == nil && k == PullKey{Head: head, Trunk: "t1t1t1t1t1t1", Verdict: "pass"} + }) + + // The operator approves (a person's Approve, proven by the router): the merge is asked within seconds. + opt, _ := ask.Option(OptionApprove) + approved := b.publish(t, "mesh.seat."+asks.Seat+".event.decided."+Module, asks.Warrant{Ask: ask.ID, Asker: Module, + Outcome: asks.OutcomeChosen, Option: OptionApprove, Label: opt.Label, Level: opt.Level, + By: &asks.Person{Who: asks.Operator, Kind: "telegram", Verified: "user id verified"}, At: time.Now(), AskDigest: ask.Digest()}) + rt.until("the merge to be asked of the forge", 5*time.Second, func() bool { return len(rt.asksOf("gitea.gitea_merge_pull_request")) >= 1 }) + merges := rt.asksOf("gitea.gitea_merge_pull_request") + if len(merges) != 1 || merges[0].Body["head"] != head || merges[0].Body["number"] != float64(7) { + t.Fatalf("the merge was asked %+v; wanted once, for pull request 7 at the approved head", merges) + } + if got := fmt.Sprint(merges[0].Body["approved_heads"]); got != "[novox/app@"+head+"]" { + t.Fatalf("the merge tool was handed the approved heads %s", got) + } + if took := merges[0].At.Sub(approved); took > 5*time.Second { + t.Fatalf("the merge was asked %s after the Approve's event; wanted within 5s", took) + } else { + t.Logf("the merge was asked %s after the Approve's event", took) + } + // Quiet follows: nothing asks the forge or the controller again with no event to say so. + quiet := len(rt.asked) + time.Sleep(1500 * time.Millisecond) + rt.mu.Lock() + more := len(rt.asked) - quiet + rt.mu.Unlock() + if more > 3 { + t.Fatalf("%d asks of the mesh in a quiet second and a half after the merge: something polls", more) + } +} + +// **The acceptance of one group, one Approve** (novox/hq ADR 0418 §4, ADR 0421, design 47 milestone 3, issues 978 and +// 1022): a group of two pull requests on one branch name, in two repositories, with green checks, gets one ask of the +// operator — no label, comment or mark written by anyone — and on the Approve both merge in one pass, each handed the +// approved heads of both. +func TestAGroupOfTwoWithGreenChecksGetsOneAskAndOnItsApproveBothMergeInOnePassOverARealBus(t *testing.T) { + b := newTestBus(t) + rt := b.start(t) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + rt.deliverEvents(ctx) + rt.until("the module to subscribe to the events it follows", 20*time.Second, func() bool { + rt.mu.Lock() + defer rt.mu.Unlock() + return len(rt.subs) >= len(followed)+1 + }) + const ctlHead, catHead = "c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1", "d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2" + for _, p := range []PullEvent{pr("novox/mesh-controller", 1, ctlHead, "feat/x", "internal/a.go"), + pr("novox/mesh-catalog", 2, catHead, "feat/x", "modules/app/module.json")} { + p.BaseSHA = "t1t1t1t1t1t1" + p.Body = "Reviewed-by: lead@shanks at " + p.HeadSHA + b.publish(t, "mesh.mod.gitea.event.pull.updated", p) + } + b.publish(t, "mesh.seat."+ControllerSeat+".event.checked", judgedAt("novox/mesh-controller", 1, ctlHead, "pass", "t1t1t1t1t1t1")) + b.publish(t, "mesh.seat."+ControllerSeat+".event.checked", judgedAt("novox/mesh-catalog", 2, catHead, "pass", "t1t1t1t1t1t1")) + + // Both members green: the composed check is asked, once, for both. + // Each member's own check is asked at once (issue 1115), before the group's: wait for the one carrying the group. + rt.until("the composed check to be asked", 10*time.Second, func() bool { + for _, c := range rt.asksOf("seat:" + ControllerSeat + ".delivery-check") { + if c.Body["group"] == "feat/x" { + return true + } + } + return false + }) + if n := len(rt.asksOf("seat:" + ControllerSeat + ".delivery-order")); n != 0 { + t.Fatalf("the controller's delivery-order was asked %d times: no order is asked for", n) + } + rt.mu.Lock() + nothing := len(rt.operatorAsks) + rt.mu.Unlock() + if nothing != 0 { + t.Fatalf("the operator was asked before the group's composed check passed") + } + // The composed check passes at both heads: the operator is asked once, for the group. + b.publish(t, "mesh.seat."+ControllerSeat+".event.checked", map[string]any{"group": "feat/x", "id": "check-1", "verdict": "pass", + "summary": "both compose", "members": []map[string]any{ + {"owner": "novox", "repo": "mesh-controller", "commit": ctlHead, "trunk": "t1t1t1t1t1t1"}, + {"owner": "novox", "repo": "mesh-catalog", "commit": catHead, "trunk": "t1t1t1t1t1t1"}}}) + rt.until("the operator to be asked once for the group", 10*time.Second, func() bool { + rt.mu.Lock() + defer rt.mu.Unlock() + return len(rt.operatorAsks) == 1 + }) + time.Sleep(500 * time.Millisecond) + rt.mu.Lock() + asked, ask := len(rt.operatorAsks), rt.operatorAsks[0] + rt.mu.Unlock() + if asked != 1 { + t.Fatalf("the operator was asked %d times for one group", asked) + } + if len(rt.asksOf("gitea.gitea_merge_pull_request")) != 0 { + t.Fatalf("merged before the Approve") + } + opt, _ := ask.Option(OptionApprove) + approved := b.publish(t, "mesh.seat."+asks.Seat+".event.decided."+Module, asks.Warrant{Ask: ask.ID, Asker: Module, + Outcome: asks.OutcomeChosen, Option: OptionApprove, Label: opt.Label, Level: opt.Level, + By: &asks.Person{Who: asks.Operator, Kind: "telegram", Verified: "user id verified"}, At: time.Now(), AskDigest: ask.Digest()}) + + // Both merge on that one Approve, in one pass: asked within seconds of the Approve's event, each with both heads. + rt.until("both merges to be asked of the forge", 5*time.Second, func() bool { return len(rt.asksOf("gitea.gitea_merge_pull_request")) >= 2 }) + merges := rt.asksOf("gitea.gitea_merge_pull_request") + if len(merges) != 2 { + t.Fatalf("the merge was asked %d times; wanted once for each member: %+v", len(merges), merges) + } + heads := map[string]bool{} + for _, m := range merges { + heads[fmt.Sprint(m.Body["repo"], "@", m.Body["head"])] = true + if got := fmt.Sprint(m.Body["approved_heads"]); got != "[novox/mesh-catalog@"+catHead+" novox/mesh-controller@"+ctlHead+"]" { + t.Fatalf("a merge was handed the approved heads %s", got) + } + if took := m.At.Sub(approved); took > 5*time.Second { + t.Fatalf("a merge was asked %s after the Approve's event; wanted within 5s", took) + } + } + if !heads["mesh-controller@"+ctlHead] || !heads["mesh-catalog@"+catHead] { + t.Fatalf("the merges were asked for %v", heads) + } + rt.mu.Lock() + again := len(rt.operatorAsks) + rt.mu.Unlock() + if again != 1 { + t.Fatalf("a second ask was made: %d", again) + } +} + +// **The acceptance of a failed backup before the send, on the mesh-delivery side** (novox/hq ADR 0404 clause 4, design 47 +// milestone 4, issue 1068): a merged delivery whose walk runs is held when the walk's record, as the controller's +// `plan-moved` carries it, names a backup that failed and nothing was sent; the hold names the machine, the module and the +// backup's words, and nothing asks the controller to let it go. One test function, one bus. The walk's backups field is the controller's +// contract (issue 1065). +func TestAPlanMovedCarryingAFailedBackupHoldsTheDeliveryWithTheBackupsWordsOverARealBus(t *testing.T) { + b := newTestBus(t) + rt := b.start(t) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + rt.deliverEvents(ctx) + rt.until("the module to subscribe to the events it follows", 20*time.Second, func() bool { + rt.mu.Lock() + defer rt.mu.Unlock() + return len(rt.subs) >= len(followed)+1 + }) + const head, merge = "3333333333333333333333333333333333333333", "4444444444444444444444444444444444444444" + announced := pr("novox/app", 9, head, "fix/y", "src/a.go") + announced.BaseSHA = "t1t1t1t1t1t1" + deliveryOf := func() Delivery { + e, err := b.kvs["deliveries"].Get(context.Background(), kvKey(IDOf("novox/app", head))) + var d Delivery + if err == nil { + _ = json.Unmarshal(e.Value(), &d) + } + return d + } + // Each event is published after the module has taken the one before it: they are on two subjects, and the runner + // may hand over a later one first, which the table would rightly judge on its own (merged before checked). + b.publish(t, "mesh.mod.gitea.event.pull.updated", announced) + rt.until("the pull request to be taken", 10*time.Second, func() bool { return deliveryOf().State != "" }) + b.publish(t, "mesh.seat."+ControllerSeat+".event.checked", judgedAt("novox/app", 9, head, "pass", "t1t1t1t1t1t1")) + rt.until("the verdict to be taken", 10*time.Second, func() bool { d := deliveryOf(); return d.State != "" && d.State != Proposed }) + b.publish(t, "mesh.mod.gitea.event.pull.merged", merged("novox/app", 9, head, merge)) + rt.until("the merge to be taken", 10*time.Second, func() bool { return deliveryOf().MergedAs != "" }) + + // Its walk runs, let go: the record names no failed backup yet. + now := time.Now() + walk := aWalk("plan-1", "novox/app", merge, true) + walk.Created, walk.State, walk.Revision = now, walkRolling, 1 + walk.Delivery.Go, walk.Delivery.By = &now, "mesh-delivery" + b.publish(t, "mesh.seat."+ControllerSeat+".event.plan-moved", walk) + rt.until("the delivery to be delivering", 10*time.Second, func() bool { return deliveryOf().State == Delivering }) + + // A backup failed before the send: the delivery is held with the backup's words. + walk.Revision, walk.Backups = 2, &WalkBackups{State: "failed", Asked: []WalkBackupAsked{{Machine: "ace", Module: "pg", State: "failed", Words: "no space left on device"}}} + b.publish(t, "mesh.seat."+ControllerSeat+".event.plan-moved", walk) + rt.until("the delivery to be held for the failed backup", 10*time.Second, func() bool { return deliveryOf().State == Held }) + d := deliveryOf() + for _, want := range []string{"ace", "pg", "no space left on device"} { + if !strings.Contains(d.HeldWhy, want) { + t.Fatalf("the held reason %q does not name %q", d.HeldWhy, want) + } + } + // It stays held, and the controller is asked nothing to let it go. + time.Sleep(500 * time.Millisecond) + if s := deliveryOf().State; s != Held { + t.Fatalf("the hold lifted by itself: %s", s) + } + if n := len(rt.asksOf("seat:"+ControllerSeat+".deliver")) + len(rt.asksOf("seat:"+ControllerSeat+".delivery-backups-again")); n != 0 { + t.Fatalf("the controller was asked %d times to let the held walk go or take its backups again", n) + } +} diff --git a/modules/mesh-delivery/cmd/mesh-delivery/checkask_test.go b/modules/mesh-delivery/cmd/mesh-delivery/checkask_test.go new file mode 100644 index 00000000..506035c8 --- /dev/null +++ b/modules/mesh-delivery/cmd/mesh-delivery/checkask_test.go @@ -0,0 +1,163 @@ +package main + +import ( + "strings" + "testing" + "time" +) + +// A delivery that waits for its check gets one (novox/hq issue 290): a head adopted while it already waited — +// its announcement heard again from the bus's history, which the controller took long ago — has its check asked +// by this owner; so does one whose announcement the controller missed. Once asked, the proposed bound means +// "asked and not answered", and healer H2 may ask it once more before it is the operator's. + +// The live case: a pull request opened before merge checks existed, announced again at this owner's first +// start, read back proposed after a restart. Its check is asked at the first tick, once, for that head alone. +func TestAProposedDeliveryAdoptedWithNoVerdictHasItsCheckAsked(t *testing.T) { + w := newWorld(t) + id := IDOf("novox/hq", head) + w.h.PullUpdated(pr("novox/hq", 94, head, "issues/245-246", "04-ISSUES/245/00-report.md")) + w.settleAll() + if len(w.ctl.checks) != 1 { + t.Fatalf("a head just announced was not asked at once: %v", w.ctl.checks) + } + + // A restart, a minute later: read back proposed with its ask kept; it is not asked again. + again := newWorldOver(t, w.store, w.ctl, w.forge, w.now.Add(time.Minute)) + if s := again.state(id); s != Proposed { + t.Fatalf("read back, it is %s", s) + } + again.settleAll() + if len(w.ctl.checks) != 1 || w.ctl.checks[0] != ":"+id { + t.Fatalf("its check was asked %v, wanted once for its head alone", w.ctl.checks) + } + d := again.delivery(id) + if d.CheckAsk == nil || d.CheckAsk.ID != "check-1" || d.CheckAsk.By != byOwner || d.State != Proposed { + t.Fatalf("the ask is not kept: %+v (%s)", d.CheckAsk, d.State) + } + if notes := w.forge.notesOn(head); len(notes) == 0 || !strings.Contains(notes[len(notes)-1], "its check asked") { + t.Fatalf("the ask is not noted on the head: %q", notes) + } + // Asked, it is not stalled: the bound runs from the ask. + if st := again.h.Stalled(); len(st) != 0 { + t.Fatalf("a check just asked is stalled: %+v", st) + } + // Asked once: the ticks after do not ask again. + again.later(30 * time.Minute) + again.settleAll() + if len(w.ctl.checks) != 1 { + t.Fatalf("a check asked and in flight was asked again: %v", w.ctl.checks) + } + // The verdict arrives: checked, then ready, by itself. + again.h.Checked(verdict("novox/hq", 94, head, "pass")) + if s := again.state(id); s != Ready { + t.Fatalf("its verdict left it %s", s) + } +} + +// The same head adopted at a first start with no restart: its check is asked at once, with no grace (issue 1115). +func TestAMissedAnnouncementHasItsCheckAskedAtOnce(t *testing.T) { + w := newWorld(t) + id := IDOf("novox/app", head) + w.h.PullUpdated(pr("novox/app", 7, head, "feat/x", "src/a.go")) + // The controller does not answer: nothing is kept as asked, and it is tried again a minute on. + w.ctl.down = true + w.settleAll() + if d := w.delivery(id); d.CheckAsk != nil { + t.Fatalf("an ask the controller did not take is kept as asked: %+v", d.CheckAsk) + } + w.ctl.down = false + w.settleAll() + if len(w.ctl.checks) != 0 { + t.Fatalf("asked again within the minute: %v", w.ctl.checks) + } + w.later(time.Minute) + w.settleAll() + if len(w.ctl.checks) != 1 || w.delivery(id).CheckAsk == nil { + t.Fatalf("a missed announcement was not asked: %v", w.ctl.checks) + } + // A verdict that arrived in the meantime, or a delivery past its proposal, is never asked. + other := IDOf("novox/app", "bbbbbbbbbbbb2222") + w.h.PullUpdated(pr("novox/app", 8, "bbbbbbbbbbbb2222", "feat/y")) + w.h.Checked(verdict("novox/app", 8, "bbbbbbbbbbbb2222", "fail")) + w.later(time.Hour) + w.settleAll() + if w.state(other) != Rejected || len(w.ctl.checks) != 1 { + t.Fatalf("a head with a verdict was asked: %s %v", w.state(other), w.ctl.checks) + } +} + +// An announcement heard again carrying its head's decided verdict takes it; a pending one does not. +func TestAnAnnouncementCarryingItsHeadsVerdictTakesIt(t *testing.T) { + w := newWorld(t) + p := pr("novox/app", 7, head, "feat/x", "src/a.go") + p.HeadChecks = map[string]string{"mesh/merge-gate": "success", "mesh/repo-check": "success"} + w.h.PullUpdated(p) + if s := w.state(IDOf("novox/app", head)); s != Ready { + t.Fatalf("a head whose gate passed on the forge is %s", s) + } + q := pr("novox/app", 8, "bbbbbbbbbbbb2222", "feat/y") + q.HeadChecks = map[string]string{"mesh/merge-gate": "pending"} + w.h.PullUpdated(q) + if s := w.state(IDOf("novox/app", "bbbbbbbbbbbb2222")); s != Proposed { + t.Fatalf("a pending gate on the forge left it %s", s) + } +} + +// Asked and not answered past the bound: stalled, and H2's close re-asks once — the second close is refused +// by the table, and the delivery is the operator's. +func TestH2ReasksAStalledCheckOnceThenItIsTheOperators(t *testing.T) { + w := newWorld(t) + id := IDOf("novox/app", head) + w.h.PullUpdated(pr("novox/app", 7, head, "feat/x", "src/a.go")) + w.settleAll() + if len(w.ctl.checks) != 1 { + t.Fatalf("asked %v", w.ctl.checks) + } + if _, err := w.h.Close(id, "too early"); err == nil { + t.Fatal("re-asked within the bound") + } + w.later(Bounds[Proposed].For + time.Minute) + st := w.h.Stalled() + if len(st) != 1 || st[0].ID != id || !strings.Contains(st[0].H2, "re-ask") { + t.Fatalf("stalled %+v", st) + } + // The controller does not take the re-ask: nothing is spent. + w.ctl.down = true + if _, err := w.h.Close(id, "delivery stalled"); err == nil { + t.Fatal("a re-ask the controller did not take was said done") + } + if d := w.delivery(id); d.CheckAsk.Reasked != 0 { + t.Fatal("a re-ask that was not made was spent") + } + w.ctl.down = false + said, err := w.h.Close(id, "delivery stalled") + if err != nil || !strings.Contains(said, "re-asked") { + t.Fatalf("H2's close: %q %v", said, err) + } + d := w.delivery(id) + if len(w.ctl.checks) != 2 || d.State != Proposed || d.CheckAsk.Reasked != 1 || d.CheckAsk.By != byH2 || + d.Transitions[len(d.Transitions)-1].Event != EvReask { + t.Fatalf("after the re-ask: %v %s %+v", w.ctl.checks, d.State, d.CheckAsk) + } + if len(w.h.Stalled()) != 0 { + t.Fatal("a check just re-asked is stalled") + } + // Still not answered: stalled again, and the operator's. + w.later(Bounds[Proposed].For + time.Minute) + st = w.h.Stalled() + if len(st) != 1 || !strings.Contains(st[0].H2, "operator's") { + t.Fatalf("stalled after the re-ask %+v", st) + } + if _, err := w.h.Close(id, "delivery stalled"); err == nil || !strings.Contains(err.Error(), "once already") { + t.Fatalf("a second re-ask: %v", err) + } + if len(w.ctl.checks) != 2 { + t.Fatalf("a second re-ask reached the controller: %v", w.ctl.checks) + } + // The verdict, late, still settles it. + w.h.Checked(verdict("novox/app", 7, head, "pass")) + if s := w.state(id); s != Ready { + t.Fatalf("a late verdict left it %s", s) + } +} diff --git a/modules/mesh-delivery/cmd/mesh-delivery/checks.go b/modules/mesh-delivery/cmd/mesh-delivery/checks.go new file mode 100644 index 00000000..a631a5dd --- /dev/null +++ b/modules/mesh-delivery/cmd/mesh-delivery/checks.go @@ -0,0 +1,297 @@ +package main + +import ( + "errors" + "fmt" + "sort" + "strings" + "time" +) + +// The `checks` verb (novox/hq ADR 0239): what the mesh's checks said of a pull request's head, or of one +// commit. Two records answer it, and neither alone is enough. The forge keeps each status — the newest of +// each context, a description clipped to what it shows, and the protection that decides whether the merge +// button works. This owner keeps the verdict whole as the controller said it: each layer's summary +// unclipped ("0 of 4 compose" is not lost to a 140-character description), the build seat's ask that ran +// it, the machine it ran on, and its report. So the forge is asked for its statuses and this owner joins +// its own verdict to the two statuses that verdict set. Nothing is written: a read, by anyone. + +// Context names of the statuses this owner can say more of than the forge keeps. +const ( + ctxGate = "mesh/merge-gate" + ctxRepo = "mesh/repo-check" + ctxDelivery = "mesh/delivery" + ctxGroup = "mesh/delivery-group" +) + +// CheckStatus is one status of the commit, with what this owner knows of it beyond the forge. +type CheckStatus struct { + Name string `json:"name"` + State string `json:"state"` + Description string `json:"description,omitempty"` + SetAt string `json:"set_at,omitempty"` + SetBy string `json:"set_by,omitempty"` + Link string `json:"link,omitempty"` + // From the verdict that set it, when this owner holds it: the build seat's ask, the machine that ran it, + // when it was heard, and the layer's verdict in full. + Build string `json:"build,omitempty"` + CheckedOn string `json:"checked_on,omitempty"` + CheckedAt string `json:"checked_at,omitempty"` + Verdict string `json:"verdict,omitempty"` + // Source says where the status was read: the forge, or — the forge unreachable — this owner's verdict. + Source string `json:"source"` +} + +// ChecksAnswer is the `checks` verb's answer. +type ChecksAnswer struct { + Repository string `json:"repository"` + Commit string `json:"commit"` + Pull *ForgePull `json:"pull,omitempty"` + Delivery *ChecksOf `json:"delivery,omitempty"` + Statuses []CheckStatus `json:"statuses"` + // Verdict is the merge check's verdict whole, as this owner heard it for this commit. + Verdict *Verdict `json:"verdict,omitempty"` + // Judged is what the verdict judged: the head merged into the trunk commit (novox/hq ADR 0323 §3); OutOfDate + // why it no longer counts, when its trunk has moved since. + Judged string `json:"judged,omitempty"` + OutOfDate string `json:"out_of_date,omitempty"` + // Merge is whether the branch's protection lets the commit merge; Mergeable nil when it is not known. + Merge ForgeMerge `json:"merge"` + // Forge is why the forge could not be read, when it could not. + Forge string `json:"forge,omitempty"` + Says string `json:"says"` +} + +// ChecksOf is the delivery of the commit, in brief: `show` has it whole. +type ChecksOf struct { + ID string `json:"id"` + State State `json:"state"` + Since string `json:"since"` + Waits string `json:"waits,omitempty"` + Group string `json:"group,omitempty"` + // Class, Opened and MergedAt are what the issue tracker measures a fix's time by (novox/hq ADR 0313): the + // class of what it moves, when its pull request was first heard, and when it merged. + Class string `json:"class,omitempty"` + Opened string `json:"opened,omitempty"` + MergedAt string `json:"merged_at,omitempty"` +} + +// Checks is the `checks` verb: a repository and a pull request's number, or a repository and a commit. +func (h *Holder) Checks(repository string, number int, commit string) (*ChecksAnswer, error) { + repository = strings.ToLower(strings.TrimSuffix(strings.TrimSpace(repository), ".git")) + owner, repo, ok := strings.Cut(repository, "/") + if !ok || owner == "" || repo == "" || strings.Contains(repo, "/") { + return nil, fmt.Errorf("%q is not owner/repository", repository) + } + commit = strings.ToLower(strings.TrimSpace(commit)) + if number < 0 { + return nil, fmt.Errorf("%d is not a pull request's number", number) + } + if number == 0 && commit == "" { + return nil, errors.New("name a pull request's number or a commit") + } + if commit != "" && (len(commit) < 7 || strings.Trim(commit, "0123456789abcdef") != "") { + return nil, fmt.Errorf("%q is not a commit: seven or more hexadecimal characters", commit) + } + + // What this owner knows first, for the base whose protection is read; the forge then says the head. + base := "" + if d := h.deliveryOf(repository, number, commit); d != nil { + base = d.Base + } + a := &ChecksAnswer{Repository: repository, Commit: commit} + fc, err := h.Forge.Statuses(owner, repo, number, commit, base) + if err != nil { + a.Forge = err.Error() + } else { + a.Commit, a.Pull = fc.Commit, fc.Pull + } + + d := h.deliveryOf(repository, number, a.Commit) + if a.Commit == "" && d != nil { + a.Commit = d.Commit // the forge away: the newest head this owner holds for the pull request + } + if d != nil { + a.Delivery = &ChecksOf{ID: d.ID, State: d.State, Since: d.Since.UTC().Format(time.RFC3339), Waits: d.waits, + Group: d.Group, Class: fixClass(&d.Delivery)} + if !d.opened.IsZero() { + a.Delivery.Opened = d.opened.UTC().Format(time.RFC3339) + } + if !d.MergedAt.IsZero() { + a.Delivery.MergedAt = d.MergedAt.UTC().Format(time.RFC3339) + } + if d.Check != nil && sameCommit(d.Commit, a.Commit) { + a.Verdict = d.Check + a.Judged, a.OutOfDate = d.Check.Judged(), d.outOfDate() + } + } + + if fc != nil { + for _, s := range fc.Statuses { + a.Statuses = append(a.Statuses, CheckStatus{Name: s.Context, State: s.State, Description: s.Description, + SetAt: firstOf(s.UpdatedAt, s.CreatedAt), SetBy: s.Creator, Link: s.TargetURL, Source: "the forge"}) + } + a.Merge = fc.Merge + } else { + // The forge away: the statuses the verdict set, as the forge holder sets them from it, said as such. + if v := a.Verdict; v != nil { + a.Statuses = append(a.Statuses, CheckStatus{Name: ctxGate, State: forgeStateOf(v.Gate), Source: "mesh-delivery's verdict; the forge could not be read"}) + if v.Repo != "" { + a.Statuses = append(a.Statuses, CheckStatus{Name: ctxRepo, State: forgeStateOf(v.Repo), Source: "mesh-delivery's verdict; the forge could not be read"}) + } + } + a.Merge = ForgeMerge{Branch: firstOf(base, "main"), Says: "not known: the forge could not be read, and only it holds the protection"} + } + for i := range a.Statuses { + h.addWhatIsKnown(&a.Statuses[i], a.Verdict, d) + } + sort.SliceStable(a.Statuses, func(i, j int) bool { return a.Statuses[i].Name < a.Statuses[j].Name }) + a.Says = a.says() + return a, nil +} + +// checksDelivery is the part of a delivery `checks` reads, copied under the lock. +type checksDelivery struct { + Delivery + waits string + groupSaid string + opened time.Time +} + +// deliveryOf is the delivery of a commit — its head, or the commit it merged as — else, for a pull request, +// its newest head that was not superseded, else its newest. +func (h *Holder) deliveryOf(repository string, number int, commit string) *checksDelivery { + h.mu.Lock() + defer h.mu.Unlock() + var found *Delivery + if commit != "" { + for _, d := range h.deliveries { + if strings.EqualFold(d.Repository, repository) && (sameCommit(d.Commit, commit) || sameCommit(d.MergedAs, commit)) { + found = d + break + } + } + } else if number > 0 { + for _, d := range h.deliveries { + if !strings.EqualFold(d.Repository, repository) || d.Number != number { + continue + } + if found == nil || newerHead(d, found) { + found = d + } + } + } + if found == nil { + return nil + } + c := &checksDelivery{Delivery: *found, waits: h.waitsFor(found), opened: h.openedOf(found)} + if found.Check != nil { + v := *found.Check + c.Check = &v + } + if g := h.groups[found.Group]; g != nil { + state, why := GroupState(g, h.membersOf(g)) + c.groupSaid = state + ": " + why + if g.Check != nil && g.Check.Verdict != "" { + c.groupSaid += "; composed check " + g.Check.Verdict + " — " + g.Check.Summary + } + } + return c +} + +// newerHead is whether d is a pull request's head to read before e: one not superseded first, then the newest. +func newerHead(d, e *Delivery) bool { + if (d.State == Superseded) != (e.State == Superseded) { + return e.State == Superseded + } + return d.Created.After(e.Created) +} + +// addWhatIsKnown joins this owner's verdict to the status it set, and the delivery to its own statuses. +func (h *Holder) addWhatIsKnown(s *CheckStatus, v *Verdict, d *checksDelivery) { + switch s.Name { + case ctxGate, ctxRepo: + if v == nil { + return + } + s.Build, s.CheckedOn = v.ID, v.On + if !v.At.IsZero() { + s.CheckedAt = v.At.UTC().Format(time.RFC3339) + } + if s.Name == ctxGate { + s.Verdict = strings.ToUpper(firstOf(v.Gate, "error")) + " — " + v.Summary + if len(v.Modules) > 0 { + s.Verdict += " (modules: " + strings.Join(v.Modules, ", ") + if len(v.Dependents) > 0 { + s.Verdict += "; built after them: " + strings.Join(v.Dependents, ", ") + } + s.Verdict += ")" + } + } else if v.Repo != "" { + s.Verdict = strings.ToUpper(v.Repo) + " — " + v.RepoSaid + } + case ctxDelivery: + if d != nil { + s.Verdict = string(d.State) + if d.waits != "" { + s.Verdict += ": waits for " + d.waits + } + } + case ctxGroup: + if d != nil && d.groupSaid != "" { + s.Verdict = d.groupSaid + } + } +} + +// forgeStateOf is the forge state the forge's holder sets for a verdict (the gitea module's pulls.ts): a +// warning is a pass with a note, an error never a success. +func forgeStateOf(verdict string) string { + switch verdict { + case "pass", "warning": + return "success" + case "fail": + return "failure" + } + return "error" +} + +func (a *ChecksAnswer) says() string { + what := a.Repository + "@" + shortOf(a.Commit, 8) + if a.Pull != nil { + what = fmt.Sprintf("%s#%d at %s", a.Repository, a.Pull.Number, shortOf(a.Commit, 8)) + } + var parts []string + for _, s := range a.Statuses { + parts = append(parts, s.Name+" "+s.State) + } + statuses := "no status" + if len(parts) > 0 { + statuses = strings.Join(parts, ", ") + } + merge := a.Merge.Says + switch { + case a.Merge.Mergeable == nil: + merge = "mergeable: not known — " + merge + case *a.Merge.Mergeable: + merge = "mergeable — " + merge + default: + merge = "NOT mergeable — " + merge + } + if a.Judged != "" { + what += " (" + a.Judged + ")" + } + if a.OutOfDate != "" { + merge += "; its verdict is " + a.OutOfDate + } + return what + ": " + statuses + "; " + merge +} + +func firstOf(s ...string) string { + for _, x := range s { + if x != "" { + return x + } + } + return "" +} diff --git a/modules/mesh-delivery/cmd/mesh-delivery/checks_test.go b/modules/mesh-delivery/cmd/mesh-delivery/checks_test.go new file mode 100644 index 00000000..b2565ef2 --- /dev/null +++ b/modules/mesh-delivery/cmd/mesh-delivery/checks_test.go @@ -0,0 +1,242 @@ +package main + +import ( + "encoding/json" + "strings" + "testing" +) + +// The `checks` verb (novox/hq ADR 0239): what the mesh's checks said of a pull request's head or a commit — +// the forge's statuses, the verdict whole as the controller said it, and whether the base's protection lets +// it merge. + +// checkedAs is the controller's `checked` as it comes off the bus, both layers said. +func checkedAs(t *testing.T, repo string, number int, sha, gate, gateSaid, repoVerdict, repoSaid string) CheckedEvent { + t.Helper() + owner, name, _ := strings.Cut(repo, "/") + raw := map[string]any{"owner": owner, "repo": name, "number": number, "commit": sha, "verdict": gate, + "summary": gateSaid, "id": "build-" + sha[:8], "on": "the-build-node", "report": "== compose\n" + gateSaid, + "gate": map[string]any{"verdict": gate, "summary": gateSaid, "modules": []string{"app"}, "dependents": []string{"web"}}, + "repo-check": map[string]any{"verdict": repoVerdict, "summary": repoSaid}} + var c CheckedEvent + if err := json.Unmarshal(mustJSON(raw), &c); err != nil { + t.Fatal(err) + } + return c +} + +// statusesAsTheForgeSets puts the two statuses the forge's holder sets from a verdict: clipped, as it keeps them. +func statusesAsTheForgeSets(w *world, repo, sha string, c CheckedEvent) { + owner, name, _ := strings.Cut(repo, "/") + _ = w.forge.Status(owner, name, sha, ctxGate, forgeStateOf(c.Gate.Verdict), clip(c.Gate.Verdict+": "+c.Gate.Summary, 40), "https://forge.invalid/pr") + _ = w.forge.Status(owner, name, sha, ctxRepo, forgeStateOf(c.RepoCheck.Verdict), clip(c.RepoCheck.Verdict+": "+c.RepoCheck.Summary, 40), "https://forge.invalid/pr") +} + +func statusNamed(a *ChecksAnswer, name string) *CheckStatus { + for i := range a.Statuses { + if a.Statuses[i].Name == name { + return &a.Statuses[i] + } + } + return nil +} + +func TestChecksOfAPullRequestSayEveryStatusTheVerdictWholeAndWhetherItMerges(t *testing.T) { + w := newWorld(t) + w.forge.required = []string{ctxGate, ctxRepo} + w.forge.pulls["novox/app#7"] = ForgePull{Number: 7, Title: "a change", State: "open", Base: "main", Head: "feat/x", HeadSHA: head} + w.h.PullUpdated(pr("novox/app", 7, head, "feat/x", "src/a.go")) + said := "0 of 4 compose: anchor refuses app's manifest — a provision nothing provides, and three more machines alike" + c := checkedAs(t, "novox/app", 7, head, "fail", said, "pass", "its merge-check.sh passed") + w.h.Checked(c) + w.settleAll() + statusesAsTheForgeSets(w, "novox/app", head, c) + + a, err := w.h.Checks("novox/app", 7, "") + if err != nil { + t.Fatal(err) + } + if a.Commit != head || a.Pull == nil || a.Pull.Number != 7 || a.Forge != "" { + t.Fatalf("the head read is %q, pull %+v, forge %q", a.Commit, a.Pull, a.Forge) + } + gate := statusNamed(a, ctxGate) + if gate == nil || gate.State != "failure" || gate.Source != "the forge" || gate.SetBy != "mesh-admin" { + t.Fatalf("the gate's status is %+v", gate) + } + // The forge clipped it; the verdict says it whole, with the machine, the build and the modules. + if strings.Contains(gate.Description, "three more machines") { + t.Fatalf("the fake forge did not clip: %q", gate.Description) + } + if !strings.Contains(gate.Verdict, "FAIL — 0 of 4 compose") || !strings.Contains(gate.Verdict, "three more machines") || + !strings.Contains(gate.Verdict, "modules: app; built after them: web") { + t.Fatalf("the gate's verdict is %q", gate.Verdict) + } + if gate.Build != "build-"+head[:8] || gate.CheckedOn != "the-build-node" || gate.CheckedAt == "" { + t.Fatalf("the gate was checked as %q on %q at %q", gate.Build, gate.CheckedOn, gate.CheckedAt) + } + repo := statusNamed(a, ctxRepo) + if repo == nil || repo.State != "success" || repo.Verdict != "PASS — its merge-check.sh passed" || repo.CheckedOn != "the-build-node" { + t.Fatalf("the repository check's status is %+v", repo) + } + if d := statusNamed(a, ctxDelivery); d == nil || !strings.HasPrefix(d.Verdict, "rejected: waits for a new head") { + t.Fatalf("the delivery's status is %+v", d) + } + if a.Verdict == nil || !strings.Contains(a.Verdict.Report, "== compose") { + t.Fatalf("the verdict whole is %+v", a.Verdict) + } + if a.Delivery == nil || a.Delivery.ID != IDOf("novox/app", head) || a.Delivery.State != Rejected { + t.Fatalf("the delivery is %+v", a.Delivery) + } + if a.Merge.Mergeable == nil || *a.Merge.Mergeable || len(a.Merge.Blocking) != 1 || a.Merge.Blocking[0].Context != ctxGate { + t.Fatalf("the merge is %+v", a.Merge) + } + if !strings.Contains(a.Says, "novox/app#7 at aaaaaaaa") || !strings.Contains(a.Says, "NOT mergeable") { + t.Fatalf("it says %q", a.Says) + } + // The forge was asked for the pull request, with the base this owner knew. + if len(w.forge.asked) != 1 || w.forge.asked[0] != "novox/app#7@ base=main" { + t.Fatalf("the forge was asked %v", w.forge.asked) + } +} + +func TestANewHeadIsReadWithItsOwnVerdictNeverTheOlderOnes(t *testing.T) { + w := newWorld(t) + w.forge.required = []string{ctxGate, ctxRepo} + newer := "bbbbbbbbbbbb2222" + w.h.PullUpdated(pr("novox/app", 7, head, "feat/x")) + w.h.Checked(checkedAs(t, "novox/app", 7, head, "fail", "0 of 4 compose", "pass", "passed")) + w.h.PullUpdated(pr("novox/app", 7, newer, "feat/x")) + w.forge.pulls["novox/app#7"] = ForgePull{Number: 7, State: "open", Base: "main", HeadSHA: newer} + + a, err := w.h.Checks("novox/app", 7, "") + if err != nil { + t.Fatal(err) + } + if a.Commit != newer || a.Delivery == nil || a.Delivery.ID != IDOf("novox/app", newer) || a.Delivery.State != Proposed { + t.Fatalf("the new head reads as %q, %+v", a.Commit, a.Delivery) + } + if a.Verdict != nil { + t.Fatalf("the older head's verdict was given for the newer: %+v", a.Verdict) + } + if a.Merge.Mergeable == nil || *a.Merge.Mergeable || len(a.Merge.Blocking) != 2 || a.Merge.Blocking[0].State != "missing" { + t.Fatalf("a head not checked yet merges as %+v", a.Merge) + } + + // Passed, it merges. + c := checkedAs(t, "novox/app", 7, newer, "pass", "every machine composes", "pass", "its merge-check.sh passed") + w.h.Checked(c) + statusesAsTheForgeSets(w, "novox/app", newer, c) + a, _ = w.h.Checks("novox/app", 7, "") + if a.Merge.Mergeable == nil || !*a.Merge.Mergeable || !strings.Contains(a.Says, "mergeable") || strings.Contains(a.Says, "NOT") { + t.Fatalf("a head that passed: %+v, says %q", a.Merge, a.Says) + } + if g := statusNamed(a, ctxGate); g == nil || g.Verdict != "PASS — every machine composes (modules: app; built after them: web)" { + t.Fatalf("its gate: %+v", g) + } +} + +func TestACommitIsReadByTheStartOfItsShaAndAMergeByItsDelivery(t *testing.T) { + w := newWorld(t) + w.h.PullUpdated(pr("novox/app", 7, head, "feat/x")) + c := checkedAs(t, "novox/app", 7, head, "pass", "every machine composes", "pass", "passed") + w.h.Checked(c) + statusesAsTheForgeSets(w, "novox/app", head, c) + + a, err := w.h.Checks("Novox/App.git", 0, head[:8]) + if err != nil { + t.Fatal(err) + } + if a.Commit != head || a.Verdict == nil || a.Delivery == nil || a.Pull != nil { + t.Fatalf("by the start of its sha: %q, %+v, %+v", a.Commit, a.Verdict, a.Delivery) + } + // No protection: nothing is required, so it merges. + if a.Merge.Mergeable == nil || !*a.Merge.Mergeable { + t.Fatalf("with no protection: %+v", a.Merge) + } + + mergedAs := "dddddddddddd9999" + w.h.PullMerged(merged("novox/app", 7, head, mergedAs)) + _ = w.forge.Status("novox", "app", mergedAs, ctxDelivery, "pending", "published", "") + a, err = w.h.Checks("novox/app", 0, mergedAs) + if err != nil { + t.Fatal(err) + } + if a.Delivery == nil || a.Delivery.ID != IDOf("novox/app", head) { + t.Fatalf("the merge commit's delivery is %+v", a.Delivery) + } + if a.Verdict != nil { + t.Fatalf("the head's verdict was given for the merge commit: %+v", a.Verdict) + } + if s := statusNamed(a, ctxDelivery); s == nil || s.State != "pending" || s.Verdict == "" { + t.Fatalf("the merge's delivery status: %+v", s) + } +} + +func TestWithTheForgeAwayTheVerdictIsSaidAndWhetherItMergesIsNotKnown(t *testing.T) { + w := newWorld(t) + w.h.PullUpdated(pr("novox/app", 7, head, "feat/x")) + w.h.Checked(checkedAs(t, "novox/app", 7, head, "warning", "every machine composes, with a note", "error", "the toolchain was not there")) + w.forge.down = true + + a, err := w.h.Checks("novox/app", 7, "") + if err != nil { + t.Fatal(err) + } + if a.Forge == "" || a.Commit != head || a.Verdict == nil { + t.Fatalf("with the forge away: forge %q, commit %q, verdict %+v", a.Forge, a.Commit, a.Verdict) + } + gate, repo := statusNamed(a, ctxGate), statusNamed(a, ctxRepo) + if gate == nil || gate.State != "success" || !strings.Contains(gate.Source, "could not be read") || + repo == nil || repo.State != "error" || repo.Verdict != "ERROR — the toolchain was not there" { + t.Fatalf("the statuses from the verdict: %+v %+v", gate, repo) + } + if a.Merge.Mergeable != nil || !strings.Contains(a.Says, "mergeable: not known") { + t.Fatalf("whether it merges, with the forge away: %+v, %q", a.Merge, a.Says) + } + + // The protection unreadable: the statuses are said, and whether it merges is not known. + w.forge.down, w.forge.protectionDown = false, true + w.forge.pulls["novox/app#7"] = ForgePull{Number: 7, Base: "main", HeadSHA: head} + a, _ = w.h.Checks("novox/app", 7, "") + if a.Merge.Mergeable != nil || a.Forge != "" { + t.Fatalf("with the protection unreadable: %+v", a.Merge) + } +} + +func TestChecksRefuseWhatNamesNoCommit(t *testing.T) { + w := newWorld(t) + for _, c := range []struct { + repo string + number int + commit string + says string + }{ + {"novox/app", 0, "", "number or a commit"}, + {"app", 7, "", "owner/repository"}, + {"novox/app/x", 7, "", "owner/repository"}, + {"novox/app", 0, "abc", "not a commit"}, + {"novox/app", 0, "zzzzzzzzzz", "not a commit"}, + {"novox/app", -1, "", "not a pull request's number"}, + } { + if _, err := w.h.Checks(c.repo, c.number, c.commit); err == nil || !strings.Contains(err.Error(), c.says) { + t.Errorf("%s #%d %q: %v", c.repo, c.number, c.commit, err) + } + } + if len(w.forge.asked) != 0 { + t.Fatalf("the forge was asked for what names nothing: %v", w.forge.asked) + } +} + +func TestANumberIsTakenAsANumberOrAsText(t *testing.T) { + for _, c := range []struct { + in any + want int + ok bool + }{{float64(168), 168, true}, {"168", 168, true}, {"#168", 168, true}, {nil, 0, true}, {"", 0, true}, + {"x", 0, false}, {1.5, 0, false}} { + got, err := intArg(map[string]any{"number": c.in}, "number") + if (err == nil) != c.ok || got != c.want { + t.Errorf("%v → %d %v", c.in, got, err) + } + } +} diff --git a/modules/mesh-delivery/cmd/mesh-delivery/covered_test.go b/modules/mesh-delivery/cmd/mesh-delivery/covered_test.go new file mode 100644 index 00000000..a7caf67d --- /dev/null +++ b/modules/mesh-delivery/cmd/mesh-delivery/covered_test.go @@ -0,0 +1,681 @@ +package main + +import ( + "fmt" + "strings" + "testing" + "time" +) + +// novox/hq issues 319 and 316: a held delivery whose change already runs, or that builds nothing, is not a +// question for a person. A merge that moves nothing is delivered at once; a merge whose every module a later +// merge on the same trunk delivered is covered — never stopped, never an error on its pull request; and a +// group member held for a composed check that had not answered goes on once it passes. + +var t0 = time.Date(2026, 10, 6, 12, 0, 0, 0, time.UTC) + +// mergedAt is the forge's `pull.merged` at a moment. +func mergedAt(p PullEvent, merge string, at time.Time) PullEvent { + p.MergeCommit = merge + p.MergedAt = at.UTC().Format(time.RFC3339) + return p +} + +// verdictMoving is a passing verdict whose plan moves these modules — or builds nothing, given none. +func verdictMoving(repo string, number int, sha string, modules ...string) CheckedEvent { + c := verdict(repo, number, sha, "pass") + c.Plan = planMoving(repo, modules...) + return c +} + +func planMoving(repo string, modules ...string) *DeliveryPlan { + if len(modules) == 0 { + return &DeliveryPlan{Repository: repo, Base: "main", Summary: "builds nothing: the change touches no module of the mesh's graph"} + } + return &DeliveryPlan{Repository: repo, Base: "main", Moved: modules, Summary: "builds " + strings.Join(modules, ", ") + " → anchor"} +} + +// deliverLater is a later merge into a trunk, walked: its walk opened for its merge commit, let go, and left +// in the state given, every module of it sent. +func deliverLater(w *world, repo string, number int, base, headSHA, merge string, at time.Time, state string, + modules ...string) string { + w.t.Helper() + p := pr(repo, number, headSHA, fmt.Sprintf("feat/later-%d", number)) + p.Base = base + w.h.PullUpdated(p) + w.h.Checked(verdictMoving(repo, number, headSHA, modules...)) + w.h.PullMerged(mergedAt(p, merge, at)) + walk := aWalk("plan-"+merge[:6], repo, merge, true) + walk.Branch = base + walk.Modules = map[string]*WalkModule{} + for _, m := range modules { + walk.Modules[m] = &WalkModule{} + } + w.h.WalkMoved(w.ctl.put(walk)) + w.settleAll() + walk = w.ctl.walk(walk.ID) + sent := at.Add(5 * time.Minute) + for _, m := range modules { + walk.Modules[m] = &WalkModule{State: "built", SentAt: &sent} + } + walk.State = state + w.h.WalkMoved(w.ctl.put(walk)) + w.settleAll() + return IDOf(repo, headSHA) +} + +func statusOn(w *world, repo, commit string) string { + w.forge.mu.Lock() + defer w.forge.mu.Unlock() + return w.forge.statuses[repo+"@"+commit+" mesh/delivery"] +} + +// **A merge that moves nothing is delivered at once** (issue 319): there is nothing for a walk to move and +// nothing for a person to decide — whether its check passed, failed or never answered, and whether its plan +// builds nothing at all or only a new module the controller builds and sends nowhere (issue 300). +func TestAMergeThatMovesNothingIsDeliveredAtOnce(t *testing.T) { + for _, c := range []struct { + name string + verdict string // "" is a merge before any verdict + plan *DeliveryPlan + }{ + {"builds nothing, merged before its check answered", "", planMoving("novox/hq")}, + {"builds nothing, merged on a failed check", "fail", planMoving("novox/hq")}, + {"builds nothing, merged on a passing check", "pass", planMoving("novox/hq")}, + {"a new module, sent nowhere", "pass", &DeliveryPlan{Repository: "novox/hq", New: []string{"modules/zfs"}, + Summary: "builds new: modules/zfs, sent nowhere; no bus step"}}, + {"a new module, sent nowhere, merged before its check answered", "", &DeliveryPlan{Repository: "novox/hq", + New: []string{"modules/zfs"}, Summary: "builds new: modules/zfs, sent nowhere; no bus step"}}, + } { + t.Run(c.name, func(t *testing.T) { + w := newWorld(t) + w.ctl.planned = c.plan + p := pr("novox/hq", 7, head, "docs/x", "02-DECISIONS/x.md") + w.h.PullUpdated(p) + if c.verdict != "" { + v := verdict("novox/hq", 7, head, c.verdict) + v.Plan = c.plan + w.h.Checked(v) + } + w.later(31 * time.Minute) + w.h.PullMerged(mergedAt(p, merge, t0.Add(30*time.Minute))) + w.settleAll() + // Past the bound a merge waits for its walk: nothing holds it then either. + w.later(15 * time.Minute) + if err := w.h.Reconcile(); err != nil { + t.Fatal(err) + } + w.settleAll() + d := w.delivery(IDOf("novox/hq", head)) + if d.State != Delivered { + t.Fatalf("a merge whose plan is %q is %s (%s), not delivered", c.plan.Summary, d.State, d.HeldWhy) + } + if s := statusOn(w, "novox/hq", head); !strings.HasPrefix(s, "success") { + t.Fatalf("its pull request's mesh/delivery says %q", s) + } + }) + } +} + +// **A held delivery whose every module a later merge on its trunk delivered is covered** (issue 319): the +// later merge commit contains it, its walk is done, and it sent each module this one moves. Anything short +// of that — a module not carried, an earlier merge, another trunk, a walk that did not finish — leaves it held +// for a person, as before. +func TestAMergeALaterDeliveredWalkCarriedIsCovered(t *testing.T) { + const ctl = "novox/mesh-controller" + for _, c := range []struct { + name string + // The later merge: its trunk, minutes after t0 it merged (this one merged at 30), its walk's end, and + // what it sent. + base string + at int + state string + modules []string + // Whether this one was held for a person before the later merge was delivered. + heldFirst bool + // What the forge says of the later merge containing this one: yes, no, or away until asked again. + forge string + want State + }{ + {"carried every module, delivered after this one was held", "main", 35, walkDone, + []string{"build-agent", "mesh-controller"}, true, "yes", Covered}, + {"carried every module, delivered before this one was due to be held", "main", 35, walkDone, + []string{"build-agent", "mesh-controller", "route-proxy"}, false, "yes", Covered}, + {"carried one of its two modules", "main", 35, walkDone, []string{"mesh-controller"}, true, "yes", Held}, + {"merged before this one", "main", 25, walkDone, []string{"build-agent", "mesh-controller"}, true, "yes", Held}, + {"merged into another branch", "release", 35, walkDone, []string{"build-agent", "mesh-controller"}, true, "yes", Held}, + {"its walk failed", "main", 35, walkFailed, []string{"build-agent", "mesh-controller"}, true, "yes", Held}, + {"the forge says the later merge does not contain it", "main", 35, walkDone, + []string{"build-agent", "mesh-controller"}, true, "no", Held}, + {"the forge could not say at first, then said it does", "main", 35, walkDone, + []string{"build-agent", "mesh-controller"}, true, "away", Covered}, + } { + t.Run(c.name, func(t *testing.T) { + w := newWorld(t) + pair := merge + " in nnnnnnnnnnnn8888" + switch c.forge { + case "yes", "away": + w.forge.contains[pair] = true + case "no": + w.forge.contains[pair] = false + } + w.forge.containsDown = c.forge == "away" + p := pr(ctl, 1, head, "feat/one", "cmd/a.go") + w.h.PullUpdated(p) + w.h.Checked(verdictMoving(ctl, 1, head, "mesh-controller", "build-agent")) + w.later(31 * time.Minute) + w.h.PullMerged(mergedAt(p, merge, t0.Add(30*time.Minute))) + w.settleAll() + if c.heldFirst { + w.later(15 * time.Minute) + if err := w.h.Reconcile(); err != nil { + t.Fatal(err) + } + w.settleAll() + if s := w.state(IDOf(ctl, head)); s != Held { + t.Fatalf("a merge no walk opened for is %s before the later one", s) + } + } + later := deliverLater(w, ctl, 2, c.base, "bbbbbbbbbbbb2222", "nnnnnnnnnnnn8888", t0.Add(time.Duration(c.at)*time.Minute), + c.state, c.modules...) + if c.state == walkDone && w.state(later) != Delivered { + t.Fatalf("the later merge is %s", w.state(later)) + } + w.later(20 * time.Minute) + if err := w.h.Reconcile(); err != nil { + t.Fatal(err) + } + w.settleAll() + if c.forge == "away" { + // A forge that cannot say is "not yet covered": held, and asked again a minute on. + if s := w.state(IDOf(ctl, head)); s != Held { + t.Fatalf("with the forge away it is %s", s) + } + w.forge.mu.Lock() + w.forge.containsDown = false + w.forge.mu.Unlock() + w.later(2 * time.Minute) + w.settleAll() + } + d := w.delivery(IDOf(ctl, head)) + if d.State != c.want { + t.Fatalf("it is %s (%s), want %s", d.State, d.HeldWhy, c.want) + } + s := statusOn(w, ctl, head) + if c.want == Covered { + if !strings.HasPrefix(s, "success covered: a later merge that contains it was delivered") || + !strings.Contains(s, "nnnnnnnn") { + t.Fatalf("its pull request's mesh/delivery says %q", s) + } + last := d.Transitions[len(d.Transitions)-1] + if !strings.Contains(last.Why, IDOf(ctl, "bbbbbbbbbbbb2222")) { + t.Fatalf("its transition does not name what covered it: %q", last.Why) + } + } + if strings.HasPrefix(s, "error") { + t.Fatalf("its pull request's mesh/delivery is an error: %q", s) + } + // Asked once for the pair, however many ticks read it — once more only after the forge was away. + asked := 0 + for _, a := range w.forge.containsAsks { + if a == pair { + asked++ + } + } + candidate := c.base == "main" && c.at > 30 && c.state == walkDone && len(c.modules) >= 2 + switch { + case !candidate && asked != 0: + t.Fatalf("the forge was asked whether %s for a merge order and walk that cannot cover it", pair) + case candidate && c.forge != "away" && asked != 1: + t.Fatalf("the forge was asked %d time(s) whether %s, want once", asked, pair) + case candidate && c.forge == "away" && asked < 2: + t.Fatalf("the forge, away, was asked %d time(s) whether %s: never again", asked, pair) + } + }) + } +} + +// A covered member of a group counts as delivered: the member after it is let go, not stopped. +func TestACoveredGroupMemberLetsTheNextOneGo(t *testing.T) { + w := newWorld(t) + const ctl, lab = "novox/mesh-controller", "novox/mesh-lab" + const ctlHead, labHead = "c1c1c1c1c1c1c1c1", "d2d2d2d2d2d2d2d2" + ctlID, labID := IDOf(ctl, ctlHead), IDOf(lab, labHead) + pc, pl := pr(ctl, 1, ctlHead, "feat/g", "cmd/a.go"), pr(lab, 2, labHead, "feat/g", "replays/a.go") + w.h.PullUpdated(pc) + w.h.PullUpdated(pl) + w.h.Checked(verdictMoving(ctl, 1, ctlHead, "mesh-controller")) + w.h.Checked(verdictMoving(lab, 2, labHead, "mesh-lab")) + w.settleAll() + w.h.Checked(CheckedEvent{Group: "feat/g", ID: "check-1", Verdict: "pass", Summary: "both compose"}) + w.later(31 * time.Minute) + w.h.PullMerged(mergedAt(pc, "m1m1m1m1m1m1", t0.Add(30*time.Minute))) + w.h.PullMerged(mergedAt(pl, "m2m2m2m2m2m2", t0.Add(31*time.Minute))) + w.h.WalkMoved(w.ctl.put(aWalk("plan-lab", lab, "m2m2m2m2m2m2", true))) + w.settleAll() + w.forge.contains["m1m1m1m1m1m1 in m3m3m3m3m3m3"] = true + deliverLater(w, ctl, 3, "main", "e3e3e3e3e3e3e3e3", "m3m3m3m3m3m3", t0.Add(35*time.Minute), walkDone, "mesh-controller") + w.later(20 * time.Minute) + if err := w.h.Reconcile(); err != nil { + t.Fatal(err) + } + w.settleAll() + if w.state(ctlID) != Covered { + t.Fatalf("the controller's member is %s", w.state(ctlID)) + } + if w.state(labID) != Delivering || !containsString(w.ctl.delivers, "plan-lab") { + t.Fatalf("the member after it is %s, let go %v", w.state(labID), w.ctl.delivers) + } + lw := w.ctl.walk("plan-lab") + lw.State = walkDone + w.h.WalkMoved(w.ctl.put(lw)) + w.settleAll() + if state, why := GroupState(w.h.groups["feat/g"], w.h.membersOf(w.h.groups["feat/g"])); state != "delivered" { + t.Fatalf("the group is %s: %s", state, why) + } +} + +// **On rollout, the deliveries already held close by themselves** (issue 319's held ones), and a restart +// after that moves nothing again: what was read back is where it stays. +func TestHeldDeliveriesReadBackCloseByThemselvesOnce(t *testing.T) { + const ctl, cat = "novox/mesh-controller", "novox/mesh-catalog" + merged1, merged2 := t0, t0.Add(2*time.Hour) + coverer := func(repo, module string) *Delivery { + return &Delivery{ID: IDOf(repo, "c0c0c0c0c0c0c0c0"), Repository: repo, Commit: "c0c0c0c0c0c0c0c0", Number: 90, + Base: "main", State: Delivered, Since: merged2.Add(time.Hour), Created: merged2.Add(-time.Hour), + MergedAs: "f0f0f0f0f0f0f0f0", MergedAt: merged2, Plan: planMoving(repo, module), Check: &Verdict{Gate: "pass"}, + Walk: &WalkSeen{ID: "plan-90", Commit: "f0f0f0f0f0f0f0f0", State: walkDone}, + Steps: []Step{{Machine: "anchor", Module: module, State: StepPassed}, {Machine: "the rest running it", Module: module, State: StepSent}}} + } + held := func(repo string, plan *DeliveryPlan, why string) *Delivery { + return &Delivery{ID: IDOf(repo, head), Repository: repo, Commit: head, Number: 7, Base: "main", State: Held, + Since: merged1.Add(10 * time.Minute), Created: merged1.Add(-time.Hour), MergedAs: merge, MergedAt: merged1, + HeldWhy: why, Plan: plan, Transitions: []Transition{{At: merged1, From: Ready, To: Held, Event: EvHold, Why: why}}} + } + const unwalkedWhy = "merged, and the controller opened no walk for it within 10m0s — nothing it holds follows that branch, " + + "or the merge was not heard: a person decides" + withCoverer := func(d *Delivery, change func(c *Delivery)) []*Delivery { + c := coverer(d.Repository, d.Plan.Moved[0]) + change(c) + return []*Delivery{d, c} + } + const uncheckedWhy = "merged without a passing check: only a person decides that it goes on" + for _, c := range []struct { + name string + kept []*Delivery + group *Group + want State + }{ + {"merged without a passing check, builds nothing", + []*Delivery{held("novox/hq", planMoving("novox/hq"), "merged without a passing check: only a person decides that it goes on")}, + nil, Delivered}, + {"a new module sent nowhere, no walk opened", + []*Delivery{held(cat, &DeliveryPlan{New: []string{"modules/zfs"}, Summary: "builds new: modules/zfs, sent nowhere"}, unwalkedWhy)}, + nil, Delivered}, + {"no walk opened, a later delivered walk carried its module", + []*Delivery{held(ctl, planMoving(ctl, "mesh-controller"), unwalkedWhy), coverer(ctl, "mesh-controller")}, nil, Covered}, + {"merged without a passing check, a later delivered walk whose own check passed carried its module", + withCoverer(held(ctl, planMoving(ctl, "mesh-controller"), uncheckedWhy), func(*Delivery) {}), nil, Covered}, + {"merged without a passing check, the later delivery's own check did not pass", + withCoverer(held(ctl, planMoving(ctl, "mesh-controller"), uncheckedWhy), func(c *Delivery) { + c.Check = &Verdict{Gate: "fail"} + }), nil, Held}, + {"no walk opened, the later delivery's walk was of another commit", + withCoverer(held(ctl, planMoving(ctl, "mesh-controller"), unwalkedWhy), func(c *Delivery) { + c.Walk.Commit = "e0e0e0e0e0e0e0e0" + }), nil, Held}, + {"no walk opened, the later walk sent its module to its first machine only", + withCoverer(held(ctl, planMoving(ctl, "mesh-controller"), unwalkedWhy), func(c *Delivery) { + c.Steps = c.Steps[:1] + }), nil, Held}, + {"no walk opened, the later walk's branch is another", + withCoverer(held(ctl, planMoving(ctl, "mesh-controller"), unwalkedWhy), func(c *Delivery) { + c.Walk.Branch = "release" + }), nil, Held}, + {"no walk opened, a later delivered walk carried another module", + []*Delivery{held(ctl, planMoving(ctl, "mesh-controller"), unwalkedWhy), coverer(ctl, "route-proxy")}, nil, Held}, + {"held for its group, its own walk superseded, a later delivered walk carried its module", + func() []*Delivery { + d := held(cat, planMoving(cat, "systemd"), "its group feat/j's composed check did not pass for the heads that "+ + "merged; a person decides that it goes on") + d.Group = "feat/j" + d.Walk = &WalkSeen{ID: "plan-7", Commit: merge, State: walkSuperseded, Waits: true, WaitedFor: "mesh-delivery"} + other := &Delivery{ID: IDOf("novox/mesh-host", "abababababababab"), Repository: "novox/mesh-host", + Commit: "abababababababab", Number: 3, Base: "main", Group: "feat/j", State: Delivered, + MergedAs: "acacacacacacacac", MergedAt: merged1} + return []*Delivery{d, other, coverer(cat, "systemd")} + }(), + &Group{ID: "feat/j", Members: []string{IDOf(cat, head), IDOf("novox/mesh-host", "abababababababab")}, Closed: true, + Check: &GroupCheck{Asked: "check-1", Verdict: "pass", Commits: map[string]string{IDOf(cat, head): head, + IDOf("novox/mesh-host", "abababababababab"): "abababababababab"}}}, + Covered}, + } { + t.Run(c.name, func(t *testing.T) { + store := newMemStore() + for _, d := range c.kept { + if err := store.PutDelivery(d); err != nil { + t.Fatal(err) + } + } + if c.group != nil { + if err := store.PutGroup(c.group); err != nil { + t.Fatal(err) + } + } + now := merged2.Add(30 * time.Hour) + forge := newFakeForge() + forge.contains[merge+" in f0f0f0f0f0f0f0f0"] = true + w := newWorldOver(t, store, newFakeController(), forge, now) + if err := w.h.Reconcile(); err != nil { + t.Fatal(err) + } + w.settleAll() + id := c.kept[0].ID + d := w.delivery(id) + if d.State != c.want { + t.Fatalf("read back held, it is %s, want %s", d.State, c.want) + } + if s := statusOn(w, d.Repository, d.Commit); c.want != Held && !strings.HasPrefix(s, "success") { + t.Fatalf("its pull request's mesh/delivery says %q", s) + } + // A restart over what it kept: nothing moves again. + again := newWorldOver(t, store, newFakeController(), forge, now.Add(time.Hour)) + if err := again.h.Reconcile(); err != nil { + t.Fatal(err) + } + again.settleAll() + if d2 := again.delivery(id); d2.State != d.State || len(d2.Transitions) != len(d.Transitions) { + t.Fatalf("a restart moved it again: %s with %d transitions, was %s with %d", d2.State, len(d2.Transitions), + d.State, len(d.Transitions)) + } + }) + } +} + +// **A group member is not held for a composed check that has not answered** (issue 316): it waits published, +// and goes on once the check passes for the heads that merged. +func TestAGroupMemberWaitsForItsComposedCheckToAnswer(t *testing.T) { + w := newWorld(t) + const a, b = "novox/a", "novox/b" + const aHead, bHead = "a1a1a1a1a1a1a1a1", "b2b2b2b2b2b2b2b2" + aID := IDOf(a, aHead) + pa, pb := pr(a, 1, aHead, "feat/k", "x"), pr(b, 2, bHead, "feat/k", "y") + w.h.PullUpdated(pa) + w.h.PullUpdated(pb) + w.h.Checked(verdictMoving(a, 1, aHead, "a")) + w.h.Checked(verdictMoving(b, 2, bHead, "b")) + w.settleAll() + if len(w.ctl.checks) != 1 { + t.Fatalf("the composed check was asked %v", w.ctl.checks) + } + w.h.PullMerged(mergedAt(pa, "m1m1m1m1m1m1", t0)) + w.h.WalkMoved(w.ctl.put(aWalk("plan-a", a, "m1m1m1m1m1m1", true))) + w.settleAll() + if s := w.state(aID); s != Published { + t.Fatalf("merged while its group's composed check had not answered, it is %s (%s)", s, w.delivery(aID).HeldWhy) + } + w.h.Checked(CheckedEvent{Group: "feat/k", ID: "check-1", Verdict: "pass", Summary: "both compose"}) + w.settleAll() + if s := w.state(aID); s != Published || len(w.ctl.delivers) != 0 { + t.Fatalf("its composed check passed, but the other member is not merged: %s, let go %v", s, w.ctl.delivers) + } + // The other member merges: every member is on the trunk, and the walk is let go (ADR 0418 §2). + w.h.PullMerged(mergedAt(pb, "m2m2m2m2m2m2", t0)) + w.h.WalkMoved(w.ctl.put(aWalk("plan-b", b, "m2m2m2m2m2m2", true))) + w.settleAll() + if s := w.state(aID); s != Delivering || !containsString(w.ctl.delivers, "plan-a") { + t.Fatalf("every member merged, and it is %s, let go %v", s, w.ctl.delivers) + } +} + +// **A member already held for its group's composed check goes on when that check passes for the heads that +// merged** (issue 316); a check that failed, or passed for other heads, keeps it held for a person. +func TestAMemberHeldForItsComposedCheckIsLiftedWhenItPasses(t *testing.T) { + const a, b = "novox/a", "novox/b" + const aHead, bHead = "a1a1a1a1a1a1a1a1", "b2b2b2b2b2b2b2b2" + aID, bID := IDOf(a, aHead), IDOf(b, bHead) + for _, c := range []struct { + name string + verdict string + commits map[string]string + want State + }{ + {"passed for the heads that merged", "pass", map[string]string{aID: aHead, bID: bHead}, Delivering}, + {"warned for the heads that merged", "warning", map[string]string{aID: aHead, bID: bHead}, Delivering}, + {"failed", "fail", map[string]string{aID: aHead, bID: bHead}, Held}, + {"passed for other heads", "pass", map[string]string{aID: "a9a9a9a9a9a9a9a9", bID: bHead}, Held}, + } { + t.Run(c.name, func(t *testing.T) { + store := newMemStore() + why := "its group feat/k's composed check did not pass for the heads that merged; a person decides that it goes on" + da := &Delivery{ID: aID, Repository: a, Commit: aHead, Number: 1, Base: "main", Branch: "feat/k", Group: "feat/k", + State: Held, Since: t0, Created: t0.Add(-time.Hour), MergedAs: "m1m1m1m1m1m1", MergedAt: t0, HeldWhy: why, + Plan: planMoving(a, "a"), Check: &Verdict{Gate: "pass"}, + Walk: &WalkSeen{ID: "plan-a", Commit: "m1m1m1m1m1m1", State: walkBuilding, Waits: true, WaitedFor: "mesh-delivery"}} + db := &Delivery{ID: bID, Repository: b, Commit: bHead, Number: 2, Base: "main", Branch: "feat/k", Group: "feat/k", + State: Published, Since: t0, Created: t0.Add(-time.Hour), MergedAs: "m2m2m2m2m2m2", MergedAt: t0, + Plan: planMoving(b, "b"), Check: &Verdict{Gate: "pass"}, + Walk: &WalkSeen{ID: "plan-b", Commit: "m2m2m2m2m2m2", State: walkBuilding, Waits: true, WaitedFor: "mesh-delivery"}} + g := &Group{ID: "feat/k", Members: []string{aID, bID}, Closed: true, + Check: &GroupCheck{Asked: "check-old", At: t0, Commits: c.commits}} + for _, d := range []*Delivery{da, db} { + _ = store.PutDelivery(d) + } + _ = store.PutGroup(g) + ctl := newFakeController() + ctl.put(aWalk("plan-a", a, "m1m1m1m1m1m1", true)) + ctl.put(aWalk("plan-b", b, "m2m2m2m2m2m2", true)) + w := newWorldOver(t, store, ctl, newFakeForge(), t0.Add(time.Minute)) + w.settleAll() + if s := w.state(aID); s != Held { + t.Fatalf("before its composed verdict it is %s", s) + } + w.h.Checked(CheckedEvent{Group: "feat/k", ID: "check-old", Verdict: c.verdict, Summary: "composed: " + c.verdict}) + w.settleAll() + if s := w.state(aID); s != c.want { + t.Fatalf("its composed check %s, and it is %s (%s), want %s", c.name, s, w.delivery(aID).HeldWhy, c.want) + } + if c.want == Delivering && !containsString(ctl.delivers, "plan-a") { + t.Fatalf("lifted, its walk was not let go: %v", ctl.delivers) + } + // A composed check of other heads is asked again for these heads (novox/hq ADR 0323 §6). + if c.name == "passed for other heads" && !containsString(ctl.checks, "feat/k:"+aID+","+bID) { + t.Fatalf("the composed check was not asked again for the heads that merged: %v", ctl.checks) + } + }) + } +} + +// **A module a release carried to every machine the held delivery sends it to covers it** (novox/hq issue +// 351). A build the controller held back for a person and released on their word reaches each machine by a +// backlog walk, which credits the delivery that published the build one step per machine — never the walk's +// own "every machine running it" or "the rest running it". The held delivery's plan says which machines +// receive the module: a passed step on each of them is the module sent everywhere. A machine still judging, +// one that failed, or a plan that names no machines leaves it held, as before. +func TestAModuleAReleaseCarriedToEveryMachineCovers(t *testing.T) { + const cat = "novox/mesh-catalog" + merged1, merged2 := t0, t0.Add(2*time.Hour) + laptop, desktop := "laptop", "desktop" + plan := func(machines ...string) *DeliveryPlan { + p := planMoving(cat, "openrazer") + for _, m := range machines { + p.Machines = append(p.Machines, MachinePlan{Machine: m, Receives: []string{"openrazer"}}) + } + return p + } + held := func(p *DeliveryPlan) *Delivery { + why := "its group feat/j's composed check did not pass for the heads that merged; a person decides that it goes on" + return &Delivery{ID: IDOf(cat, head), Repository: cat, Commit: head, Number: 7, Base: "main", State: Held, + Since: merged1.Add(10 * time.Minute), Created: merged1.Add(-time.Hour), MergedAs: merge, MergedAt: merged1, + HeldWhy: why, Plan: p, Walk: &WalkSeen{ID: "plan-7", Commit: merge, State: walkSuperseded, Waits: true, WaitedFor: "mesh-delivery"}, + Transitions: []Transition{{At: merged1, From: Published, To: Held, Event: EvHold, Why: why}}} + } + step := func(machine string, state StepState) Step { + return Step{Machine: machine, Module: "openrazer", Build: "build-1", State: state, Why: "carried by release-1"} + } + coverer := func(steps ...Step) *Delivery { + return &Delivery{ID: IDOf(cat, "c0c0c0c0c0c0c0c0"), Repository: cat, Commit: "c0c0c0c0c0c0c0c0", Number: 90, + Base: "main", State: Delivered, Since: merged2.Add(time.Hour), Created: merged2.Add(-time.Hour), + MergedAs: "f0f0f0f0f0f0f0f0", MergedAt: merged2, Plan: planMoving(cat, "power"), Check: &Verdict{Gate: "pass"}, + Walk: &WalkSeen{ID: "plan-90", Commit: "f0f0f0f0f0f0f0f0", State: walkDone}, + Steps: append([]Step{{Machine: "anchor", Module: "power", State: StepPassed}, + {Machine: "the rest running it", Module: "power", State: StepSent}}, steps...)} + } + for _, c := range []struct { + name string + held *Delivery + by *Delivery + want State + }{ + {"passed on every machine its plan sends it to", held(plan(laptop, desktop)), + coverer(step(laptop, StepPassed), step(desktop, StepPassed)), Covered}, + {"still judging on one of them", held(plan(laptop, desktop)), + coverer(step(laptop, StepPassed), step(desktop, StepJudging)), Held}, + {"failed on one of them", held(plan(laptop, desktop)), + coverer(step(laptop, StepPassed), step(desktop, StepFailed)), Held}, + {"never sent to one of them", held(plan(laptop, desktop)), coverer(step(laptop, StepPassed)), Held}, + {"its plan names no machines", held(plan()), coverer(step(laptop, StepPassed), step(desktop, StepPassed)), Held}, + } { + t.Run(c.name, func(t *testing.T) { + store := newMemStore() + for _, d := range []*Delivery{c.held, c.by} { + if err := store.PutDelivery(d); err != nil { + t.Fatal(err) + } + } + forge := newFakeForge() + forge.contains[merge+" in f0f0f0f0f0f0f0f0"] = true + w := newWorldOver(t, store, newFakeController(), forge, merged2.Add(30*time.Hour)) + if err := w.h.Reconcile(); err != nil { + t.Fatal(err) + } + w.settleAll() + d := w.delivery(c.held.ID) + if d.State != c.want { + t.Fatalf("it is %s (%s), want %s", d.State, d.HeldWhy, c.want) + } + if c.want == Covered { + if s := statusOn(w, cat, head); !strings.HasPrefix(s, "success covered: a later merge that contains it was delivered") { + t.Fatalf("its pull request's mesh/delivery says %q", s) + } + if last := d.Transitions[len(d.Transitions)-1]; !strings.Contains(last.Why, "openrazer") { + t.Fatalf("its transition does not name the module: %q", last.Why) + } + } + }) + } +} + +// **A finished release moves the steps it carried to passed** (novox/hq issue 351): a backlog walk's gate +// credits a step per machine while it judges, and its last record keeps no gate — only which machines it is +// done on. A step credited to it, on a machine it is done on, is passed; a step it failed on, one on a machine +// it is not done on, and a step another walk carried are left as they are. +func TestAFinishedReleaseMovesTheStepsItCarriedToPassed(t *testing.T) { + const cat = "novox/mesh-catalog" + carried := func(machine string, state StepState, by string) Step { + return Step{Machine: machine, Module: "openrazer", Build: "build-1", State: state, Why: "carried by " + by} + } + for _, c := range []struct { + name string + steps []Step + done []string + want map[string]StepState + }{ + {"done on both", []Step{carried("laptop", StepJudging, "release-1"), carried("desktop", StepJudging, "release-1")}, + []string{"laptop", "desktop"}, map[string]StepState{"laptop": StepPassed, "desktop": StepPassed}}, + {"done on one, the other still to come", []Step{carried("laptop", StepJudging, "release-1"), carried("desktop", StepJudging, "release-1")}, + []string{"laptop"}, map[string]StepState{"laptop": StepPassed, "desktop": StepJudging}}, + {"a step it failed on stays failed", []Step{carried("laptop", StepFailed, "release-1")}, + []string{"laptop"}, map[string]StepState{"laptop": StepFailed}}, + {"a step another walk carried is not its", []Step{carried("laptop", StepJudging, "release-2")}, + []string{"laptop"}, map[string]StepState{"laptop": StepJudging}}, + } { + t.Run(c.name, func(t *testing.T) { + d := &Delivery{ID: IDOf(cat, "c0c0c0c0c0c0c0c0"), Repository: cat, Commit: "c0c0c0c0c0c0c0c0", Number: 90, + Base: "main", State: Delivered, Since: t0.Add(time.Hour), Created: t0, MergedAs: "f0f0f0f0f0f0f0f0", MergedAt: t0, + Plan: planMoving(cat, "power"), Check: &Verdict{Gate: "pass"}, + Walk: &WalkSeen{ID: "plan-90", Commit: "f0f0f0f0f0f0f0f0", State: walkDone}, Steps: c.steps} + store := newMemStore() + if err := store.PutDelivery(d); err != nil { + t.Fatal(err) + } + w := newWorldOver(t, store, newFakeController(), newFakeForge(), t0.Add(2*time.Hour)) + w.h.WalkMoved(Walk{ID: "release-1", Repository: "the builds waiting for a gate", State: walkDone, + Tiers: [][]string{{"openrazer"}}, Modules: map[string]*WalkModule{}, + Release: &WalkRelease{Order: []string{"laptop", "desktop"}, Next: len(c.done), Done: c.done}}) + got := map[string]StepState{} + for _, s := range w.delivery(d.ID).Steps { + got[s.Machine] = s.State + } + for machine, want := range c.want { + if got[machine] != want { + t.Fatalf("openrazer on %s is %s, want %s (steps %v)", machine, got[machine], want, got) + } + } + }) + } +} + +// The day's case whole (novox/hq issue 351): a member held for its group's check, its own walk taken over by +// later merges; the one later walk that carried its module failed its gate; a later merge's walk was delivered +// without the module; the controller then held the module's build back for a person, who released it, and the +// release reached both machines. The held delivery is covered by that later merge once the release is done — +// read by its id, since a walk that ended before the holder's restart may be older than the ended walks the +// controller lists back. +func TestAHeldDeliveryWhoseModuleAReleaseCarriedIsCoveredOnceTheReleaseIsDone(t *testing.T) { + const cat = "novox/mesh-catalog" + merged1, merged2 := t0, t0.Add(2*time.Hour) + plan := planMoving(cat, "openrazer") + plan.Machines = []MachinePlan{{Machine: "laptop", Receives: []string{"openrazer"}}, {Machine: "desktop", Receives: []string{"openrazer"}}} + why := "its group feat/j's composed check did not pass for the heads that merged; a person decides that it goes on" + held := &Delivery{ID: IDOf(cat, head), Repository: cat, Commit: head, Number: 7, Base: "main", State: Held, + Since: merged1.Add(10 * time.Minute), Created: merged1.Add(-time.Hour), MergedAs: merge, MergedAt: merged1, + HeldWhy: why, Plan: plan, Walk: &WalkSeen{ID: "plan-7", Commit: merge, State: walkSuperseded, Waits: true, WaitedFor: "mesh-delivery"}, + Transitions: []Transition{{At: merged1, From: Published, To: Held, Event: EvHold, Why: why}}} + later := &Delivery{ID: IDOf(cat, "c0c0c0c0c0c0c0c0"), Repository: cat, Commit: "c0c0c0c0c0c0c0c0", Number: 90, + Base: "main", State: Delivered, Since: merged2.Add(time.Hour), Created: merged2.Add(-time.Hour), + MergedAs: "f0f0f0f0f0f0f0f0", MergedAt: merged2, Plan: planMoving(cat, "power"), Check: &Verdict{Gate: "pass"}, + Walk: &WalkSeen{ID: "plan-90", Commit: "f0f0f0f0f0f0f0f0", State: walkDone}, + Steps: []Step{{Machine: "anchor", Module: "power", State: StepPassed}, {Machine: "the rest running it", Module: "power", State: StepSent}}} + store := newMemStore() + for _, d := range []*Delivery{held, later} { + if err := store.PutDelivery(d); err != nil { + t.Fatal(err) + } + } + forge := newFakeForge() + forge.contains[merge+" in f0f0f0f0f0f0f0f0"] = true + w := newWorldOver(t, store, newFakeController(), forge, merged2.Add(3*time.Hour)) + if err := w.h.Reconcile(); err != nil { + t.Fatal(err) + } + w.settleAll() + if s := w.state(held.ID); s != Held { + t.Fatalf("before any release it is %s", s) + } + // The release judges the build on each machine in turn: its gate names what it carried. + release := Walk{ID: "release-1", Repository: "the builds waiting for a gate", State: walkRolling, + Tiers: [][]string{{"openrazer"}}, Modules: map[string]*WalkModule{}, + Release: &WalkRelease{Order: []string{"laptop", "desktop"}, Gate: &WalkGate{Machines: []string{"laptop"}, + Carried: []WalkCarry{{Module: "openrazer", Node: "laptop", To: "f0f0f0f0f0f0f0f0", Build: "build-1"}, + {Module: "openrazer", Node: "desktop", To: "f0f0f0f0f0f0f0f0", Build: "build-1"}}}}} + w.h.WalkMoved(release) + w.settleAll() + if s := w.state(held.ID); s != Held { + t.Fatalf("while the release judges it is %s", s) + } + // Done on both: its last record keeps no gate. It ended while this holder was down, and is older than the + // ended walks the controller lists back: the holder asks for it by its id, since a step it carried still judges. + release.State, release.Release.Gate, release.Release.Done, release.Release.Next = walkDone, nil, []string{"laptop", "desktop"}, 2 + w.ctl.put(release) + w.later(time.Hour) // a walk a step still judges is asked for once an hour; at a start, at once + w.settleAll() + d := w.delivery(held.ID) + if d.State != Covered { + t.Fatalf("once the release is done it is %s (%s), want covered", d.State, d.HeldWhy) + } + if last := d.Transitions[len(d.Transitions)-1]; !strings.Contains(last.Why, IDOf(cat, "c0c0c0c0c0c0c0c0")) || + !strings.Contains(last.Why, "openrazer") { + t.Fatalf("its transition does not name what covered it: %q", last.Why) + } +} diff --git a/modules/mesh-delivery/cmd/mesh-delivery/effects.go b/modules/mesh-delivery/cmd/mesh-delivery/effects.go new file mode 100644 index 00000000..525a612d --- /dev/null +++ b/modules/mesh-delivery/cmd/mesh-delivery/effects.go @@ -0,0 +1,453 @@ +package main + +import ( + "encoding/json" + "fmt" + "regexp" + "sort" + "strings" + "time" +) + +// What a transition owes outside the state (novox/hq ADR 0239 decision 5), each by its owner: the event on +// the bus (said by this module), a line on the commit's note and the pull request's view and statuses (by +// the forge's holder, asked through its tools). Owed, never lost: kept with the delivery before anything is +// tried, and tried again until done. + +// transitionEvent is what `mesh-delivery.transition` says: no secret, no address. opened is when its pull +// request was first heard (zero when not known); with its class and its merge's moment it is what the issue +// tracker measures a fix's time by (novox/hq ADR 0313 §2, §4). +func transitionEvent(d *Delivery, t Transition, opened time.Time) map[string]any { + e := map[string]any{"id": d.ID, "repository": d.Repository, "commit": d.Commit, "from": t.From, "to": t.To, + "event": t.Event, "why": t.Why, "at": t.At} + if t.By != "" { + e["by"] = t.By + } + if d.Number > 0 { + e["number"] = d.Number + } + if d.Group != "" { + e["group"] = d.Group + } + if d.MergedAs != "" { + e["merged_as"] = d.MergedAs + } + if d.Walk != nil { + e["walk"] = d.Walk.ID + } + if d.Plan != nil { + e["plan"] = d.Plan.Summary + } + if c := fixClass(d); c != "" { + e["class"] = c + } + if !opened.IsZero() { + e["opened"] = opened.UTC() + } + if !d.MergedAt.IsZero() { + e["merged_at"] = d.MergedAt.UTC() + } + return e +} + +// fixClass is the class of what a delivery moves (novox/hq ADR 0313 §2): its plan's, as the planner said it; +// none for a plan that moves no module, from a planner that did not say a class; else its walk's class once its +// delivery time is known; empty when nothing says it. +func fixClass(d *Delivery) string { + switch { + case d.Plan != nil && d.Plan.Class != "": + return d.Plan.Class + case d.Plan != nil && len(d.Plan.Moved) == 0 && len(d.Plan.Dependents) == 0 && len(d.Plan.New) == 0: + return classNone + case d.Time != nil: + return d.Time.Class + } + return "" +} + +// classNone is the class of a delivery that moves no module: nothing for a walk to move, and no fix budget. +const classNone = "none" + +// openedOf is when a delivery's pull request was first heard: the earliest head of it this owner holds, never +// one made from the forge's word at its merge; zero when none is held or it has no pull request. Under the lock. +func (h *Holder) openedOf(d *Delivery) time.Time { + if d.Number <= 0 { + return time.Time{} + } + var first time.Time + for _, c := range h.deliveries { + if c.FromForge || c.Number != d.Number || !strings.EqualFold(c.Repository, d.Repository) || c.Created.IsZero() { + continue + } + if first.IsZero() || c.Created.Before(first) { + first = c.Created + } + } + if first.IsZero() && !d.FromForge { + first = d.Created + } + return first +} + +// noteLine is one transition as the commit's note keeps it. +func noteLine(d *Delivery, t Transition) string { + by := "" + if t.By != "" { + by = " by " + t.By + } + line := fmt.Sprintf("%s delivery %s: %s -> %s (%s%s): %s", t.At.UTC().Format(time.RFC3339), d.ID, orNothing(t.From), + t.To, t.Event, by, t.Why) + if t.To == Proposed && d.Plan != nil { + line += "; predicted: " + d.Plan.Summary + } + if d.Group != "" && (t.To == Proposed || t.To == Published) { + line += "; group " + d.Group + } + return oneLine(line) +} + +// executedLine is what a delivery did, written on the commit it landed as when it ends. +func executedLine(d *Delivery) string { + var parts []string + for _, s := range d.Steps { + parts = append(parts, fmt.Sprintf("%s on %s %s", s.Module, s.Machine, s.State)) + } + walk := "" + if d.Walk != nil { + walk = " (walk " + d.Walk.ID + ")" + } + return oneLine(fmt.Sprintf("%s delivery %s executed%s: %s", time.Now().UTC().Format(time.RFC3339), d.ID, walk, + strings.Join(parts, "; "))) +} + +func oneLine(s string) string { return strings.Join(strings.Fields(s), " ") } + +// statusOf is the commit status `mesh/delivery`: where the delivery stands, in the forge's words. +func statusOf(d *Delivery) (string, string) { + why := "" + if len(d.Transitions) > 0 { + why = d.Transitions[len(d.Transitions)-1].Why + } + switch d.State { + case Proposed, Checked: + return "pending", "checking" + case Ready: + return "success", "ready: it delivers once merged" + case Rejected: + return "failure", clip("rejected: "+why, 140) + case Published: + return "pending", "published: its walk waits for its turn" + case Held: + return "pending", clip("held for a person: "+d.HeldWhy, 140) + case Delivering: + passed := 0 + for _, s := range d.Steps { + if s.State == StepPassed { + passed++ + } + } + return "pending", fmt.Sprintf("delivering: %d machine step(s) passed", passed) + case Delivered: + return "success", "delivered" + case Failed: + if d.coveredStop() { + return "success", clip(d.coveredStopSaid(), 140) // its change reached the machines (novox/hq issue 747) + } + return "failure", clip("failed: "+why, 140) + case Superseded: + return "warning", clip("superseded: "+why, 140) + case Stopped: + if d.coveredStop() { + return "success", clip(d.coveredStopSaid(), 140) // its change reached the machines (novox/hq issue 480) + } + return "error", clip("stopped: "+why, 140) + case Retired: + return "success", "history: merged before this owner delivered anything, never walked" + case Covered: + by := d.CoveredBy + if by == "" { + by = d.Carrier + } + return "success", clip("covered: a later merge that contains it was delivered: "+by, 140) + } + return "pending", string(d.State) +} + +// viewMarker is how the forge's holder finds the one view comment of a pull request. +const viewMarker = "" + +// ViewBody is a delivery's view on its pull request, written from the delivery whole. +func ViewBody(d *Delivery, g *Group) string { + var b strings.Builder + fmt.Fprintf(&b, "%s\n**Delivery** `%s` — **%s** since %s\n\n", viewMarker, d.ID, d.State, d.Since.UTC().Format(time.RFC3339)) + if d.coveredStop() { + fmt.Fprintf(&b, "**Covered**: %s\n\n", d.coveredStopSaid()) + } + if d.HeldWhy != "" && d.State == Held { + fmt.Fprintf(&b, "Held for a person: %s — `mesh-delivery.release` with why lets it go on.\n\n", d.HeldWhy) + } + if d.Plan != nil { + fmt.Fprintf(&b, "**Delivery plan** — %s\n", d.Plan.Summary) + for i, tier := range d.Plan.Tiers { + fmt.Fprintf(&b, "- tier %d: %s\n", i, strings.Join(tier, ", ")) + } + for _, m := range d.Plan.Machines { + var parts []string + if len(m.Receives) > 0 { + parts = append(parts, "receives "+strings.Join(m.Receives, ", ")) + } + if len(m.Waits) > 0 { + parts = append(parts, "waits for a person: "+strings.Join(m.Waits, ", ")) + } + fmt.Fprintf(&b, "- %s: %s\n", m.Machine, strings.Join(parts, "; ")) + } + for _, s := range d.Plan.Steps { + fmt.Fprintf(&b, "- %s\n", s) + } + b.WriteString("\n") + } + if g != nil { + fmt.Fprintf(&b, "**Group** `%s`: %s\n", g.ID, strings.Join(g.Members, ", ")) + if g.Check != nil && g.Check.Verdict != "" { + fmt.Fprintf(&b, "- composed together: %s — %s\n", g.Check.Verdict, g.Check.Summary) + } + b.WriteString("\n") + } + if len(d.Steps) > 0 { + b.WriteString("**Machines**\n") + for _, s := range d.Steps { + fmt.Fprintf(&b, "- %s on %s: %s", s.Module, s.Machine, s.State) + if s.Why != "" { + fmt.Fprintf(&b, " — %s", s.Why) + } + b.WriteString("\n") + } + b.WriteString("\n") + } + b.WriteString("**Transitions**\n") + from := 0 + if len(d.Transitions) > 12 { + from = len(d.Transitions) - 12 + } + for _, t := range d.Transitions[from:] { + fmt.Fprintf(&b, "- %s %s → %s (%s): %s\n", t.At.UTC().Format("2006-01-02 15:04"), orNothing(t.From), t.To, t.Event, t.Why) + } + b.WriteString("\nThe commit's note under `refs/notes/mesh-plan` keeps every transition: `git log --notes=mesh-plan`.\n") + return b.String() +} + +// Flush does what is owed, kept deliveries first: a delivery not yet kept says nothing. +func (h *Holder) Flush() { + type job struct { + id string + group bool + e Effect + body string + repo [2]string + num int + url string + } + h.mu.Lock() + now := h.Now() + for id := range h.dirty { + if d := h.deliveries[id]; d != nil { + h.keep(d) + } + } + for id := range h.dirtyGroup { + if g := h.groups[id]; g != nil { + h.keepGroup(g) + } + } + var jobs []job + ids := make([]string, 0, len(h.deliveries)) + for id := range h.deliveries { + ids = append(ids, id) + } + sort.Strings(ids) + for _, id := range ids { + d := h.deliveries[id] + if h.dirty[id] { + continue + } + for _, e := range d.Owed { + if e.Next != nil && now.Before(*e.Next) { + continue + } + j := job{id: id, e: e, repo: [2]string{d.Owner(), d.Repo()}, num: d.Number, url: d.HTMLURL} + if e.Kind == EffectView { + j.body = ViewBody(d, h.groups[d.Group]) + } + jobs = append(jobs, j) + } + } + for id, g := range h.groups { + if h.dirtyGroup[id] { + continue + } + for _, e := range g.Owed { + if e.Next != nil && now.Before(*e.Next) { + continue + } + jobs = append(jobs, job{id: id, group: true, e: e}) + } + } + h.mu.Unlock() + + for _, j := range jobs { + err := h.do(j.e, j.repo, j.num, j.body, j.url, j.group) + h.mu.Lock() + // Why it is given up, when it is: the forge's word that what it is for does not exist is final, and + // so is a failure tried retryLimit times, the last ones retryCap apart. + dropped := "" + switch { + case err != nil && forgeKinds[j.e.Kind] && doesNotExist(err): + dropped = "the forge answers that what it is for does not exist" + case err != nil && j.e.Tries+1 >= retryLimit: + dropped = fmt.Sprintf("tried %d times since %s", j.e.Tries+1, j.e.Since.UTC().Format(time.RFC3339)) + } + owed := func(list []Effect) []Effect { + for i, o := range list { + if o.Kind == j.e.Kind && o.Commit == j.e.Commit && o.Context == j.e.Context && o.Line == j.e.Line && + o.Since.Equal(j.e.Since) { + if err == nil || dropped != "" { + return append(list[:i:i], list[i+1:]...) + } + list[i].Tries++ + list[i].Last = err.Error() + next := h.Now().Add(retryAfter(list[i].Tries)) + list[i].Next = &next + return list + } + } + return list + } + if j.group { + if g := h.groups[j.id]; g != nil { + g.Owed = owed(g.Owed) + h.keepGroup(g) + } + } else if d := h.deliveries[j.id]; d != nil { + d.Owed = owed(d.Owed) + h.keep(d) + } + // Counted for delivery_status, so a window's writes can be judged against its tries (novox/hq issue 407). + h.tried.last = h.Now() + switch { + case err == nil: + h.tried.done++ + case dropped != "": + h.tried.givenUp++ + default: + h.tried.failed++ + } + if dropped != "" { + h.refuse(fmt.Errorf("%s: its owed %s given up — %s: %v", j.id, j.e.Kind, dropped, err)) + } + h.mu.Unlock() + if err != nil && dropped == "" && j.e.Tries%10 == 0 { + h.Logf("[mesh-delivery] %s: %s not done yet (%v); tried again in %s", j.id, j.e.Kind, err, + retryAfter(j.e.Tries+1)) + } + } + h.owedAgain() +} + +// owedAgain says when what is still owed is tried again: an effect the forge or the bus could not take, at the time +// retryAfter gave it, and a delivery or group not yet kept, after retryFirst. Flush is the bounded retry of both. +func (h *Holder) owedAgain() { + h.mu.Lock() + defer h.mu.Unlock() + for id := range h.dirty { + if h.deliveries[id] != nil { + h.lookAt(h.Now().Add(retryFirst), "a delivery not yet kept, kept again before it is said") + } + } + for id := range h.dirtyGroup { + if h.groups[id] != nil { + h.lookAt(h.Now().Add(retryFirst), "a group not yet kept, kept again before it is said") + } + } + for _, d := range h.deliveries { + for _, e := range d.Owed { + if e.Next != nil { + h.lookAt(*e.Next, "an owed effect the forge or the bus could not take") + } + } + } + for _, g := range h.groups { + for _, e := range g.Owed { + if e.Next != nil { + h.lookAt(*e.Next, "an owed effect the forge or the bus could not take") + } + } + } +} + +// An owed effect that fails is tried again later each time: retryFirst after the first failure, doubling up +// to retryCap, and given up after retryLimit tries — a little over two days — saying so (novox/hq issue 313). +const ( + retryFirst = 10 * time.Second + retryCap = time.Hour + retryLimit = 60 +) + +// retryAfter is how long after its tries-th failure an effect is tried again. +func retryAfter(tries int) time.Duration { + wait := retryFirst + for i := 1; i < tries && wait < retryCap; i++ { + wait *= 2 + } + return min(wait, retryCap) +} + +// forgeKinds are the effects done by the forge's holder. +var forgeKinds = map[string]bool{EffectNote: true, EffectView: true, EffectStatus: true} + +// notThere is the forge's own answer that what was asked of does not exist: a pull request whose number the +// forge no longer holds, a commit it does not know. Tried again it answers the same, for ever. +var notThere = regexp.MustCompile(`(?i)Gitea API \S+: (404\b|\d{3}\b.*\bdoes not exist\b)`) + +func doesNotExist(err error) bool { return err != nil && notThere.MatchString(err.Error()) } + +// do is one owed effect. +func (h *Holder) do(e Effect, repo [2]string, number int, body, url string, group bool) error { + switch e.Kind { + case EffectEmit: + var payload any + if err := json.Unmarshal([]byte(e.Line), &payload); err != nil { + return nil // nothing sayable: dropped + } + event := "transition" + if group { + event = "group" + } + if e.Event != "" { + event = e.Event + } + return h.Emit(event, payload) + case EffectNote: + if h.Forge == nil { + return fmt.Errorf("no forge") + } + return h.Forge.Note(repo[0], repo[1], e.Commit, e.Line) + case EffectView: + if h.Forge == nil || number == 0 { + return nil + } + return h.Forge.View(repo[0], repo[1], number, body) + case EffectStatus: + if h.Forge == nil { + return fmt.Errorf("no forge") + } + if group { + // A group's status names its member's repository and page in Last. + where, page, _ := strings.Cut(e.Last, " ") + ownerRepo, _, _ := strings.Cut(where, "#") + owner, name, _ := strings.Cut(ownerRepo, "/") + return h.Forge.Status(owner, name, e.Commit, e.Context, e.State, e.Line, page) + } + return h.Forge.Status(repo[0], repo[1], e.Commit, e.Context, e.State, e.Line, url) + } + return nil +} diff --git a/modules/mesh-delivery/cmd/mesh-delivery/events.go b/modules/mesh-delivery/cmd/mesh-delivery/events.go new file mode 100644 index 00000000..ac92d25d --- /dev/null +++ b/modules/mesh-delivery/cmd/mesh-delivery/events.go @@ -0,0 +1,110 @@ +package main + +import ( + "fmt" + "slices" + "strings" + + "git.novox.be/novox/mesh-sdk/go/asks" +) + +// Events that start a step (novox/hq ADR 0406 §1, issue 975, ADR 0421 issue 1022). +// +// This owner acts on events: a verdict, a commit status, an Approve. The operator is asked within the event that makes +// the Approve possible (the checks passing), with no label or comment written by anyone: the forge's label and comment +// events start nothing here. + +// ForgeStatusEvent is the forge module's `forge.status`: a commit's status was set. +type ForgeStatusEvent struct { + Owner string `json:"owner"` + Repo string `json:"repo"` + SHA string `json:"sha"` + Context string `json:"context"` + State string `json:"state"` +} + +// wantForge says the forge's open pull requests are to be read at the next look: a check may have answered. +func (h *Holder) wantForge() { + h.mu.Lock() + h.forgeWanted = true + h.mu.Unlock() +} + +// takeForgeWanted answers whether the forge's open pull requests are to be read, and takes the want. +func (h *Holder) takeForgeWanted() bool { + h.mu.Lock() + defer h.mu.Unlock() + wanted := h.forgeWanted + h.forgeWanted = false + return wanted +} + +// AskDecided takes the router's `decided` event for an ask this owner made: its warrant is applied to the deliveries +// the ask covers, and what is said on their pull requests is said. +func (h *Holder) AskDecided(id, state string, w *asks.Warrant) { + h.mu.Lock() + said := h.decided(id, state, w) + h.mu.Unlock() + for _, s := range said { + s() + } +} + +// CommitStatus takes a commit status set on the forge: one of the merge checks (mesh/merge-gate, mesh/repo-check) says a +// check started, answered, or was put back, so the forge's open pull requests are read to see which heads still wait on +// one (SeeOpenPulls). A status of any other context shows nothing the merge needs. +// +// A merge check turning success for the head an approved, ready, not-yet-merged delivery's merge was refused at is +// itself an event that decides whether it may merge (issue 1144): the merge tool may have refused it on a check still +// pending, which this status answers. The merge is asked again at once (mergeAgainNow), under the lock, so the next +// look asks it; no timer does this (issue 1115). A status of another context, another commit or a state other than +// success changes nothing. +func (h *Holder) CommitStatus(e ForgeStatusEvent) { + if !slices.Contains(mergeChecks, e.Context) { + return + } + h.wantForge() + h.mu.Lock() + defer h.mu.Unlock() + if e.State != "success" { + h.withdrawForStatus(e) + return + } + repository := strings.ToLower(e.Owner + "/" + e.Repo) + for _, d := range h.deliveries { + if !strings.EqualFold(d.Repository, repository) || d.Ready == nil || d.Ready.Merged != nil || d.MergedAs != "" || + d.State.Final() { + continue + } + head := d.Commit + if sha := d.Ready.Heads[d.ID]; sha != "" { + head = sha + } + if sameCommit(head, e.SHA) { + h.mergeAgainNow(d) // its approved head's check turned success on the forge: the merge is asked again at once + } + } +} + +// withdrawForStatus takes back the ask of an Approve nobody answered yet when a merge check of a head it covers is no +// longer a success (issue 1275, AC5): the answer to a withdrawn ask merges nothing, and the operator is asked again once +// every member may merge. Under the lock. +func (h *Holder) withdrawForStatus(e ForgeStatusEvent) { + repository := strings.ToLower(e.Owner + "/" + e.Repo) + done := map[string]bool{} + for _, d := range h.deliveries { + r := d.Ready + if !strings.EqualFold(d.Repository, repository) || r == nil || r.AskID == "" || r.Approved != nil || r.Declined != nil || done[r.AskID] { + continue + } + head := d.Commit + if sha := r.Heads[d.ID]; sha != "" { + head = sha + } + if sameCommit(head, e.SHA) { + done[r.AskID] = true + h.cancels = append(h.cancels, h.withdrawAsk(r.AskID, fmt.Sprintf("%s of %s is %s at its head %s", e.Context, d.ID, e.State, shortOf(head, 12)))) + h.Poke() + } + } +} diff --git a/modules/mesh-delivery/cmd/mesh-delivery/events_test.go b/modules/mesh-delivery/cmd/mesh-delivery/events_test.go new file mode 100644 index 00000000..84324b11 --- /dev/null +++ b/modules/mesh-delivery/cmd/mesh-delivery/events_test.go @@ -0,0 +1,266 @@ +package main + +import ( + "encoding/json" + "strings" + "testing" + + stdio "git.novox.be/novox/mesh-sdk/go" +) + +// **A verdict and an Approve are acted on as events** (novox/hq ADR 0406 §1, issue 975, ADR 0421 §1, issue 1022): the +// operator is asked within the event that makes the Approve possible, and the merge within the Approve's event, never at +// the next tick or the next minute, and with no label or comment written by anyone. + +// hear is the runtime handing this owner one event, as handle takes it. +func hear(t *testing.T, w *world, key string, body any) { + t.Helper() + raw, err := json.Marshal(body) + if err != nil { + t.Fatal(err) + } + if err := handle(w.h, stdio.Envelope{Key: key, Body: raw}); err != nil { + t.Fatal(err) + } + w.h.React() // the look the event pokes, taken by the holder's worker +} + +// greenAnnounced is a pull request into main, announced, the operator not answering by himself. +func announced(t *testing.T) *world { + w := newWorld(t) + w.op.auto = false + w.h.PullUpdated(pr("novox/app", 7, readyHead, "feat/x", "src/a.go")) + return w +} + +// Issue 1022, design 47 acceptance test 6: the Approve is asked within the event of the checks passing, with no ready +// mark, label or comment written by anyone. +func TestAPassingVerdictAsksTheOperatorAtOnceWithNoMark(t *testing.T) { + w := announced(t) + if w.op.count() != 0 { + t.Fatalf("asked before the checks passed: %d", w.op.count()) + } + hear(t, w, ControllerSeat+".checked", judgedAt("novox/app", 7, readyHead, "pass", "t1t1t1t1t1t1")) + if w.op.count() != 1 { + t.Fatalf("a pull request whose checks passed asked the operator %d times, wanted once, with no mark", w.op.count()) + } + if len(w.forge.comments) != 0 || len(w.forge.calls) != 0 { + t.Fatalf("something was written on the forge to ask: %v", w.forge.calls) + } + // Asked once for the head, however often it is looked at again. + hear(t, w, "gitea.forge.status", ForgeStatusEvent{Owner: "novox", Repo: "app", SHA: readyHead, Context: "mesh/merge-gate", State: "success"}) + if w.op.count() != 1 { + t.Fatalf("asked %d times for one head", w.op.count()) + } +} + +func TestAFailedVerdictAsksNothing(t *testing.T) { + w := announced(t) + hear(t, w, ControllerSeat+".checked", judgedAt("novox/app", 7, readyHead, "fail", "t1t1t1t1t1t1")) + if w.op.count() != 0 { + t.Fatalf("a pull request whose checks failed asked the operator %d times", w.op.count()) + } +} + +// approve is the operator's Approve of the one open ask, heard as the router's `decided` event. +func approve(w *world) { + open := w.op.open() + w.op.answer(open[0], OptionApprove) + state, warrant, _ := w.op.Where(open[0]) + w.h.AskDecided(open[0], state, warrant) + w.h.React() +} + +func TestAnApproveHeardAsAnEventMergesAtOnceAtTheApprovedHead(t *testing.T) { + w := announced(t) + hear(t, w, ControllerSeat+".checked", judgedAt("novox/app", 7, readyHead, "pass", "t1t1t1t1t1t1")) + if len(w.op.open()) != 1 || len(w.forge.merges) != 0 { + t.Fatalf("asks %v, merges %v", w.op.open(), w.forge.merges) + } + approve(w) + if len(w.forge.merges) != 1 || w.forge.merges[0] != "novox/app#7@"+readyHead { + t.Fatalf("the Approve, heard as an event, merged %v; wanted once at the approved head", w.forge.merges) + } + if got := w.forge.approvedHeads[0]; len(got) != 1 || got[0] != "novox/app@"+readyHead { + t.Fatalf("the merge tool was not handed the approved heads: %v", got) + } +} + +// Issue 1022 (ADR 0421 §3): a passing verdict that arrives out of date is not asked; its check against the trunk's +// head is, and the Approve is asked on the passing verdict. +func TestAStalePassingVerdictAsksTheCheckAgainBeforeTheOperator(t *testing.T) { + w := announced(t) + // Another merge moved the trunk before the verdict, judged against the trunk before, arrived. + w.h.PullMerged(merged("novox/app", 9, "9999999999999999", "t2t2t2t2t2t2")) + hear(t, w, ControllerSeat+".checked", judgedAt("novox/app", 7, readyHead, "pass", "t1t1t1t1t1t1")) + if w.op.count() != 0 { + t.Fatalf("the operator was asked for a verdict out of date") + } + if len(w.ctl.checks) != 1 || w.ctl.baseSHAs[0] != "t2t2t2t2t2t2" { + t.Fatalf("its check against the trunk's head was not asked once: %v %v", w.ctl.checks, w.ctl.baseSHAs) + } + hear(t, w, ControllerSeat+".checked", judgedAt("novox/app", 7, readyHead, "pass", "t2t2t2t2t2t2")) + if w.op.count() != 1 { + t.Fatalf("the passing verdict at the trunk's head asked the operator %d times, wanted once", w.op.count()) + } + if len(w.ctl.checks) != 1 { + t.Fatalf("its check was asked again: %v", w.ctl.checks) + } +} + +// Issue 1022 (ADR 0421 §3, §4): an ask raised stays when the trunk moves; the Approve is given; the merge waits for the +// fresh verdict and lands on it; no second Approve is asked. +func TestAnAskRaisedStaysThroughATrunkMoveAndTheMergeLandsOnTheFreshVerdict(t *testing.T) { + w := announced(t) + id := IDOf("novox/app", readyHead) + hear(t, w, ControllerSeat+".checked", judgedAt("novox/app", 7, readyHead, "pass", "t1t1t1t1t1t1")) + w.h.PullMerged(merged("novox/app", 9, "9999999999999999", "t2t2t2t2t2t2")) + w.h.React() + if w.state(id) != Proposed || w.op.count() != 1 || len(w.op.open()) != 1 { + t.Fatalf("after the trunk moved: %s, asks %d, open %v", w.state(id), w.op.count(), w.op.open()) + } + if len(w.ctl.checks) != 1 { + t.Fatalf("its check against the trunk's head was not asked at once: %v", w.ctl.checks) + } + approve(w) + if len(w.forge.merges) != 0 { + t.Fatalf("merged on a verdict out of date: %v", w.forge.merges) + } + hear(t, w, ControllerSeat+".checked", judgedAt("novox/app", 7, readyHead, "pass", "t2t2t2t2t2t2")) + if len(w.forge.merges) != 1 || w.delivery(id).Ready.Merged == nil || w.op.count() != 1 { + t.Fatalf("the passing verdict did not ask the merge at once, on the same Approve: %v %+v asks %d", w.forge.merges, + w.delivery(id).Ready, w.op.count()) + } +} + +// Issue 1022: an Approve whose merge the tool refused is asked again at once on the verdict that decides it. +func TestAPassingVerdictAsksTheMergeAtOnceEvenJustAfterARefusal(t *testing.T) { + w := announced(t) + id := IDOf("novox/app", readyHead) + hear(t, w, ControllerSeat+".checked", judgedAt("novox/app", 7, readyHead, "pass", "t1t1t1t1t1t1")) + w.forge.refuse = errAskedAgain + approve(w) + if len(w.forge.merges) != 1 { + t.Fatalf("the merge tool was asked %v; wanted once", w.forge.merges) + } + w.forge.refuse = nil + w.h.PullMerged(merged("novox/app", 9, "9999999999999999", "t2t2t2t2t2t2")) + w.h.PullUpdated(pr("novox/app", 7, readyHead, "feat/x", "src/a.go")) + hear(t, w, ControllerSeat+".checked", judgedAt("novox/app", 7, readyHead, "pass", "t2t2t2t2t2t2")) + if len(w.forge.merges) != 2 || w.delivery(id).Ready.Merged == nil { + t.Fatalf("the passing verdict did not ask the merge at once: %v %+v", w.forge.merges, w.delivery(id).Ready) + } +} + +var errAskedAgain = &askedAgainError{} + +type askedAgainError struct{} + +func (*askedAgainError) Error() string { + return "the forge refused gitea_merge_pull_request: novox/app#7 is not merged: its verdict was judged against trunk aaaaaaaaaaaa, and main is now bbbbbbbbbbbb. The check of head cccccccccccc merged into trunk bbbbbbbbbbbb was asked; took 3s. Ask again at cccccccccccc once both statuses pass. Nothing was merged" +} + +// Issue 1022 (ADR 0421 §5): the review record is read and put in the ask; it is never a gate. +func TestTheAskSaysFirstWhenThePullRequestIsNotReviewedAtThisHead(t *testing.T) { + for _, c := range []struct{ name, body, want string }{ + {"no line", "Fixes issue 7.", "not reviewed at this head"}, + {"another head", "Reviewed-by: lead@shanks at " + pushed, "the Reviewed-by line names " + pushed[:12]}, + } { + t.Run(c.name, func(t *testing.T) { + w := newWorld(t) + w.op.auto = false + p := pr("novox/app", 7, readyHead, "feat/x", "src/a.go") + p.Body = c.body + w.h.PullUpdated(p) + hear(t, w, ControllerSeat+".checked", judgedAt("novox/app", 7, readyHead, "pass", "t1t1t1t1t1t1")) + if w.op.count() != 1 { + t.Fatalf("an unreviewed pull request must still be asked: %d", w.op.count()) + } + a := w.op.ask(w.op.open()[0]) + first := strings.SplitN(a.Explanation, "\n", 2)[0] + if !strings.Contains(first, "not reviewed at this head") || !strings.Contains(a.Explanation, c.want) { + t.Fatalf("the ask does not say first that it is not reviewed at this head:\n%s", a.Explanation) + } + }) + } +} + +func TestTheAskNamesTheReviewerOfTheHead(t *testing.T) { + w := newWorld(t) + w.op.auto = false + p := pr("novox/app", 7, readyHead, "feat/x", "src/a.go") + p.Body = "Fixes issue 7.\n\nReviewed-by: lead@shanks at " + readyHead + "\n" + w.h.PullUpdated(p) + hear(t, w, ControllerSeat+".checked", judgedAt("novox/app", 7, readyHead, "pass", "t1t1t1t1t1t1")) + a := w.op.ask(w.op.open()[0]) + if strings.Contains(a.Explanation, "not reviewed") || !strings.Contains(a.Explanation, "reviewed by lead") { + t.Fatalf("a reviewed head is not said reviewed:\n%s", a.Explanation) + } +} + +// The description corrected without a push is read again. +func TestTheReviewLineCorrectedInTheDescriptionIsReadWithoutAPush(t *testing.T) { + w := newWorld(t) + w.op.auto = false + p := pr("novox/app", 7, readyHead, "feat/x", "src/a.go") + w.h.PullUpdated(p) + p.Body = "Reviewed-by: lead@shanks at " + readyHead + w.h.PullUpdated(p) + if d := w.delivery(IDOf("novox/app", readyHead)); d.ReviewedBy != "lead@shanks" || d.ReviewedAt != readyHead { + t.Fatalf("the corrected line is not read: %q %q", d.ReviewedBy, d.ReviewedAt) + } +} + +// Issue 1022 (ADR 0421 §6): a push ends the Approve and says so once on the pull request. +func TestAPushAfterTheAskEndsItAndSaysSoOnceOnThePullRequest(t *testing.T) { + w := announced(t) + hear(t, w, ControllerSeat+".checked", judgedAt("novox/app", 7, readyHead, "pass", "t1t1t1t1t1t1")) + old := w.op.open()[0] + w.h.PullUpdated(pr("novox/app", 7, pushed, "feat/x", "src/a.go")) + w.h.React() + w.h.React() + if len(w.op.cancelled) != 1 || w.op.cancelled[0] != old || len(w.op.open()) != 0 { + t.Fatalf("the ask was not cancelled: %v, open %v", w.op.cancelled, w.op.open()) + } + if len(w.forge.comments) != 1 || !strings.Contains(w.forge.comments[0], "Approve") || + !strings.Contains(w.forge.comments[0], "ended") { + t.Fatalf("the end of the Approve is not said once on the pull request: %v", w.forge.comments) + } + // The new head is asked on its own when its checks pass. + hear(t, w, ControllerSeat+".checked", judgedAt("novox/app", 7, pushed, "pass", "t1t1t1t1t1t1")) + if open := w.op.open(); len(open) != 1 || open[0] == old { + t.Fatalf("the new head was not asked afresh: %v", open) + } + if len(w.forge.comments) != 1 { + t.Fatalf("said more than once: %v", w.forge.comments) + } +} + +// Issue 1022 (ADR 0421 §7): a check that failed after the ask merges nothing and is said once and as a stalled line. +func TestACheckThatFailsAfterTheAskIsNotMergedAndIsSaid(t *testing.T) { + w := announced(t) + id := IDOf("novox/app", readyHead) + hear(t, w, ControllerSeat+".checked", judgedAt("novox/app", 7, readyHead, "pass", "t1t1t1t1t1t1")) + if _, err := w.h.Recheck(id, "a second look", "tester"); err != nil { + t.Fatal(err) + } + hear(t, w, ControllerSeat+".checked", judgedAt("novox/app", 7, readyHead, "fail", "t1t1t1t1t1t1")) + approve(w) + if len(w.forge.merges) != 0 { + t.Fatalf("merged past a failed check: %v", w.forge.merges) + } + w.h.React() + said := 0 + for _, c := range w.forge.comments { + if strings.Contains(c, "not merged") { + said++ + } + } + if said != 1 { + t.Fatalf("the failure is not said once: %v", w.forge.comments) + } + lines := w.h.Stalled() + if len(lines) != 1 || lines[0].ID != id || strings.Contains(lines[0].Says, "marked") { + t.Fatalf("the failure is not a stalled line: %+v", lines) + } +} diff --git a/modules/mesh-delivery/cmd/mesh-delivery/fakes_test.go b/modules/mesh-delivery/cmd/mesh-delivery/fakes_test.go new file mode 100644 index 00000000..e04f0196 --- /dev/null +++ b/modules/mesh-delivery/cmd/mesh-delivery/fakes_test.go @@ -0,0 +1,733 @@ +package main + +import ( + "encoding/json" + "errors" + "fmt" + "sort" + "strings" + "sync" + "testing" + "time" +) + +// What a test stands the holder on: a store in memory that can be made to fail, a controller that records +// what it was asked and answers what it is told, a forge that records every note, view and status. + +type memStore struct { + mu sync.Mutex + deliveries map[string][]byte + groups map[string][]byte + failing bool + // puts counts the deliveries written, failed or not. + puts int +} + +func newMemStore() *memStore { + return &memStore{deliveries: map[string][]byte{}, groups: map[string][]byte{}} +} + +func (s *memStore) PutDelivery(d *Delivery) error { + s.mu.Lock() + defer s.mu.Unlock() + s.puts++ + if s.failing { + return errors.New("the bus is away") + } + s.deliveries[d.ID] = mustJSON(d) + return nil +} +func (s *memStore) DeleteDelivery(id string) error { + s.mu.Lock() + defer s.mu.Unlock() + delete(s.deliveries, id) + return nil +} +func (s *memStore) PutGroup(g *Group) error { + s.mu.Lock() + defer s.mu.Unlock() + if s.failing { + return errors.New("the bus is away") + } + s.groups[g.ID] = mustJSON(g) + return nil +} +func (s *memStore) DeleteGroup(id string) error { + s.mu.Lock() + defer s.mu.Unlock() + delete(s.groups, id) + return nil +} +func (s *memStore) Deliveries() ([]*Delivery, error) { + s.mu.Lock() + defer s.mu.Unlock() + var out []*Delivery + for _, raw := range s.deliveries { + var d Delivery + if err := json.Unmarshal(raw, &d); err != nil { + return nil, err + } + out = append(out, &d) + } + return out, nil +} +func (s *memStore) Groups() ([]*Group, error) { + s.mu.Lock() + defer s.mu.Unlock() + var out []*Group + for _, raw := range s.groups { + var g Group + if err := json.Unmarshal(raw, &g); err != nil { + return nil, err + } + out = append(out, &g) + } + return out, nil +} + +type fakeController struct { + mu sync.Mutex + walks map[string]Walk + held bool + checks []string + asked int + delivers []string + // backupsAgain are the walks whose failed backups were asked again. + backupsAgain []string + // refuseBackupsAgain refuses the word to take a walk's backups again. + refuseBackupsAgain bool + stops []string + // withouts are the stops that left merges out (novox/hq ADR 0299): " without ,…". + withouts []string + // refuseWithout refuses every stop with merges to leave out, as a controller before ADR 0299 does; mergedAt are + // the merges' times, by commit, for what a later merge contains. + refuseWithout bool + mergedAt map[string]time.Time + // refuseStop refuses every whole stop too; moves are the modules each merge moves, by commit, for what a left-out + // merge shares (ADR 0299). + refuseStop bool + moves map[string][]string + plans int + down bool + // emptyCheck makes Check answer with no id and no error, as the port does when the answer holds none. + emptyCheck bool + // walkReads counts the reads of the controller's walks (Walks). + walkReads int + // checkReady is the ready flag of each member of each check asked. + checkReady []bool + // baseSHAs is the base head each member of each check asked was sent with. + baseSHAs []string + // refuseDeliver refuses every word, as a controller that will not take it does (review of novox/hq issue 753). + refuseDeliver bool + // onCheck runs as a check is asked, before it is answered: what the forge held at that moment. + onCheck func() + // planned is what Plan answers instead of "builds app", when set. + planned *DeliveryPlan + // forgot are the pull requests whose check builds were let go of (novox/hq ADR 0350 §5), "owner/repo#n". + forgot []string +} + +// BackupsAgain records the word, and clears the walk's failed backups as the controller does. +func (c *fakeController) BackupsAgain(walk, why string) error { + c.mu.Lock() + defer c.mu.Unlock() + if c.down { + return errors.New("down") + } + if c.refuseBackupsAgain { + return errors.New("the controller refuses the backups") + } + w, known := c.walks[walk] + if !known || w.Backups == nil || w.Backups.State != "failed" { + return fmt.Errorf("%s holds no failed backup", walk) + } + w.Backups = nil + w.Revision++ + c.walks[walk] = w + c.backupsAgain = append(c.backupsAgain, walk) + return nil +} + +func (c *fakeController) ForgetChecks(owner, repo string, number int) error { + c.mu.Lock() + defer c.mu.Unlock() + if c.down { + return errors.New("down") + } + c.forgot = append(c.forgot, fmt.Sprintf("%s/%s#%d", owner, repo, number)) + return nil +} + +func newFakeController() *fakeController { + return &fakeController{walks: map[string]Walk{}, held: true, mergedAt: map[string]time.Time{}, + moves: map[string][]string{}} +} + +func (c *fakeController) Plan(repository, base, head string, paths, dirs, removed []string) (*DeliveryPlan, error) { + c.mu.Lock() + defer c.mu.Unlock() + c.plans++ + if c.down { + return nil, errors.New("down") + } + if c.planned != nil { + p := *c.planned + return &p, nil + } + return &DeliveryPlan{Repository: repository, Base: base, Head: head, Moved: []string{"app"}, Summary: "builds app"}, nil +} +func (c *fakeController) Check(group string, ms []Member) (string, error) { + c.mu.Lock() + defer c.mu.Unlock() + if c.down { + return "", errors.New("down") + } + if c.onCheck != nil { + c.onCheck() + } + c.asked++ + id := fmt.Sprintf("check-%d", c.asked) + if c.emptyCheck { + id = "" + } + var heads []string + for _, m := range ms { + heads = append(heads, m.ID) + } + c.checks = append(c.checks, group+":"+strings.Join(heads, ",")) + for _, m := range ms { + c.checkReady = append(c.checkReady, m.Ready) + c.baseSHAs = append(c.baseSHAs, m.BaseSHA) + } + return id, nil +} +func (c *fakeController) Deliver(walk, why string) error { + c.mu.Lock() + defer c.mu.Unlock() + if c.down { + return errors.New("down") + } + if c.refuseDeliver { + return errors.New("the controller refuses the word") + } + w, known := c.walks[walk] + if !known || w.Delivery == nil || w.Delivery.Go != nil { + return fmt.Errorf("%s waits for nobody", walk) + } + now := time.Now() + w.Delivery.Go, w.Delivery.By, w.Delivery.Why = &now, byOwner, why + w.Revision++ + c.walks[walk] = w + c.delivers = append(c.delivers, walk) + return nil +} +func (c *fakeController) Stop(walk, why, by string) error { + c.mu.Lock() + defer c.mu.Unlock() + if c.down { + return errors.New("down") + } + c.stops = append(c.stops, walk) + if c.refuseStop { + return errors.New("the controller refuses") + } + if w, known := c.walks[walk]; known && !w.Open() { + // As the controller: only a walk that is building or rolling is stopped (`delivery-stop`). + return fmt.Errorf("%s is already %s", walk, w.State) + } else if known { + w.State = walkFailed + if w.Delivery == nil { + w.Delivery = &WalkDelivery{} + } + w.Delivery.Stopped, w.Delivery.StoppedWhy = by, why + w.Revision++ + c.walks[walk] = w + } + return nil +} + +// StopWithout ends a walk without the merges named, as the controller does (novox/hq ADR 0299): each merge named, +// and each later merge of its repository by merge time (mergedAt), is left out; the others are put in a batch, +// "-again", assembling until a test cuts it. Given a batch, the merges left out go to a stopped record, +// "-left", and the batch goes on with the others. refuseWithout refuses every such stop, as an older +// controller does. +func (c *fakeController) StopWithout(walk, why, by string, without []string) (StopAnswer, error) { + c.mu.Lock() + defer c.mu.Unlock() + if c.down { + return StopAnswer{}, errors.New("down") + } + c.withouts = append(c.withouts, walk+" without "+strings.Join(without, ",")) + if c.refuseWithout { + return StopAnswer{}, errors.New(`delivery-stop takes no argument "without"`) + } + w, known := c.walks[walk] + if !known || !(w.Open() || w.IsBatch()) { + return StopAnswer{}, fmt.Errorf("%s is no open walk or batch", walk) + } + wd := WalkDelivery{} + if w.Delivery != nil { + wd = *w.Delivery + } + named := func(m WalkMerge) bool { return containsString(without, m.Repository+"@"+m.Commit) } + contains := func(m WalkMerge) string { + for _, s := range wd.Merges { + if named(s) && s.Repository == m.Repository && c.mergedAt[m.Commit].After(c.mergedAt[s.Commit]) { + return s.Commit + } + } + return "" + } + shares := func(m WalkMerge) (string, []string) { + for _, s := range wd.Merges { + if named(s) { + var both []string + for _, x := range c.moves[m.Commit] { + if containsString(c.moves[s.Commit], x) { + both = append(both, x) + } + } + if len(both) > 0 { + return s.Commit, both + } + } + } + return "", nil + } + a := StopAnswer{Stopped: walk} + var again, left []WalkMerge + for _, m := range wd.Merges { + s, both := shares(m) + switch { + case named(m): + a.LeftOut = append(a.LeftOut, WalkLeftOut{Repository: m.Repository, Commit: m.Commit}) + left = append(left, m) + case len(both) > 0: + a.LeftOut = append(a.LeftOut, WalkLeftOut{Repository: m.Repository, Commit: m.Commit, Contains: s, Shares: both}) + left = append(left, m) + case contains(m) != "": + a.LeftOut = append(a.LeftOut, WalkLeftOut{Repository: m.Repository, Commit: m.Commit, Contains: contains(m)}) + left = append(left, m) + default: + m.Carried = "" + again = append(again, m) + a.WalkedAgain = append(a.WalkedAgain, m.Repository+"@"+m.Commit) + } + } + if len(a.LeftOut) == 0 { + return StopAnswer{}, fmt.Errorf("%s answers no merge %s", walk, strings.Join(without, ", ")) + } + if w.IsBatch() && len(again) == 0 { + // Every merge left out: the batch itself is stopped, as the controller does. + wd.Stopped, wd.StoppedWhy, wd.LeftOut, wd.Batch = by, why, a.LeftOut, nil + w.Delivery, w.State = &wd, walkFailed + w.Revision++ + c.walks[walk] = w + return a, nil + } + if w.IsBatch() { + rid := walk + "-left" + c.walks[rid] = Walk{ID: rid, Repository: left[0].Repository, Branch: "main", Commit: left[0].Commit, + Created: w.Created, State: walkFailed, Revision: 1, Modules: map[string]*WalkModule{}, + Delivery: &WalkDelivery{Stopped: by, StoppedWhy: why, Merges: left, LeftOut: a.LeftOut, WalkedAgainBy: walk}} + wd.Merges = again + w.Delivery = &wd + w.Repository, w.Commit = again[0].Repository, again[0].Commit + w.Revision++ + c.walks[walk] = w + a.Stopped, a.WalkedAgainBy = rid, walk + return a, nil + } + if len(again) > 0 { + id := walk + "-again" + closes := w.Created.Add(time.Minute) + c.walks[id] = Walk{ID: id, Repository: again[0].Repository, Branch: "main", Commit: again[0].Commit, + Created: w.Created.Add(time.Minute), State: walkAssembling, Revision: 1, + Modules: map[string]*WalkModule{}, Delivery: &WalkDelivery{Merges: again, Batch: &WalkBatch{ClosesAt: &closes}}} + wd.WalkedAgainBy, a.WalkedAgainBy = id, id + } + wd.Stopped, wd.StoppedWhy, wd.LeftOut = by, why, a.LeftOut + w.Delivery = &wd + w.State = walkFailed + w.Note = "stopped by " + by + ": " + why + w.Revision++ + c.walks[walk] = w + return a, nil +} + +// cut makes a batch the walk under the same id, waiting for its word, as the controller cuts it. +func (c *fakeController) cut(id string) Walk { + c.mu.Lock() + w := c.walks[id] + c.mu.Unlock() + wd := *w.Delivery + wd.Batch, wd.Awaits = nil, "mesh-delivery" + w.Delivery = &wd + w.State, w.Tiers, w.Modules = walkBuilding, [][]string{{"app"}}, map[string]*WalkModule{"app": {}} + return c.put(w) +} + +func (c *fakeController) Walks(walk string) (bool, []Walk, error) { + c.mu.Lock() + defer c.mu.Unlock() + c.walkReads++ + if c.down { + return false, nil, errors.New("down") + } + var out []Walk + for id, w := range c.walks { + if walk == "" || walk == id { + out = append(out, w) + } + } + sort.Slice(out, func(i, j int) bool { return out[i].ID < out[j].ID }) + return c.held, out, nil +} + +// put is the controller keeping a walk; it answers the walk as `plan-moved` would say it. +func (c *fakeController) put(w Walk) Walk { + c.mu.Lock() + defer c.mu.Unlock() + if old, known := c.walks[w.ID]; known { + w.Revision = old.Revision + 1 + } else if w.Revision == 0 { + w.Revision = 1 + } + c.walks[w.ID] = w + return w +} + +func (c *fakeController) walk(id string) Walk { + c.mu.Lock() + defer c.mu.Unlock() + return c.walks[id] +} + +type fakeForge struct { + mu sync.Mutex + notes map[string][]string + views map[string]string + statuses map[string]string + // statusWrites counts the statuses set. + statusWrites int + down bool + // viewErr is what View answers instead, when set: a pull request the forge no longer holds. + viewErr error + viewCalls int + // What Statuses reads: the pull requests by owner/repo#number, the base's required statuses, whether the + // protection cannot be read, and what it was asked. + pulls map[string]ForgePull + required []string + protectionDown bool + asked []string + // contains answers gitea_contains by " in "; a pair it does not hold is a forge that cannot say. + contains map[string]bool + containsDown bool + containsAsks []string + // open is what OpenPulls answers; openDown a forge that cannot say. + open []OpenPull + openDown bool + openCalls int + // merges and comments are what was asked of the forge for an approved merge (novox/hq issue 722); approvedHeads the + // heads each merge was handed as approved (ADR 0421 §9). + merges, comments []string + approvedHeads [][]string + refuse error + // refuseRepo refuses the merges of one repository only ("owner/name"), beside refuse (issue 1049). + refuseRepo map[string]error + // calls are the comments in the order asked. + calls []string +} + +func (f *fakeForge) OpenPulls() ([]OpenPull, error) { + f.mu.Lock() + defer f.mu.Unlock() + f.openCalls++ + if f.openDown { + return nil, errors.New("the forge is away") + } + return append([]OpenPull(nil), f.open...), nil +} + +func newFakeForge() *fakeForge { + return &fakeForge{notes: map[string][]string{}, views: map[string]string{}, statuses: map[string]string{}, + pulls: map[string]ForgePull{}, contains: map[string]bool{}} +} + +func (f *fakeForge) Note(owner, repo, commit, line string) error { + f.mu.Lock() + defer f.mu.Unlock() + if f.down { + return errors.New("the forge is away") + } + key := owner + "/" + repo + "@" + commit + for _, l := range f.notes[key] { + if l == line { + return nil // appended once + } + } + f.notes[key] = append(f.notes[key], line) + return nil +} +func (f *fakeForge) View(owner, repo string, number int, body string) error { + f.mu.Lock() + defer f.mu.Unlock() + f.viewCalls++ + if f.viewErr != nil { + return f.viewErr + } + if f.down { + return errors.New("the forge is away") + } + f.views[fmt.Sprintf("%s/%s#%d", owner, repo, number)] = body + return nil +} +func (f *fakeForge) Status(owner, repo, commit, context, state, description, target string) error { + f.mu.Lock() + defer f.mu.Unlock() + if f.down { + return errors.New("the forge is away") + } + f.statusWrites++ + f.statuses[owner+"/"+repo+"@"+commit+" "+context] = state + " " + description + " → " + target + return nil +} + +// Statuses answers what the fake was told: a pull request's head from pulls, the statuses set on the commit +// (those Status set too, as the forge would keep them), and the base's required statuses from required. +func (f *fakeForge) Statuses(owner, repo string, number int, commit, base string) (*ForgeChecks, error) { + f.mu.Lock() + defer f.mu.Unlock() + if f.down { + return nil, errors.New("the forge is away") + } + f.asked = append(f.asked, fmt.Sprintf("%s/%s#%d@%s base=%s", owner, repo, number, commit, base)) + out := &ForgeChecks{} + if number > 0 { + if p, ok := f.pulls[fmt.Sprintf("%s/%s#%d", owner, repo, number)]; ok { + out.Pull = &p + if commit == "" { + commit = p.HeadSHA + } + if base == "" { + base = p.Base + } + } else if commit == "" { + return nil, fmt.Errorf("no pull request %d", number) + } + } + for key, said := range f.statuses { + at, context, _ := strings.Cut(key, " ") + if !strings.HasPrefix(at, owner+"/"+repo+"@") || !sameCommit(strings.TrimPrefix(at, owner+"/"+repo+"@"), commit) { + continue + } + if len(strings.TrimPrefix(at, owner+"/"+repo+"@")) > len(commit) { + commit = strings.TrimPrefix(at, owner+"/"+repo+"@") + } + state, rest, _ := strings.Cut(said, " ") + description, link, _ := strings.Cut(rest, " → ") + out.Statuses = append(out.Statuses, ForgeStatus{Context: context, State: state, Description: description, + TargetURL: link, UpdatedAt: "2026-10-06T12:00:00Z", Creator: "mesh-admin"}) + } + sort.Slice(out.Statuses, func(i, j int) bool { return out.Statuses[i].Context < out.Statuses[j].Context }) + out.Commit = commit + if base == "" { + base = "main" + } + out.Merge = ForgeMerge{Branch: base, Protected: f.required != nil, Required: f.required} + if f.protectionDown { + out.Merge.Says = "the protection of " + base + " could not be read" + return out, nil + } + ok := true + for _, r := range f.required { + state := "missing" + for _, st := range out.Statuses { + if st.Context == r { + state = st.State + } + } + if state != "success" { + ok = false + out.Merge.Blocking = append(out.Merge.Blocking, struct { + Context string `json:"context"` + State string `json:"state"` + }{r, state}) + } + } + out.Merge.Mergeable = &ok + out.Merge.Says = fmt.Sprintf("%s requires %v", base, f.required) + return out, nil +} + +// Contains answers what the fake was told of two commits. +func (f *fakeForge) Contains(owner, repo, commit, in string) (bool, error) { + f.mu.Lock() + defer f.mu.Unlock() + f.containsAsks = append(f.containsAsks, commit+" in "+in) + if f.containsDown || f.down { + return false, errors.New("the forge is away") + } + yes, said := f.contains[commit+" in "+in] + if !said { + return false, fmt.Errorf("the forge has no commit %s or %s", commit, in) + } + return yes, nil +} + +func (f *fakeForge) notesOn(commit string) []string { + f.mu.Lock() + defer f.mu.Unlock() + for key, lines := range f.notes { + if strings.HasSuffix(key, "@"+commit) { + return append([]string(nil), lines...) + } + } + return nil +} + +// world is a holder over fakes, with a clock a test moves. +type world struct { + t *testing.T + h *Holder + store *memStore + ctl *fakeController + forge *fakeForge + op *fakeOperator + now time.Time + said []map[string]any + mu sync.Mutex +} + +func newWorld(t *testing.T) *world { + return newWorldOver(t, newMemStore(), newFakeController(), newFakeForge(), time.Date(2026, 10, 6, 12, 0, 0, 0, time.UTC)) +} + +func newWorldOver(t *testing.T, store *memStore, ctl *fakeController, forge *fakeForge, now time.Time) *world { + w := &world{t: t, store: store, ctl: ctl, forge: forge, now: now} + w.h = &Holder{Store: store, Controller: ctl, Forge: forge, + Now: func() time.Time { w.mu.Lock(); defer w.mu.Unlock(); return w.now }, + Logf: func(format string, a ...any) { t.Logf(format, a...) }, + Emit: func(event string, body any) error { + w.mu.Lock() + defer w.mu.Unlock() + var m map[string]any + _ = json.Unmarshal(mustJSON(body), &m) + m["event-kind"] = event + w.said = append(w.said, m) + return nil + }} + // The operator approves at once unless a test says otherwise (w.op.auto = false): most tests are not about the Approve. + w.op = newFakeOperator(w.h.Now) + w.op.auto = true + w.h.Operator = w.op + w.h.init() + if err := w.h.Load(); err != nil { + t.Fatal(err) + } + return w +} + +func (w *world) later(d time.Duration) { + w.mu.Lock() + w.now = w.now.Add(d) + w.mu.Unlock() +} + +// settleAll lets the router's answers reach the holder as its `decided` events, and takes a look, a few times. +func (w *world) settleAll() { + for range 4 { + w.decide() + w.h.React() + } +} + +// decide hands the holder the router's `decided` event for every ask of its that has closed. +func (w *world) decide() { + w.h.mu.Lock() + ids := w.h.pendingAsks() + w.h.mu.Unlock() + for _, id := range ids { + state, warrant, _ := w.op.Where(id) + if state == "" || state == "open" || state == "held" { + continue + } + w.h.AskDecided(id, state, warrant) + } +} + +func (w *world) delivery(id string) *Delivery { + w.t.Helper() + w.h.mu.Lock() + defer w.h.mu.Unlock() + d := w.h.deliveries[id] + if d == nil { + w.t.Fatalf("no delivery %s; there are %v", id, w.ids()) + } + c := *d + return &c +} + +func (w *world) ids() []string { + var out []string + for id := range w.h.deliveries { + out = append(out, id) + } + sort.Strings(out) + return out +} + +func (w *world) state(id string) State { return w.delivery(id).State } + +// pr is a pull request's head as the forge announces it. +func pr(repo string, number int, sha, branch string, paths ...string) PullEvent { + owner, name, _ := strings.Cut(repo, "/") + return PullEvent{Owner: owner, Repo: name, Number: number, Base: "main", Head: branch, HeadSHA: sha, + HTMLURL: fmt.Sprintf("https://forge.invalid/%s/pulls/%d", repo, number), Paths: paths} +} + +// verdict is the controller's `checked` for a head. +func verdict(repo string, number int, sha, gate string) CheckedEvent { + owner, name, _ := strings.Cut(repo, "/") + c := CheckedEvent{Owner: owner, Repo: name, Number: number, Commit: sha, Verdict: gate, Summary: "the gate said " + gate, + ID: "check-of-" + sha, Plan: &DeliveryPlan{Repository: repo, Moved: []string{"app"}, Summary: "builds app → anchor"}} + return c +} + +// merged is the forge's `pull.merged`. +func merged(repo string, number int, head, merge string) PullEvent { + p := pr(repo, number, head, "feature") + p.MergeCommit = merge + p.MergedAt = "2026-10-06T12:30:00Z" + return p +} + +// aWalk is the controller's walk of a merge commit, waiting for its delivery's word. +func aWalk(id, repo, commit string, waits bool) Walk { + w := Walk{ID: id, Repository: repo, Branch: "main", Commit: commit, Created: time.Date(2026, 10, 6, 12, 31, 0, 0, time.UTC), + State: walkBuilding, Tiers: [][]string{{"app"}}, Modules: map[string]*WalkModule{"app": {}}} + if waits { + w.Delivery = &WalkDelivery{Awaits: "mesh-delivery"} + } + return w +} + +// Merge and Comment record what was asked (novox/hq issue 722); Merge answers refuse when set. +func (f *fakeForge) Merge(owner, repo string, number int, head string, approved []string) error { + f.mu.Lock() + defer f.mu.Unlock() + f.merges = append(f.merges, fmt.Sprintf("%s/%s#%d@%s", owner, repo, number, head)) + f.approvedHeads = append(f.approvedHeads, approved) + if err := f.refuseRepo[owner+"/"+repo]; err != nil { + return err + } + return f.refuse +} + +func (f *fakeForge) Comment(owner, repo string, number int, body string) error { + f.mu.Lock() + defer f.mu.Unlock() + f.comments = append(f.comments, fmt.Sprintf("%s/%s#%d: %s", owner, repo, number, body)) + f.calls = append(f.calls, "comment "+body) + return nil +} diff --git a/modules/mesh-delivery/cmd/mesh-delivery/group_merge.go b/modules/mesh-delivery/cmd/mesh-delivery/group_merge.go new file mode 100644 index 00000000..049d8179 --- /dev/null +++ b/modules/mesh-delivery/cmd/mesh-delivery/group_merge.go @@ -0,0 +1,229 @@ +package main + +import ( + "fmt" + "strings" + "time" +) + +// A group is merged whole or not at all, in its order (issue 1275, novox/hq ADR 0418). +// +// - Before the operator is asked, and again before any member is merged, every member's required statuses are read at +// its head on the forge; one that is not a success holds the group, and nothing is merged. +// - The members are merged one after the other in the group's order, and the pass stops at the first refusal: no later +// member is merged. +// - A group held or half merged is said as a stalled line, urgent, until every member is merged or a person decides. + +// memberRead is what a status read of one member needs, taken under the lock. +type memberRead struct { + id, owner, repo, base, head string + number int +} + +// readOf is a delivery as a status read needs it, at the head the Approve or the ask covers. +func readOf(d *Delivery, head string) memberRead { + if head == "" { + head = d.Commit + } + return memberRead{id: d.ID, owner: d.Owner(), repo: d.Repo(), base: d.Base, head: head, number: d.Number} +} + +// notAsSuccess reads a member's required statuses at its head and says, in the forge's words, which are not a success; +// empty when all are. ownOnly is whether only the member's own repository check is missing or pending. An error is a +// read that could not be made: the member then counts as not mergeable. +func (h *Holder) notASuccess(m memberRead) (says string, ownOnly bool, err error) { + fc, err := h.Forge.Statuses(m.owner, m.repo, m.number, m.head, m.base) + if err != nil { + return "", false, err + } + if fc.Merge.Mergeable == nil { + return "", false, fmt.Errorf("the forge could not say whether %s may merge: %s", m.id, fc.Merge.Says) + } + if *fc.Merge.Mergeable { + return "", false, nil + } + var parts []string + ownOnly = len(fc.Merge.Blocking) > 0 + for _, b := range fc.Merge.Blocking { + parts = append(parts, fmt.Sprintf("%s (%s)", b.Context, b.State)) + if b.Context != ctxRepo || (b.State != "pending" && b.State != "missing") { + ownOnly = false + } + } + return fmt.Sprintf("%s requires %s; not a success: %s", fc.Merge.Branch, strings.Join(fc.Merge.Required, ", "), + strings.Join(parts, ", ")), ownOnly, nil +} + +// unmergedLive are a group's members not yet merged and not over, in the group's order. Under the lock. +func (h *Holder) unmergedLive(g *Group) []*Delivery { + var out []*Delivery + for _, d := range h.membersOf(g) { + if d.MergedAs == "" && !d.State.Final() && (d.Ready == nil || d.Ready.Merged == nil) { + out = append(out, d) + } + } + return out +} + +// groupPass is the one pass that merges a group's members, made by the first of them in the group's order that may be +// merged; nil when it is not d's turn, or a member is not approved at its head. Under the lock; the pass runs outside it. +func (h *Holder) groupPass(g *Group, d *Delivery, now time.Time) func() { + order := h.unmergedLive(g) + if len(order) == 0 || order[0] != d { + return nil + } + var pass []passMember + for _, m := range order { + c := h.covering(m) + if m.Ready == nil || m.Ready.Declined != nil || c == nil || c.Approved == nil || c.Heads[m.ID] != m.Commit || h.readyWaits(m) != "" { + return nil // the others are not approved at their heads yet: nothing is merged before they are + } + pass = append(pass, passMember{read: readOf(m, c.Heads[m.ID]), heads: c.Heads}) + } + for _, m := range order { + m.Ready.Tries++ + m.Ready.LastTry, m.Ready.merging = now, true + h.keep(m) + } + gid := g.ID + return func() { h.mergeGroupPass(gid, pass) } +} + +// passMember is one member of a pass as it was taken under the lock. +type passMember struct { + read memberRead + heads map[string]string +} + +// mergeGroupPass reads every member's required statuses, and only when all are a success merges the members in order, +// stopping at the first the merge tool refuses. +func (h *Holder) mergeGroupPass(gid string, pass []passMember) { + release := func(from int, retry bool) { + for _, p := range pass[from:] { + if d := h.deliveries[p.read.id]; d != nil && d.Ready != nil { + d.Ready.merging = false + if retry { + d.Ready.LastTry = time.Time{} + } + } + } + } + for _, p := range pass { + says, ownOnly, err := h.notASuccess(p.read) + if err == nil && says == "" { + continue + } + h.mu.Lock() + if err != nil { + h.Logf("[mesh-delivery] group %s: the required statuses of %s cannot be read, so nothing is merged yet: %v", gid, p.read.id, err) + release(0, true) + h.again("a group's merge whose required statuses could not be read") + h.mu.Unlock() + return + } + var own func() + if g := h.groups[gid]; g != nil { + if g.Hold == nil || g.Hold.Kind != "blocked" || g.Hold.Says != says || g.Hold.Member != p.read.id { + h.Logf("[mesh-delivery] group %s is approved and NOT merged: %s at %s: %s", gid, p.read.id, shortOf(p.read.head, 12), says) + } + since := time.Time{} + if g.Hold != nil && g.Hold.Kind == "blocked" { + since = g.Hold.Since + } + if since.IsZero() { + since = h.Now() + } + g.Hold = &GroupHold{Kind: "blocked", Member: p.read.id, Says: says, Since: since} + h.keepGroup(g) + } + if d := h.deliveries[p.read.id]; ownOnly && d != nil && d.Ready != nil { + own = h.askOwnAfterRefusal(d) // its own repository check is what is missing: asked, once for the head + } + release(0, true) + h.again("a group's merge held for a required status") + h.mu.Unlock() + if own != nil { + own() + } + return + } + h.mu.Lock() + if g := h.groups[gid]; g != nil && g.Hold != nil { + g.Hold = nil + h.keepGroup(g) + } + h.mu.Unlock() + for i, p := range pass { + if h.mergeOne(p.read.id, p.heads) { + continue + } + h.mu.Lock() + release(i+1, false) + if g := h.groups[gid]; g != nil { + h.Logf("[mesh-delivery] group %s: the merge of %s was refused, so none of the members after it is merged", gid, p.read.id) + h.keepGroup(g) + } + h.mu.Unlock() + return + } +} + +// groupLines are the groups that are held or half merged, as urgent stalled lines the controller raises as conditions: +// one for each, on its first member not merged, until every member is merged or a person decides (issue 1275). A member +// the merge tool refused is said by readyLines, with its group; this adds a group held for a required status, and a +// group half merged with no refusal. Under the lock. +func (h *Holder) groupLines(now time.Time) []StalledLine { + var out []StalledLine + for _, g := range h.groups { + live := h.unmergedLive(g) + if len(live) == 0 { + continue + } + first, refused, running, merged := live[0], false, false, 0 + for _, d := range live { + if d.Ready != nil && (d.Ready.RefusedSince != nil || d.Ready.Failed != "" || d.Ready.Refused != "") { + refused = true + } + if d.Ready != nil && d.Ready.merging { + running = true + } + } + for _, d := range h.membersOf(g) { + if d.MergedAs != "" || (d.Ready != nil && d.Ready.Merged != nil) { + merged++ + } + } + heldOn := h.deliveries[holdMember(g)] + switch { + // A held member the merge tool refused is said by readyLines already: one line for an id, not two. + case g.Hold != nil && g.Hold.Kind == "blocked" && !running && !(heldOn != nil && heldOn.Ready != nil && heldOn.Ready.RefusedSince != nil): + out = append(out, StalledLine{ID: g.Hold.Member, Number: numberOf(h.deliveries[g.Hold.Member]), State: first.State, + For: now.Sub(g.Hold.Since).Round(time.Second).String(), Bound: "0s", Severity: "urgent", + H2: "none: the required status turns success, which asks the merge again — or the operator's decision", + Says: fmt.Sprintf("group %s is approved and nothing is merged (merged %d of %d): %s waits on a required status "+ + "at its head: %s. Needed: that status turns success (its check asked again), or the operator decides.", + g.ID, merged, len(h.membersOf(g)), g.Hold.Member, g.Hold.Says)}) + case merged > 0 && !refused && !running: + out = append(out, StalledLine{ID: first.ID, Number: first.Number, State: first.State, + For: now.Sub(g.Updated).Round(time.Second).String(), Bound: "0s", Severity: "urgent", + H2: "none: every member merged, or the operator decides", + Says: fmt.Sprintf("group %s is half merged (%d of %d members merged): %s is not merged. Needed: it merges "+ + "(its Approve and checks hold at its head), or the operator decides.", g.ID, merged, len(h.membersOf(g)), first.ID)}) + } + } + return out +} + +func numberOf(d *Delivery) int { + if d == nil { + return 0 + } + return d.Number +} + +func holdMember(g *Group) string { + if g.Hold == nil { + return "" + } + return g.Hold.Member +} diff --git a/modules/mesh-delivery/cmd/mesh-delivery/group_test.go b/modules/mesh-delivery/cmd/mesh-delivery/group_test.go new file mode 100644 index 00000000..952c40f9 --- /dev/null +++ b/modules/mesh-delivery/cmd/mesh-delivery/group_test.go @@ -0,0 +1,443 @@ +package main + +import ( + "encoding/json" + "errors" + "sort" + "strings" + "testing" + "time" +) + +// **A group is merged in one pass on its one Approve, and delivered by one plan** (novox/hq ADR 0418, ADR 0419, ADR +// 0421, issues 978 and 1047): the members are not ordered, none waits for another's merge or delivery, and a trunk +// move does not ask the operator a second time. + +const ( + gCtl, gCat = "novox/mesh-controller", "novox/mesh-catalog" + gHc, gHk = "c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1", "d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2" + gTrunk0, gT1 = "a2c2c35000000000000000000000000000000000", "d28ca95000000000000000000000000000000000" +) + +// approvedGroup is a group of two pull requests on one branch, checks green, composed green and the operator's Approve +// asked and not yet answered (w.op.auto is off). +func approvedGroup(t *testing.T) *world { + w := newWorld(t) + w.op.auto = false + w.h.PullUpdated(pr(gCtl, 1, gHc, "feat/g", "internal/a.go")) + w.h.PullUpdated(pr(gCat, 2, gHk, "feat/g", "modules/app/module.json")) + w.h.Checked(judged(gCtl, 1, gHc, "pass", gTrunk0)) + w.h.Checked(judged(gCat, 2, gHk, "pass", gTrunk0)) + w.settleAll() + g := w.h.groups["feat/g"] + if g == nil || g.Check == nil { + t.Fatalf("no composed check was asked: %+v", g) + } + w.h.Checked(CheckedEvent{Group: "feat/g", ID: g.Check.Asked, Verdict: "pass", Summary: "composed", + Members: []struct { + Owner string `json:"owner"` + Repo string `json:"repo"` + Commit string `json:"commit"` + Trunk string `json:"trunk,omitempty"` + }{{Owner: "novox", Repo: "mesh-controller", Commit: gHc, Trunk: gTrunk0[:12]}, + {Owner: "novox", Repo: "mesh-catalog", Commit: gHk, Trunk: gTrunk0[:12]}}}) + w.settleAll() + if w.op.count() != 1 || len(w.op.open()) != 1 { + t.Fatalf("the green group is asked once: %d asks, open %v", w.op.count(), w.op.open()) + } + return w +} + +// Issue 1047 (the rule the lead settled, ADR 0421 §3): an approved group whose trunk moved through a merge that is not +// its member's has each out-of-date member's check asked at once, then the composed check, and merges on the same +// Approve — with no second one asked. +func TestAnApprovedGroupWhoseTrunkMovedIsCheckedAgainAndMergedOnTheSameApprove(t *testing.T) { + w := approvedGroup(t) + asked := len(w.ctl.checks) + w.h.PullMerged(merged(gCtl, 5, "5555555555555555", gT1)) + w.settleAll() + // The member whose trunk moved has its own check asked, in the same look; the other member's trunk did not move. + if got := w.ctl.checks[asked:]; len(got) != 1 || got[0] != ":"+IDOf(gCtl, gHc) { + t.Fatalf("the trunk's move asked %v; wanted the moved member's own check once", got) + } + if w.ctl.baseSHAs[len(w.ctl.baseSHAs)-1] != gT1[:12] && w.ctl.baseSHAs[len(w.ctl.baseSHAs)-1] != gT1 { + t.Fatalf("its check was not asked against the trunk's head: %v", w.ctl.baseSHAs) + } + // The Approve is given while the checks run: nothing merges on the verdict out of date. + w.op.answer(w.op.open()[0], OptionApprove) + w.settleAll() + if len(w.forge.merges) != 0 { + t.Fatalf("merged while the group's checks were out of date: %v", w.forge.merges) + } + // The member is judged again: the composed check is asked on the new verdicts. + w.later(time.Minute) + w.h.Checked(judged(gCtl, 1, gHc, "pass", gT1)) + w.settleAll() + g := w.h.groups["feat/g"] + last := w.ctl.checks[len(w.ctl.checks)-1] + if last != "feat/g:"+IDOf(gCat, gHk)+","+IDOf(gCtl, gHc) { + t.Fatalf("the composed check was not asked again: %v", w.ctl.checks) + } + w.h.Checked(CheckedEvent{Group: "feat/g", ID: g.Check.Asked, Verdict: "pass", Summary: "composed again"}) + w.settleAll() + sort.Strings(w.forge.merges) + if len(w.forge.merges) != 2 || w.forge.merges[0] != "novox/mesh-catalog#2@"+gHk || w.forge.merges[1] != "novox/mesh-controller#1@"+gHc { + t.Fatalf("both members were not merged on the new composed verdict: %v", w.forge.merges) + } + if w.op.count() != 1 { + t.Fatalf("a second Approve was asked: %d", w.op.count()) + } +} + +// ADR 0418 §4, issue 1275/AC3: a member the forge refuses stops the pass (the members after it are not asked), and is +// merged again on its fresh check, the others after it in the same pass. +func TestAMemberTheForgeRefusesStopsThePassAndIsMergedAgainOnItsFreshCheck(t *testing.T) { + w := approvedGroup(t) + w.forge.refuse = errAskedAgain + w.op.answer(w.op.open()[0], OptionApprove) + w.settleAll() + if len(w.forge.merges) != 1 { + t.Fatalf("the pass stops at the first refusal: %v", w.forge.merges) + } + w.forge.refuse = nil + w.later(2 * time.Minute) + w.settleAll() + if len(w.forge.merges) != 1 { + t.Fatalf("a refused member is asked again with no event, on a timer or at each look: %v", w.forge.merges) + } + w.h.Checked(CheckedEvent{Group: "feat/g", ID: w.h.groups["feat/g"].Check.Asked, Verdict: "pass", Summary: "composed again"}) + w.settleAll() + if len(w.forge.merges) != 3 { + t.Fatalf("a refused member is asked again on the next composed verdict, then the next: %v", w.forge.merges) + } + if d := w.delivery(IDOf(gCat, gHk)); d.Ready.Merged == nil { + t.Fatalf("not merged: %+v", d.Ready) + } +} + +// Issue 978 (the pin): a feature's single pull request in one repository is a lone delivery, asked alone. +func TestAFeaturesSinglePullRequestIsALoneDeliveryAskedAlone(t *testing.T) { + w := newWorld(t) + w.op.auto = false + w.h.PullUpdated(pr("novox/app", 7, readyHead, "feat/x", "src/a.go")) + w.h.Checked(judgedAt("novox/app", 7, readyHead, "pass", "t1t1t1t1t1t1")) + w.settleAll() + d := w.delivery(IDOf("novox/app", readyHead)) + if d.Group != "" || len(w.h.groups) != 0 { + t.Fatalf("a single pull request made a group: %q %v", d.Group, w.h.groups) + } + if w.op.count() != 1 { + t.Fatalf("a lone delivery is asked once, alone: %d", w.op.count()) + } +} + +// A group answer says its members' gate is the composed check's (agreed with the controller, issue 977). +func TestTheGroupsAnswerSaysItsGateIsTheComposedChecks(t *testing.T) { + w := approvedGroup(t) + lines := w.h.Groups(false) + if len(lines) != 1 || !lines[0].ComposedGate { + t.Fatalf("groups: %+v", lines) + } + raw, _ := json.Marshal(lines[0]) + var got map[string]any + if err := json.Unmarshal(raw, &got); err != nil || got["composed_gate"] != true { + t.Fatalf("the group line says %s", raw) + } + if _, ordered := got["order"]; ordered { + t.Fatalf("a group line still says an order: %s", raw) + } +} + +// ADR 0403 §1: a member whose own gate the group's composed check judges instead (the controller says its gate layer +// "composed") counts as passed here once its repository's check did not fail, so the group's composed check is asked. +func TestAMemberWhoseGateIsTheComposedChecksCountsAsPassed(t *testing.T) { + w := newWorld(t) + w.op.auto = false + w.h.PullUpdated(pr(gCtl, 1, gHc, "feat/g", "internal/a.go")) + w.h.PullUpdated(pr(gCat, 2, gHk, "feat/g", "modules/app/module.json")) + for _, c := range []struct { + repo string + n int + head string + }{{gCtl, 1, gHc}, {gCat, 2, gHk}} { + v := judged(c.repo, c.n, c.head, "pass", gTrunk0) + v.Gate = &struct { + Verdict string `json:"verdict"` + Summary string `json:"summary"` + Modules []string `json:"modules,omitempty"` + Dependents []string `json:"dependents,omitempty"` + }{Verdict: GateComposed, Summary: "judged by its group's composed check"} + v.RepoCheck = &struct { + Verdict string `json:"verdict"` + Summary string `json:"summary"` + }{Verdict: "pass", Summary: "the repository's own check passed"} + w.h.Checked(v) + } + w.settleAll() + if len(w.ctl.checks) != 1 || w.ctl.checks[0] != "feat/g:"+IDOf(gCat, gHk)+","+IDOf(gCtl, gHc) { + t.Fatalf("the composed check was not asked for members judged by it: %v", w.ctl.checks) + } + // A member whose repository check failed is not passed, whatever the composed check will say. + w2 := newWorld(t) + w2.h.PullUpdated(pr(gCtl, 1, gHc, "feat/g", "internal/a.go")) + v := judged(gCtl, 1, gHc, "pass", gTrunk0) + v.Gate = &struct { + Verdict string `json:"verdict"` + Summary string `json:"summary"` + Modules []string `json:"modules,omitempty"` + Dependents []string `json:"dependents,omitempty"` + }{Verdict: GateComposed} + v.RepoCheck = &struct { + Verdict string `json:"verdict"` + Summary string `json:"summary"` + }{Verdict: "fail", Summary: "a test failed"} + w2.h.Checked(v) + if w2.state(IDOf(gCtl, gHc)) == Ready { + t.Fatalf("a member whose repository check failed is ready") + } +} + +// Issue 1049 (survey question 1, confirmed here): a composed check asked after one member has already merged is asked +// with ALL the members, the merged one at the head it merged, never with the unmerged one alone. The controller needs two +// or more members to compose (mesh-controller delivery.go askGroupCheck), and a merge of a contained head is a no-op +// there. +func TestAComposedCheckAskedAfterOneMemberMergedIsAskedWithAllMembers(t *testing.T) { + w := approvedGroup(t) + w.forge.refuseRepo = map[string]error{gCtl: errAskedAgain} + w.op.answer(w.op.open()[0], OptionApprove) + w.settleAll() + w.h.PullMerged(merged(gCat, 2, gHk, gT1)) + w.settleAll() + if g := w.h.groups["feat/g"]; g == nil || !g.Closed { + t.Fatalf("the group did not close when a member merged: %+v", g) + } + // The unmerged member gets a new head and its own verdict: the group's composed check is due again. + const gHc2 = "e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3" + asked := len(w.ctl.checks) + w.h.PullUpdated(pr(gCtl, 1, gHc2, "feat/g", "internal/a.go")) + w.h.Checked(judged(gCtl, 1, gHc2, "pass", gT1)) + w.settleAll() + var composed []string + for _, c := range w.ctl.checks[asked:] { + if len(c) > 7 && c[:7] == "feat/g:" { + composed = append(composed, c) + } + } + if len(composed) != 1 || composed[0] != "feat/g:"+IDOf(gCat, gHk)+","+IDOf(gCtl, gHc2) { + t.Fatalf("the composed check was asked with %v; wanted once, with both members (the merged one at its head)", composed) + } +} + +// Issue 1082 (the 2026-10-17 23:48 group): a closed group's member whose merge the tool refused after its sibling merged +// asks the group's composed check again at once, and merges on its pass with the one Approve (ADR 0418 §4, ADR 0421). +func TestAClosedGroupsMemberWhoseMergeWasRefusedAsksTheComposedCheckAgainAtOnce(t *testing.T) { + w := approvedGroup(t) + w.forge.refuseRepo = map[string]error{gCtl: errAskedAgain} + w.op.answer(w.op.open()[0], OptionApprove) + w.settleAll() + // The catalogue member merged, the controller's was refused. + w.h.PullMerged(merged(gCat, 2, gHk, gT1)) + asked := len(w.ctl.checks) + w.settleAll() + var composed []string + for _, c := range w.ctl.checks[asked:] { + if strings.HasPrefix(c, "feat/g:") { + composed = append(composed, c) + } + } + if len(composed) != 1 || composed[0] != "feat/g:"+IDOf(gCat, gHk)+","+IDOf(gCtl, gHc) { + t.Fatalf("the refusal asked the composed check %v; wanted once, at once, with both members", composed) + } + // Not again while that check is on its way. + w.later(2 * time.Minute) + w.settleAll() + if got := w.ctl.checks[asked:]; len(got) != 1 { + t.Fatalf("asked again while the composed check was on its way: %v", got) + } + // It passes; the controller's member merges on the same Approve. + w.forge.refuseRepo = nil + g := w.h.groups["feat/g"] + w.h.Checked(CheckedEvent{Group: "feat/g", ID: g.Check.Asked, Verdict: "pass", Summary: "composed again"}) + w.settleAll() + if d := w.delivery(IDOf(gCtl, gHc)); d.Ready.Merged == nil { + t.Fatalf("the refused member was not merged on the new composed verdict: %+v", d.Ready) + } + if w.op.count() != 1 { + t.Fatalf("a second Approve was asked: %d", w.op.count()) + } +} + +// Issue 1082: a refusal loop is not silent. A merge refused past ten minutes in a closed group (ADR 0418 §3) is a stalled +// line naming the member and the merge tool's last answer, which the controller raises as a condition. +func TestARefusalLoopInAClosedGroupIsSaidAtOnce(t *testing.T) { + w := approvedGroup(t) + w.forge.refuseRepo = map[string]error{gCtl: errAskedAgain} + w.op.answer(w.op.open()[0], OptionApprove) + w.settleAll() + w.h.PullMerged(merged(gCat, 2, gHk, gT1)) + w.settleAll() + // Each composed pass is followed by a refusal again: the loop. + for range 12 { + w.later(time.Minute) + g := w.h.groups["feat/g"] + w.h.Checked(CheckedEvent{Group: "feat/g", ID: g.Check.Asked, Verdict: "pass", Summary: "composed again"}) + w.settleAll() + } + lines := w.h.Stalled() + if len(lines) != 1 || lines[0].ID != IDOf(gCtl, gHc) || lines[0].Bound != "0s" || + !strings.Contains(lines[0].Says, errAskedAgain.Error()) || !strings.Contains(lines[0].Says, "group feat/g") { + t.Fatalf("the loop was not said, naming the member, its group and the tool's answer: %+v", lines) + } +} + +// Issue 1047 (the live case of 2026-10-17 23:34, ADR 0395 §1, design 47): a group member that is ready when a third party's +// merge moves its trunk, with no Approve asked yet (the composed check has not passed), has its own check asked at once, +// and the Approve is asked once the group's composed check has passed on it. +func TestATrunkMoveAsksTheCheckOfAGroupMemberWhoseApproveIsNotAskedYet(t *testing.T) { + w := newWorld(t) + w.op.auto = false + w.h.PullUpdated(pr(gCtl, 1, gHc, "feat/g", "internal/a.go")) + w.h.PullUpdated(pr(gCat, 2, gHk, "feat/g", "modules/app/module.json")) + w.h.Checked(judged(gCtl, 1, gHc, "pass", gTrunk0)) + w.h.Checked(judged(gCat, 2, gHk, "pass", gTrunk0)) + w.settleAll() + if w.op.count() != 0 { + t.Fatalf("the composed check has not passed, yet the operator was asked: %d", w.op.count()) + } + asked := len(w.ctl.checks) + w.h.PullMerged(merged(gCat, 5, "5555555555555555", gT1)) + w.settleAll() + got := w.ctl.checks[asked:] + if len(got) != 1 || got[0] != ":"+IDOf(gCat, gHk) { + t.Fatalf("the trunk's move asked %v; wanted the moved member's own check once", got) + } + if last := w.ctl.baseSHAs[len(w.ctl.baseSHAs)-1]; last != gT1[:12] && last != gT1 { + t.Fatalf("its check was not asked against the trunk's head: %v", w.ctl.baseSHAs) + } + // Not asked again at the next look while it is on its way. + w.later(time.Second) + w.settleAll() + if len(w.ctl.checks[asked:]) != 1 { + t.Fatalf("asked twice: %v", w.ctl.checks[asked:]) + } + // The composed check that was on its way when the trunk moved answers, judged against the old trunk (ADR 0403 §6): it is + // out of date as it lands, so the group does not become ready on it and the operator is not asked. + g := w.h.groups["feat/g"] + w.later(time.Minute) + w.h.Checked(CheckedEvent{Group: "feat/g", ID: g.Check.Asked, Verdict: "pass", Summary: "composed on the old trunk", + Members: []struct { + Owner string `json:"owner"` + Repo string `json:"repo"` + Commit string `json:"commit"` + Trunk string `json:"trunk,omitempty"` + }{{Owner: "novox", Repo: "mesh-controller", Commit: gHc, Trunk: gTrunk0[:12]}, + {Owner: "novox", Repo: "mesh-catalog", Commit: gHk, Trunk: gTrunk0[:12]}}}) + w.settleAll() + if w.op.count() != 0 { + t.Fatalf("the operator was asked on a composed pass judged against the old trunk: %d", w.op.count()) + } + // The moved member's fresh verdict arrives: the composed check is asked again, and only its pass asks the operator. + w.later(time.Minute) + composedBefore := len(w.ctl.checks) + w.h.Checked(judged(gCat, 2, gHk, "pass", gT1)) + w.settleAll() + last := w.ctl.checks[len(w.ctl.checks)-1] + if len(w.ctl.checks) == composedBefore || !strings.HasPrefix(last, "feat/g:") { + t.Fatalf("the composed check was not asked again on the fresh verdict: %v", w.ctl.checks[composedBefore:]) + } + if w.op.count() != 0 { + t.Fatalf("the operator was asked before the new composed check passed: %d", w.op.count()) + } + g = w.h.groups["feat/g"] + w.h.Checked(CheckedEvent{Group: "feat/g", ID: g.Check.Asked, Verdict: "pass", Summary: "composed on the new trunk", + Members: []struct { + Owner string `json:"owner"` + Repo string `json:"repo"` + Commit string `json:"commit"` + Trunk string `json:"trunk,omitempty"` + }{{Owner: "novox", Repo: "mesh-controller", Commit: gHc, Trunk: gTrunk0[:12]}, + {Owner: "novox", Repo: "mesh-catalog", Commit: gHk, Trunk: gT1[:12]}}}) + w.settleAll() + if w.op.count() != 1 { + t.Fatalf("the operator is asked once, on the new pass: %d", w.op.count()) + } +} + +// Issue 1047 (review round 2): the other order. The moved member's fresh verdict lands before the stale composed pass (one +// build against two): the pass is out of date as it lands, the composed check is asked again, and the operator is asked +// only after that check passes. +func TestAStaleComposedPassLandingAfterTheMovedMembersFreshVerdictIsAskedAgain(t *testing.T) { + w := newWorld(t) + w.op.auto = false + w.h.PullUpdated(pr(gCtl, 1, gHc, "feat/g", "internal/a.go")) + w.h.PullUpdated(pr(gCat, 2, gHk, "feat/g", "modules/app/module.json")) + w.h.Checked(judged(gCtl, 1, gHc, "pass", gTrunk0)) + w.h.Checked(judged(gCat, 2, gHk, "pass", gTrunk0)) + w.settleAll() + g := w.h.groups["feat/g"] + w.later(time.Minute) + w.h.PullMerged(merged(gCat, 5, "5555555555555555", gT1)) + w.settleAll() + w.later(time.Minute) + w.h.Checked(judged(gCat, 2, gHk, "pass", gT1)) // the fresh verdict first + w.settleAll() + w.later(time.Minute) + before := len(w.ctl.checks) + w.h.Checked(CheckedEvent{Group: "feat/g", ID: g.Check.Asked, Verdict: "pass", Summary: "composed on the old trunk", + Members: []struct { + Owner string `json:"owner"` + Repo string `json:"repo"` + Commit string `json:"commit"` + Trunk string `json:"trunk,omitempty"` + }{{Owner: "novox", Repo: "mesh-controller", Commit: gHc, Trunk: gTrunk0[:12]}, + {Owner: "novox", Repo: "mesh-catalog", Commit: gHk, Trunk: gTrunk0[:12]}}}) + w.settleAll() + if len(w.ctl.checks) == before || !strings.HasPrefix(w.ctl.checks[len(w.ctl.checks)-1], "feat/g:") { + t.Fatalf("the composed check was not asked again after the stale pass: %v", w.ctl.checks[before:]) + } + if w.op.count() != 0 { + t.Fatalf("the operator was asked before the new composed check passed: %d", w.op.count()) + } +} + +// ownPending is the merge tool's refusal of a member whose own mesh/repo-check stands pending after the group's composed +// check set its gate (issue 1049: the forge's "not a success: mesh/repo-check (pending)"). +var ownPending = errors.New("the forge refused gitea_merge_pull_request: novox/mesh-controller#1 is not merged: main requires " + + "mesh/merge-gate, mesh/repo-check; not a success: mesh/repo-check (pending). A pull request merges once its checks succeed") + +// ownChecks are the checks asked from index on that were a head's own (no group), not a group's composed check. +func ownChecks(w *world, from int) (own, composed []string) { + for _, c := range w.ctl.checks[from:] { + if strings.HasPrefix(c, ":") { + own = append(own, c) + } else { + composed = append(composed, c) + } + } + return +} + +// Issue 1049 (the 2026-10-18 02:04 group): a closed group's member refused for a pending status of its OWN (the repository +// check a sibling's new head put back) asks its own check, once for the head, and not the composed check, which sets no +// repository check; a second pass at the same head asks nothing more. +func TestAClosedGroupsMemberRefusedForItsOwnPendingStatusAsksItsOwnCheckOncePerHead(t *testing.T) { + w := approvedGroup(t) + w.forge.refuseRepo = map[string]error{gCtl: ownPending} + w.op.answer(w.op.open()[0], OptionApprove) + w.settleAll() + w.h.PullMerged(merged(gCat, 2, gHk, gT1)) + asked := len(w.ctl.checks) + w.settleAll() + own, composed := ownChecks(w, asked) + if len(composed) != 0 || len(own) != 1 || own[0] != ":"+IDOf(gCtl, gHc) { + t.Fatalf("own checks %v, composed checks %v; wanted the member's own check once and no composed check", own, composed) + } + // Later passes, the same head, the same refusal: nothing more is asked. + w.later(2 * time.Minute) + w.settleAll() + w.later(2 * time.Minute) + w.settleAll() + if got := len(w.ctl.checks) - asked; got != 1 { + t.Fatalf("asked %d checks at the same head, wanted the one: %v", got, w.ctl.checks[asked:]) + } +} diff --git a/modules/mesh-delivery/cmd/mesh-delivery/history_test.go b/modules/mesh-delivery/cmd/mesh-delivery/history_test.go new file mode 100644 index 00000000..382964e4 --- /dev/null +++ b/modules/mesh-delivery/cmd/mesh-delivery/history_test.go @@ -0,0 +1,233 @@ +package main + +import ( + "errors" + "strings" + "testing" + "time" +) + +// History (novox/hq issue 313): merges long past, heard only from the forge's word when this owner first held +// the seat, reached the machines before it existed. They are retired — never held for a person, released or +// delivered — and what is owed for a pull request the forge no longer holds is given up, not tried for ever. + +// aPastMerge is the forge's `pull.merged` replayed from the bus's history: merged days before it is heard. +func aPastMerge(repo string, number int, head, merge string, at time.Time) PullEvent { + p := merged(repo, number, head, merge) + p.MergedAt = at.UTC().Format(time.RFC3339) + return p +} + +func TestAMergeLongPastHeardFromTheForgesWordIsRetiredAtAdoption(t *testing.T) { + w := newWorld(t) + id := IDOf("novox/app", head) + w.h.PullMerged(aPastMerge("novox/app", 274, head, merge, w.now.Add(-4*24*time.Hour))) + w.settleAll() + d := w.delivery(id) + if d.State != Retired || !strings.Contains(d.History, "history") { + t.Fatalf("a merge four days past heard from the forge's word is %s (%q)", d.State, d.History) + } + if got := statesOf([]State{d.Transitions[0].To, d.Transitions[1].To}); strings.Join(got, ",") != "proposed,retired" || + d.Transitions[1].Event != EvHistory { + t.Fatalf("transitions %+v", d.Transitions) + } + // Nothing is said on its pull request — the forge may hold another under its number — and nothing stalls. + if w.forge.viewCalls != 0 || len(w.forge.statuses) != 0 { + t.Fatalf("history was shown on a pull request: %d view(s), statuses %v", w.forge.viewCalls, w.forge.statuses) + } + if len(d.Owed) != 0 { + t.Fatalf("still owed: %+v", d.Owed) + } + if s := w.h.Stalled(); len(s) != 0 { + t.Fatalf("history stalls: %+v", s) + } + // Its record is still kept, on the commit's note. + if len(w.forge.notesOn(head)) == 0 { + t.Fatal("no note of it on its commit") + } + + // A merge heard from the forge's word within moments of happening is a live one: held for a person. + live := IDOf("novox/app", "bbbbbbbbbbbb2222") + w.h.PullMerged(aPastMerge("novox/app", 9, "bbbbbbbbbbbb2222", "cccccccccccc3333", w.now.Add(-time.Minute))) + if s := w.state(live); s != Held { + t.Fatalf("a live merge from the forge's word is %s", s) + } + // A merge long past whose walk the controller still holds open is delivered by it, never history. + walked := IDOf("novox/app", "dddddddddddd4444") + w.h.WalkMoved(aWalk("plan-9", "novox/app", "eeeeeeeeeeee5555", true)) + w.h.PullMerged(aPastMerge("novox/app", 10, "dddddddddddd4444", "eeeeeeeeeeee5555", w.now.Add(-48*time.Hour))) + if s := w.state(walked); s == Retired { + t.Fatal("a merge with an open walk was retired as history") + } + // A merge the forge announces under a number an older delivery already merged with is not that delivery. + w.h.PullMerged(aPastMerge("novox/app", 274, "", "ffffffffffff6666", w.now.Add(-time.Minute))) + if d := w.delivery(id); d.MergedAs != merge { + t.Fatalf("a reused number merged the old delivery again as %s", d.MergedAs) + } +} + +// oldHistory is a held delivery as the first adoption left it: made from the forge's word, held, owing a view +// on a pull request the forge no longer holds. +func oldHistory(id, repo string, number int, created, mergedAt time.Time) *Delivery { + commit := strings.SplitN(id, "@", 2)[1] + "0000" + return &Delivery{ID: id, Repository: repo, Commit: commit, Number: number, Base: "main", MergedAs: commit, + MergedAt: mergedAt, Created: created, Since: created, State: Held, + HeldWhy: "merged without a passing check: only a person decides that it goes on", + Transitions: []Transition{ + {At: created, To: Proposed, Event: EvAnnounced, Why: repo + "#274 merged; its head was not heard by this " + + "owner, so it is made from the forge's word"}, + {At: created, From: Proposed, To: Held, Event: EvMergedUnchecked, Why: "merged without a passing check"}}, + Owed: []Effect{{Kind: EffectView, Since: created, Tries: 11051, + Last: "Gitea API /repos/" + repo + "/issues/274/comments: 500 issue does not exist"}}} +} + +func TestRetireHistoryRetiresTheAdoptedHistoryInBulkAndLeavesEveryOtherHeldOne(t *testing.T) { + store := newMemStore() + start := time.Date(2026, 10, 6, 23, 5, 27, 0, time.UTC) + for _, d := range []*Delivery{ + oldHistory("novox/hq@018ee359ae52", "novox/hq", 274, start, start.Add(-4*24*time.Hour)), + oldHistory("novox/lab@a4f6bb8ce054", "novox/lab", 53, start, start.Add(-2*time.Hour)), + // Made from the forge's word, but heard a minute after its merge: a live merge, a person's. + oldHistory("novox/hq@7777777777aa", "novox/hq", 3, start, start.Add(-time.Minute)), + } { + if err := store.PutDelivery(d); err != nil { + t.Fatal(err) + } + } + w := newWorldOver(t, store, newFakeController(), newFakeForge(), start.Add(36*time.Hour)) + // And one heard live: rejected, then merged — a person's to decide. + w.h.PullUpdated(pr("novox/app", 7, head, "feat/x", "src/a.go")) + w.h.Checked(verdict("novox/app", 7, head, "fail")) + w.h.PullMerged(merged("novox/app", 7, head, merge)) + if len(w.h.Stalled()) != 3 { // the three adopted at the switch; the one held just now is within its bound + t.Fatalf("stalled before: %+v", w.h.Stalled()) + } + + dry, err := w.h.RetireHistory("", "", "history adopted at the switch", aPerson, true) + if err != nil || dry.Retired != 2 || dry.LeftHeld != 2 || len(dry.Left) != 2 { + t.Fatalf("a dry retire: %+v %v", dry, err) + } + if s := w.state("novox/hq@018ee359ae52"); s != Held { + t.Fatalf("a dry retire moved a delivery to %s", s) + } + if _, err := w.h.RetireHistory("", "", "", aPerson, false); err == nil { + t.Fatal("a retire with no why was taken") + } + + got, err := w.h.RetireHistory("", "", "history adopted at the switch", aPerson, false) + if err != nil || got.Retired != 2 || got.ByRepository["novox/hq"] != 1 || got.ByRepository["novox/lab"] != 1 { + t.Fatalf("retire: %+v %v", got, err) + } + for _, id := range []string{"novox/hq@018ee359ae52", "novox/lab@a4f6bb8ce054"} { + d := w.delivery(id) + if d.State != Retired || d.Transitions[len(d.Transitions)-1].By != aPerson { + t.Fatalf("%s is %s", id, d.State) + } + for _, e := range d.Owed { + if e.Kind == EffectView || e.Kind == EffectStatus { + t.Fatalf("%s still owes its pull request: %+v", id, e) + } + } + } + for _, id := range []string{"novox/hq@7777777777aa", IDOf("novox/app", head)} { + if s := w.state(id); s != Held { + t.Fatalf("%s, not history, is %s", id, s) + } + } + // Nothing was released, delivered or let go. + if len(w.ctl.delivers) != 0 || len(w.ctl.stops) != 0 { + t.Fatalf("a retire asked the controller: deliver %v stop %v", w.ctl.delivers, w.ctl.stops) + } + if s := w.h.Stalled(); len(s) != 1 || s[0].ID != "novox/hq@7777777777aa" { + t.Fatalf("stalled after: %+v", s) + } + // Retired is final: a second retire takes nothing, and nothing leaves it. + again, _ := w.h.RetireHistory("", "", "again", aPerson, false) + if again.Retired != 0 { + t.Fatalf("retired twice: %+v", again) + } + if _, err := w.h.Release("novox/hq@018ee359ae52", "go on", aPerson); err == nil { + t.Fatal("a retired delivery was released") + } +} + +func TestRetireHistoryIsRefusedForARealHeldDelivery(t *testing.T) { + w := newWorld(t) + id := IDOf("novox/app", head) + w.h.PullUpdated(pr("novox/app", 7, head, "feat/x", "src/a.go")) + w.h.Checked(verdict("novox/app", 7, head, "fail")) + w.h.PullMerged(merged("novox/app", 7, head, merge)) + w.later(48 * time.Hour) + _, err := w.h.RetireHistory(id, "", "looks old", aPerson, false) + var refused ErrRefused + if !errors.As(err, &refused) || !strings.Contains(err.Error(), "heard its head") { + t.Fatalf("a held delivery whose head was heard was retired: %v", err) + } + if s := w.state(id); s != Held { + t.Fatalf("a refused retire left it %s", s) + } + // A delivery that is not held is refused too, by the table. + other := IDOf("novox/app", "bbbbbbbbbbbb2222") + w.h.PullUpdated(pr("novox/app", 8, "bbbbbbbbbbbb2222", "feat/y", "src/b.go")) + if _, err := w.h.RetireHistory(other, "", "old", aPerson, false); err == nil || w.state(other) != Proposed { + t.Fatalf("a proposed delivery was retired: %v", err) + } + if _, err := w.h.RetireHistory("novox/app@nothing", "", "old", aPerson, false); err == nil { + t.Fatal("retired a delivery that does not exist") + } +} + +func TestOwedWorkTheForgeSaysCanNeverBeDoneIsDropped(t *testing.T) { + w := newWorld(t) + id := IDOf("novox/app", head) + w.forge.viewErr = errors.New(`Gitea API /repos/novox/app/issues/7/comments: 500 {"message":"issue does not ` + + `exist [id: 0, repo_id: 53, index: 7]"}`) + w.h.PullUpdated(pr("novox/app", 7, head, "feat/x", "src/a.go")) + w.h.Flush() + if w.forge.viewCalls != 1 { + t.Fatalf("the view was asked %d time(s)", w.forge.viewCalls) + } + if d := w.delivery(id); len(d.Owed) != 0 { + t.Fatalf("an answer that its pull request does not exist left owed %+v", d.Owed) + } + w.settleAll() + if w.forge.viewCalls != 1 { + t.Fatalf("a dropped view was asked again: %d", w.forge.viewCalls) + } + if st := w.h.Status("listening"); !strings.Contains(strings.Join(st["refused"].([]string), "\n"), "given up") { + t.Fatalf("its drop was not said: %v", st["refused"]) + } + + // Any other failure is tried again later each time, and given up after retryLimit tries. + w.forge.viewErr = errors.New("the forge is away") + w.h.Checked(verdict("novox/app", 7, head, "pass")) + before := w.forge.viewCalls + w.h.Flush() + w.h.Flush() + w.h.Flush() + if w.forge.viewCalls != before+1 { + t.Fatalf("a failed view was tried %d time(s) at once", w.forge.viewCalls-before) + } + w.later(retryFirst + time.Second) + w.h.Flush() + if w.forge.viewCalls != before+2 { + t.Fatalf("a failed view was not tried again after its wait: %d", w.forge.viewCalls-before) + } + if retryAfter(1) != retryFirst || retryAfter(2) != 2*retryFirst || retryAfter(100) != retryCap { + t.Fatalf("the waits: %s %s %s", retryAfter(1), retryAfter(2), retryAfter(100)) + } + for range retryLimit { + w.later(retryCap) + w.h.Flush() + } + for _, e := range w.delivery(id).Owed { + if e.Kind == EffectView { + t.Fatalf("a view failing %d times is still owed: %+v", e.Tries, e) + } + } + if !doesNotExist(errors.New("Gitea API /repos/a/b/issues/1/comments: 404 Not Found")) || + doesNotExist(errors.New("tool gitea.gitea_delivery_view does not exist on this machine")) || + doesNotExist(errors.New("the forge is away")) { + t.Fatal("doesNotExist reads the forge's answer wrong") + } +} diff --git a/modules/mesh-delivery/cmd/mesh-delivery/holder.go b/modules/mesh-delivery/cmd/mesh-delivery/holder.go new file mode 100644 index 00000000..362436c6 --- /dev/null +++ b/modules/mesh-delivery/cmd/mesh-delivery/holder.go @@ -0,0 +1,2717 @@ +package main + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "slices" + "sort" + "strings" + "sync" + "time" +) + +// The holder of the mesh-delivery seat (novox/hq ADR 0239, to-be 47): the one writer of deliveries and +// groups. Every input — a pull request's head, a verdict, a merge, a walk's record, a person's act — only +// changes a delivery's facts; settle then moves it through the table as far as its facts allow, so the +// order the inputs arrive in never decides where a delivery stands. A delivery is kept before anything +// about it is said, and what is owed outside — the event, the commit's note, the pull request's view and +// statuses — is retried until done. + +// Holder is the seat's holder. +type Holder struct { + Store Store + Controller Controller + Forge Forge + Emit func(event string, body any) error + Now func() time.Time + Logf func(format string, a ...any) + // Operator is the operator channel, asked for the Approve of a merge into main (approval.go). + Operator Operator + // Pulls keeps the pull requests' keys on the bus (pullkey.go); nil keeps none. + Pulls PullKeys + // cancels are asks to take back, gathered under the lock and run with the other asks of the look. + cancels []func() + + mu sync.Mutex + deliveries map[string]*Delivery + groups map[string]*Group + dirty map[string]bool + // written is each delivery as it was last written, encoded: a delivery that encodes the same is not + // written again (novox/hq issue 326). + written map[string][]byte + dirtyGroup map[string]bool + unmatched map[string]unmatched + backlog []Walk + started time.Time + loaded bool + held bool + reconciled time.Time + unreached string + // askTried is when a check this owner asks was last tried and could not be asked: tried again a minute on. + askTried map[string]time.Time + // walkAsked is when a walk that carried a step still judging was last asked for by its id: once an hour. + walkAsked map[string]time.Time + refusals []string + // unchecked are the open pull requests' heads seen without a merge check (unchecked.go). + unchecked map[string]unchecked + // walks are the walks' and batches' last records (answers.go); orphans the waiting walks no live delivery + // holds, by when that was first seen; turning the walks whose turn came this look, for some deliveries. + walks map[string]seenWalk + orphans map[string]time.Time + turning map[string]*turnWant + // waitsWhy is, by walk, why the last look did not give a waiting walk its word: a delivery it carries not at + // its turn, or a stopped change it carries (novox/hq issue 529). Said by WalksWaiting. + waitsWhy map[string]string + // asking are the walks whose word is being asked right now, outside the lock: not asked twice meanwhile. + asking map[string]bool + // deliverErr is, by walk, the controller's last refusal of its word, until it is given (review of issue 753). + deliverErr map[string]string + // leaving are, by walk, the merges a stop gathered to leave out of it, asked once (novox/hq ADR 0299). + leaving map[string][]string + // leaveAsked is when a walk was last asked to leave out the stopped merges it carries (leaveStopped). + leaveAsked map[string]leaveAsk + // leaveRefused is, by walk, what the controller answered when it refused both to leave a stopped merge out and + // to stop the walk whole. + leaveRefused map[string]string + // coverAsked are the deliveries that failed or were stopped whose cover by a later finished walk waits for the + // forge's word on containment, worked out each pass (novox/hq issue 747): a walk carrying a merge stopped for one of + // them is held back meanwhile, never asked to leave it out before the answer. + coverAsked map[string]bool + // coverAskedSince is, by delivery, when the forge was first asked about its cover and has not answered: past + // coverAskBound the walk is no longer held back for it, and is asked to leave the merge out (review of issue 747). + coverAskedSince map[string]time.Time + // tried counts the owed effects Flush tried since this owner started, by how each ended (novox/hq issue 407). + tried owedTries + // forgeWanted is whether the forge's open pull requests are to be read at the next React: a check may have + // answered (events.go). reactMu keeps one React at a time. + forgeWanted bool + // rereadWalks is whether the controller's walks are to be read once more: a merge passed the bound a walk opens + // within, and the walks were last read before it (unwalkedPast). + rereadWalks bool + reactMu sync.Mutex + // poke asks for a look (react.go); wanted is when the next is wanted, and its timer. + poke chan struct{} + wanted lookWanted +} + +// owedTries is how many owed effects were tried since this owner started, and when the last was: three numbers +// and a time, however many were tried. Not kept: a restarted owner counts from its start, and says since when, so a +// window's count is the difference of two readings in the same run (novox/hq issue 407). +type owedTries struct { + done, failed, givenUp int + last time.Time +} + +// unmatched is a walk no delivery is known for yet: its merge not heard, or merged before this owner held. +type unmatched struct { + walk Walk + first time.Time +} + +// The words this owner signs its acts with. +const ( + byOwner = "mesh-delivery" + byH2 = "healer H2" +) + +// How long a walk is given for its merge to be heard before it is made a delivery of its own. +const matchGrace = 2 * time.Minute + +// How long a merged delivery may go without a walk before it is held, saying so. +const unwalked = 10 * time.Minute + +// How long a final delivery is kept; its note on the commit keeps it after. +const keptFinal = 30 * 24 * time.Hour + +func (h *Holder) init() { + if h.Now == nil { + h.Now = func() time.Time { return time.Now().UTC() } + } + if h.Logf == nil { + h.Logf = func(string, ...any) {} + } + if h.Emit == nil { + h.Emit = func(string, any) error { return nil } + } + h.poke = make(chan struct{}, 1) + h.deliveries = map[string]*Delivery{} + h.groups = map[string]*Group{} + h.dirty = map[string]bool{} + h.written = map[string][]byte{} + h.dirtyGroup = map[string]bool{} + h.unmatched = map[string]unmatched{} + h.waitsWhy = map[string]string{} + h.askTried = map[string]time.Time{} + h.walkAsked = map[string]time.Time{} + h.unchecked = map[string]unchecked{} + h.walks = map[string]seenWalk{} + h.orphans = map[string]time.Time{} + h.asking = map[string]bool{} + h.started = h.Now() +} + +// Load reads back every delivery and group, before any input is taken: a restarted holder resumes each +// where it stood. +func (h *Holder) Load() error { + ds, err := h.Store.Deliveries() + if err != nil { + return err + } + gs, err := h.Store.Groups() + if err != nil { + return err + } + h.mu.Lock() + defer h.mu.Unlock() + for _, d := range ds { + h.deliveries[d.ID] = d + if b, err := json.Marshal(d); err == nil { + h.written[d.ID] = b + } + } + for _, g := range gs { + h.groups[g.ID] = g + } + h.loaded = true + h.Logf("[mesh-delivery] read back %d delivery(ies) and %d group(s)", len(ds), len(gs)) + return nil +} + +// Refuse keeps a refusal from outside the lock. +func (h *Holder) Refuse(err error) { + h.mu.Lock() + defer h.mu.Unlock() + h.refuse(err) +} + +// refuse keeps a refusal, said, for the status; under the lock. +func (h *Holder) refuse(err error) { + h.Logf("[mesh-delivery] %v", err) + h.refusals = append(h.refusals, h.Now().Format(time.RFC3339)+" "+err.Error()) + if len(h.refusals) > 50 { + h.refusals = h.refusals[len(h.refusals)-50:] + } +} + +// moved records what a delivery's transitions owe outside, and keeps it. +func (h *Holder) moved(d *Delivery, ts []Transition) { + now := h.Now() + opened := h.openedOf(d) + for _, t := range ts { + h.Logf("[mesh-delivery] %s: %s → %s (%s): %s", d.ID, orNothing(t.From), t.To, t.Event, t.Why) + d.Owed = append(d.Owed, Effect{Kind: EffectEmit, Line: string(mustJSON(transitionEvent(d, t, opened))), Since: now}) + line := noteLine(d, t) + d.Owed = append(d.Owed, Effect{Kind: EffectNote, Commit: d.Commit, Line: line, Since: now}) + if d.MergedAs != "" && !sameCommit(d.MergedAs, d.Commit) && afterTheTrunk(t.To) { + d.Owed = append(d.Owed, Effect{Kind: EffectNote, Commit: d.MergedAs, Line: line, Since: now}) + } + if t.To.Final() && d.MergedAs != "" && len(d.Steps) > 0 { + d.Owed = append(d.Owed, Effect{Kind: EffectNote, Commit: d.MergedAs, Line: executedLine(d), Since: now}) + } + } + if len(ts) > 0 && d.shownOnItsPullRequest() { + d.owe(Effect{Kind: EffectView, Since: now}) + state, desc := statusOf(d) + d.owe(Effect{Kind: EffectStatus, Commit: d.Commit, Context: "mesh/delivery", State: state, Line: desc, Since: now}) + } + h.keep(d) +} + +// owe adds an effect that stands for the latest of its kind: a view or a commit's status is refreshed, not +// queued twice. +func (d *Delivery) owe(e Effect) { + for i, o := range d.Owed { + if o.Kind == e.Kind && o.Commit == e.Commit && o.Context == e.Context { + d.Owed[i] = e + return + } + } + d.Owed = append(d.Owed, e) +} + +func afterTheTrunk(s State) bool { + return s == Published || s == Delivering || s == Held || s == Delivered || s == Failed || s == Stopped || + s == Superseded || s == Retired || s == Covered +} + +func orNothing(s State) string { + if s == None { + return "(new)" + } + return string(s) +} + +// keep writes a delivery; one that could not be written is written again by Flush before anything about it +// is said. A delivery that encodes exactly as it was last written is not written again: a revision of its key +// means the delivery changed (novox/hq issue 326). Any change to anything it holds — its state, steps, walk, +// owed effects and their tries — changes its encoding, so it is written. +func (h *Holder) keep(d *Delivery) { + h.deliveries[d.ID] = d + b, encoded := json.Marshal(d) + if encoded == nil && !h.dirty[d.ID] && h.written[d.ID] != nil && bytes.Equal(b, h.written[d.ID]) { + return + } + if err := h.Store.PutDelivery(d); err != nil { + h.dirty[d.ID] = true + delete(h.written, d.ID) + h.Logf("[mesh-delivery] %s could not be kept yet (%v); kept again before it is said", d.ID, err) + h.again("a delivery that could not be kept") + return + } + delete(h.dirty, d.ID) + if encoded == nil { + h.written[d.ID] = b + } else { + delete(h.written, d.ID) + } +} + +func (h *Holder) keepGroup(g *Group) { + h.groups[g.ID] = g + if err := h.Store.PutGroup(g); err != nil { + h.dirtyGroup[g.ID] = true + h.Logf("[mesh-delivery] group %s could not be kept yet (%v)", g.ID, err) + h.again("a group that could not be kept") + return + } + delete(h.dirtyGroup, g.ID) +} + +// settleAndKeep moves a delivery as far as its facts allow, and keeps it. +func (h *Holder) settleAndKeep(d *Delivery) { + ts := settle(d, Facts{Now: h.Now()}) + h.moved(d, ts) + h.closeGroupOf(d) +} + +// --- inputs --------------------------------------------------------------------------------------------- + +// PullEvent is the forge's `pull.updated`, `pull.merged` or `pull.closed`. +type PullEvent struct { + Owner string `json:"owner"` + Repo string `json:"repo"` + Number int `json:"number"` + Title string `json:"title,omitempty"` + Body string `json:"body,omitempty"` + Base string `json:"base"` + Head string `json:"head"` + HeadSHA string `json:"head_sha"` + MergeCommit string `json:"merge_commit_sha,omitempty"` + MergedAt string `json:"merged_at,omitempty"` + CloneURL string `json:"clone_url,omitempty"` + HTMLURL string `json:"html_url,omitempty"` + Paths []string `json:"paths,omitempty"` + PathsTruncated bool `json:"paths_truncated,omitempty"` + Removed []string `json:"removed,omitempty"` + ModuleDirs []string `json:"module_dirs,omitempty"` + ModuleDirsSaid bool `json:"module_dirs_said,omitempty"` + HeadChecks map[string]string `json:"head_checks,omitempty"` + // BaseSHA is the base branch's head when the head was announced (novox/hq ADR 0323). + BaseSHA string `json:"base_sha,omitempty"` + // Merged is the forge's own events' word that the pull request merged (gitea.forge.pull-request.*): its close is no + // close unmerged (issue 1107). + Merged bool `json:"merged,omitempty"` +} + +func (p PullEvent) repository() string { return strings.ToLower(p.Owner + "/" + p.Repo) } + +// trunkLike are branch names that never make a group: a pull request from them is nobody's feature. +var trunkLike = map[string]bool{"main": true, "master": true, "trunk": true, "": true} + +// PullUpdated takes a pull request's new head: a delivery proposed, an older head of the same pull request +// superseded, and the delivery put in the group its branch name makes. +func (h *Holder) PullUpdated(p PullEvent) (isNew bool) { + if p.HeadSHA == "" || p.Number == 0 { + h.Refuse(errors.New("a pull request announced with no head or no number; nothing proposed")) + return false + } + h.mu.Lock() + defer h.mu.Unlock() + return h.pullUpdated(p) +} + +// pullUpdated is PullUpdated under the lock. +func (h *Holder) pullUpdated(p PullEvent) (isNew bool) { + id := IDOf(p.repository(), p.HeadSHA) + isNew = h.deliveries[id] == nil + now := h.Now() + startedOver := "" + if d := h.deliveries[id]; d != nil { + if startedOver = startedOverFor(d, p.Number); startedOver == "" { + // Announced again: nothing new — but for a delivery whose verdict is out of date, the announcement is its + // check asked again against the trunk's head (the merge tool's ask, novox/hq ADR 0323 §5), and its bound + // runs from now (§7). The base head it was announced with is kept, for a check this owner asks after it. + if p.BaseSHA != "" && d.BaseSHA != p.BaseSHA { + d.BaseSHA = p.BaseSHA + h.keep(d) + } + // The review record, corrected in the description without a push, is read again (ADR 0421 §5). + if by, at := ReviewIn(p.Body); p.Body != "" && (by != d.ReviewedBy || at != d.ReviewedAt) { + d.ReviewedBy, d.ReviewedAt = by, at + h.keep(d) + } + // An `after:` line written into the description later is told about too (novox/hq ADR 0419 §2). + if after := AfterIn(p.Body); p.Body != "" && !slices.Equal(after, d.After) { + d.After = after + h.keep(d) + } + if d.State == Proposed && d.outOfDate() != "" && d.AskedAgain == nil { + d.AskedAgain = &now + h.keep(d) + } + // Another pull request at this head, announced while this one is live (issue 1114): kept, so this one's close + // starts the record over for it. + if d.Number != p.Number && !d.State.Final() && d.MergedAs == "" { + d.Others = withOther(d.Others, p) + h.keep(d) + } + return + } + // A delivery stopped only because its pull request closed unmerged does not end its commit's delivery for + // another pull request with the same head (novox/hq issue 783): started over below, exactly as for a first + // announcement, under the same id — never a transition out of Stopped. + } + if startedOver != "" { + // The record the new pull request takes over leaves the group it was in (issue 1083): the new one joins its own. + if old := h.deliveries[id]; old != nil && old.Group != "" { + h.leaveGroup(old) + } + } + replaced := "" + for _, old := range h.deliveries { + if old.Repository == p.repository() && old.Number == p.Number && !sameCommit(old.Commit, p.HeadSHA) && + !old.State.Final() && old.MergedAs == "" { + old.NewerHead = p.HeadSHA + h.settleAndKeep(old) + if old.Group != "" { + h.leaveGroup(old) + replaced = old.ID + } + } + } + d, t, err := h.freshAnnouncement(p, now, startedOver) + if err != nil { + h.refuse(err) + return false + } + ts := []Transition{t} + // An announcement heard again from the bus's history may carry its head's verdict already: the forge holds + // the gate's status on that very commit. A decided one is taken; a pending or errored one is asked again. + if v := decidedOnHead(p.HeadChecks, now); v != nil { + d.Check = v + ts = append(ts, settle(d, Facts{Now: now})...) + } + h.moved(d, ts) + if !h.takePlaceOf(replaced, d) { + h.joinGroup(d) + } + return isNew +} + +// PullEdited takes a pull request's edit (novox/hq issue 1079): its description is read again for the review record, the +// `after:` lines and the issues it names, exactly as an announcement reads them. An Approve ask still unanswered whose words +// the new record changes is withdrawn and asked again with it read, without a word on the pull request: nothing was +// approved, so no Approve ended. An answered ask is left as it is. The record is never a gate (ADR 0421 §5). An edit with no +// description, or of a pull request with no open delivery at that head, changes nothing. +func (h *Holder) PullEdited(p PullEvent) { + if p.HeadSHA == "" || p.Number == 0 { + return + } + h.mu.Lock() + defer h.mu.Unlock() + d := h.deliveries[IDOf(p.repository(), p.HeadSHA)] + if d != nil && startedOverFor(d, p.Number) != "" { + // The record at this head is stopped under another, closed pull request (issue 1114: this one was announced while + // that one was live and never heard again): the edit takes it over, as an announcement would. The forge's edit + // event carries no paths; the record's, for the same head, are the ones. + if p.Paths == nil { + p.Paths, p.PathsTruncated, p.Removed = d.Paths, d.PathsTruncated, d.Removed + p.ModuleDirs, p.ModuleDirsSaid = d.ModuleDirs, d.ModuleDirsSaid + } + if p.CloneURL == "" { + p.CloneURL = d.CloneURL + } + if p.BaseSHA == "" { + p.BaseSHA = d.BaseSHA + } + h.pullUpdated(p) + return + } + if p.Body == "" { + return + } + if d == nil || d.Number != p.Number || d.State.Final() || d.MergedAs != "" { + return + } + before, _ := reviewWords(d) + d.ReviewedBy, d.ReviewedAt = ReviewIn(p.Body) + d.After = AfterIn(p.Body) + title := p.Title + if title == "" { + title = d.Title + } + d.Issues = IssuesIn(title + "\n" + p.Body) + h.keep(d) + if after, _ := reviewWords(d); after == before { + return + } + if r := h.covering(d); r != nil && r.AskID != "" && r.Approved == nil && r.Declined == nil { + h.cancels = append(h.cancels, h.withdrawAsk(r.AskID, d.ID+"'s review record changed: "+shortOf(d.ReviewedAt, 12))) + } +} + +// withOther is others with p put in, replacing an earlier announcement of the same pull request. +func withOther(others []PullEvent, p PullEvent) []PullEvent { + out := withoutOther(others, p.Number) + return append(out, p) +} + +// withoutOther is others without the pull request number. +func withoutOther(others []PullEvent, number int) []PullEvent { + var out []PullEvent + for _, o := range others { + if o.Number != number { + out = append(out, o) + } + } + return out +} + +// startedOverFor is why a delivery already kept at a head's id is started over for a pull request's number rather +// than taken as announced again: it is Stopped, never merged, and under another number. Stopped by an earlier, closed +// duplicate pull request with the same head (novox/hq issue 783), or by a person or a group before its pull request +// closed (issue 1083: the new pull request at that commit takes the record over, the key staying repository@commit). +// Empty for the ordinary 'announced again' path, or for the same pull request's own stop. +func startedOverFor(d *Delivery, number int) string { + if d == nil || d.State != Stopped || d.MergedAs != "" || d.Number == number { + return "" + } + if d.ClosedUnmerged { + return fmt.Sprintf("#%d with the same head closed unmerged", d.Number) + } + said := fmt.Sprintf("#%d with the same head was stopped", d.Number) + if n := len(d.Transitions); n > 0 && d.Transitions[n-1].To == Stopped { + last := d.Transitions[n-1] + if last.By != "" { + said += " by " + last.By + } + if last.Why != "" { + said += ": " + last.Why + } + } + return said +} + +// freshAnnouncement makes a delivery for p's head exactly as a first announcement would: its own id, proposed. +// startedOver, when not empty, is folded into the transition's Why, naming the pull request this replaces +// (novox/hq issue 783); the delivery it replaces is swapped out by the caller keeping the new one under the same +// id (keep), never by a transition out of its final state. +func (h *Holder) freshAnnouncement(p PullEvent, now time.Time, startedOver string) (*Delivery, Transition, error) { + d := &Delivery{ID: IDOf(p.repository(), p.HeadSHA), Repository: p.repository(), Commit: p.HeadSHA, Number: p.Number, + Title: p.Title, Base: p.Base, Branch: p.Head, HTMLURL: p.HTMLURL, CloneURL: p.CloneURL, Paths: p.Paths, + PathsTruncated: p.PathsTruncated, Removed: p.Removed, ModuleDirs: p.ModuleDirs, ModuleDirsSaid: p.ModuleDirsSaid, + After: AfterIn(p.Body), Issues: IssuesIn(p.Title + "\n" + p.Body), Created: now, BaseSHA: p.BaseSHA} + d.ReviewedBy, d.ReviewedAt = ReviewIn(p.Body) + why := fmt.Sprintf("%s#%d's head announced", d.Repository, d.Number) + if startedOver != "" { + why = fmt.Sprintf("%s; %s", why, startedOver) + } + t, err := Apply(d, EvAnnounced, false, Facts{Now: now, Why: why}) + return d, t, err +} + +// takePlaceOf puts a member's new head — a rebase, a push — in its group in the place of the head it superseded, +// when the group is closed: a member of it already reached the trunk, and a closed group takes no new member. The +// new head has its own check, and the group's composed check runs again for the new heads rather than holding the +// group for a person (novox/hq ADR 0323 §6, issue 406). An open group takes it as joinGroup does. +func (h *Holder) takePlaceOf(old string, d *Delivery) bool { + if old == "" { + return false + } + was := h.deliveries[old] + if was == nil || was.Group == "" { + return false + } + if g := h.groups[was.Group]; g != nil && g.Closed { + if !containsString(g.Members, d.ID) { + g.Members = append(g.Members, d.ID) + sort.Strings(g.Members) + } + d.Group = g.ID + g.Updated = h.Now() + h.keep(d) + h.keepGroup(g) + h.Logf("[mesh-delivery] group %s: %s takes the place of %s, which it superseded", g.ID, d.ID, old) + return true + } + return false +} + +// decidedOnHead is the verdict the forge's statuses on a head hold, when the gate decided one: passed, warned +// or failed. Nil for none, pending, or a check that could not run — those are asked again. +func decidedOnHead(checks map[string]string, now time.Time) *Verdict { + switch checks["mesh/merge-gate"] { + case "success", "warning", "failure": + default: + return nil + } + return &Verdict{At: now, Gate: verdictOfState(checks["mesh/merge-gate"]), + Summary: "the head's mesh/merge-gate, as the forge holds it", Repo: verdictOfState(checks["mesh/repo-check"])} +} + +// joinGroup puts a new head in the group its branch name makes with the other repositories' open pull +// requests (ADR 0239 decision 4): one mechanism, the name the feature already has in every repository. +func (h *Holder) joinGroup(d *Delivery) { + if trunkLike[strings.ToLower(d.Branch)] || strings.EqualFold(d.Branch, d.Base) { + return + } + g := h.groups[d.Branch] + if g != nil && g.Closed { + h.Logf("[mesh-delivery] %s shares the branch %s with a group that is already delivering: a lone delivery", + d.ID, d.Branch) + return + } + var others []*Delivery + for _, o := range h.deliveries { + if o.ID != d.ID && o.Branch == d.Branch && o.Repository != d.Repository && o.Number > 0 && + !o.State.Final() && o.MergedAs == "" { + others = append(others, o) + } + } + if len(others) == 0 && g == nil { + return + } + now := h.Now() + if g == nil { + g = &Group{ID: d.Branch, Created: now} + } + for _, o := range append(others, d) { + if !containsString(g.Members, o.ID) { + g.Members = append(g.Members, o.ID) + } + if o.Group != g.ID { + o.Group = g.ID + h.keep(o) + } + } + sort.Strings(g.Members) + g.Updated = now + h.keepGroup(g) + h.Logf("[mesh-delivery] group %s: %s", g.ID, strings.Join(g.Members, ", ")) +} + +// leaveGroup takes a superseded or stopped head out of its group's members. +func (h *Holder) leaveGroup(d *Delivery) { + g := h.groups[d.Group] + if g == nil { + return + } + var kept []string + for _, m := range g.Members { + if m != d.ID { + kept = append(kept, m) + } + } + g.Members = kept + g.Updated = h.Now() + h.keepGroup(g) +} + +// PullMerged takes a merge: the commit it landed as, and the delivery it was — made from the forge's word +// when this owner never heard its head (merged before it held the seat). +func (h *Holder) PullMerged(p PullEvent) { + if p.MergeCommit == "" { + h.Refuse(errors.New("a merge announced with no merge commit; nothing published")) + return + } + h.mu.Lock() + defer h.mu.Unlock() + h.forgetUnchecked(p.Owner, p.Repo, p.Number) + var d *Delivery + if p.HeadSHA != "" { + d = h.deliveries[IDOf(p.repository(), p.HeadSHA)] + } + if d == nil { + var newest time.Time + for _, c := range h.deliveries { + // One already on the trunk is not merged again: a forge whose numbers began again reuses them. + if c.Repository == p.repository() && c.Number == p.Number && c.NewerHead == "" && c.MergedAs == "" && + c.Created.After(newest) { + d, newest = c, c.Created + } + } + } + if d != nil && d.MergedAs != "" && sameCommit(d.MergedAs, p.MergeCommit) { + return // this merge is already taken: heard again, past the bus's de-duplication window, it changes nothing + } + now := h.Now() + var ts []Transition + if over := startedOverFor(d, p.Number); over != "" { + // A delivery stopped only by an earlier, closed duplicate pull request with the same head is not lost when + // its merge is heard without the open announcement either: it is started over for the merging pull request + // the same way an announcement would, before the merge is taken as usual (novox/hq issue 783). Heard this + // way — its own merge, with no open announcement of this pull request — it is made exactly as a first-heard + // merge in every field but its id: FromForge, and its check the forge's word on the head, so a long-past + // merge replayed from the bus's history still falls under the history rule (issue 313) and is not held for + // a person (review round 1). + if d.Group != "" { + h.leaveGroup(d) // issue 1083: the record taken over leaves its group + } + fresh, t, err := h.freshAnnouncement(p, now, over) + if err != nil { + h.refuse(err) + return + } + fresh.FromForge = true + if s := p.HeadChecks["mesh/merge-gate"]; s != "" { + fresh.Check = &Verdict{At: now, Gate: verdictOfState(s), Summary: "the head's mesh/merge-gate, as the forge holds it", + Repo: verdictOfState(p.HeadChecks["mesh/repo-check"])} + } + d, ts = fresh, []Transition{t} + if d.Check != nil { + ts = append(ts, settle(d, Facts{Now: now})...) + } + } else if d == nil { + head := p.HeadSHA + if head == "" { + head = p.MergeCommit + } + d = &Delivery{ID: IDOf(p.repository(), head), Repository: p.repository(), Commit: head, Number: p.Number, + Title: p.Title, Base: p.Base, Branch: p.Head, HTMLURL: p.HTMLURL, CloneURL: p.CloneURL, Paths: p.Paths, + PathsTruncated: p.PathsTruncated, Removed: p.Removed, ModuleDirs: p.ModuleDirs, ModuleDirsSaid: p.ModuleDirsSaid, + After: AfterIn(p.Body), Issues: IssuesIn(p.Title + "\n" + p.Body), Created: now, FromForge: true} + // Its check, as the forge holds it on the head: the verdict the pull request merged with. + if s := p.HeadChecks["mesh/merge-gate"]; s != "" { + d.Check = &Verdict{At: now, Gate: verdictOfState(s), Summary: "the head's mesh/merge-gate, as the forge holds it", + Repo: verdictOfState(p.HeadChecks["mesh/repo-check"])} + } + t, err := Apply(d, EvAnnounced, false, Facts{Now: now, Why: fmt.Sprintf("%s#%d merged; its head was not "+ + "heard by this owner, so it is made from the forge's word", d.Repository, d.Number)}) + if err != nil { + h.refuse(err) + return + } + ts = append(ts, t) + } + d.MergedAs = p.MergeCommit + if t, err := time.Parse(time.RFC3339, p.MergedAt); err == nil { + d.MergedAt = t.UTC() + } else { + d.MergedAt = now + } + // A walk already heard of that answers this merge: its own, or one that names it. + h.linkAnswering(d) + // A merge long past heard only now, from the forge's word — replayed from the bus's history when this owner + // first held the seat — is history: it reached the machines before this owner was there to deliver it, and + // holding it for a person asks a person about the past (novox/hq issue 313). + if d.FromForge && d.History == "" && isHistory(d) == nil { + d.History = fmt.Sprintf("merged %s, %s before this owner heard of it from the forge's word: history, "+ + "never to be walked", d.MergedAt.UTC().Format(time.RFC3339), now.Sub(d.MergedAt).Round(time.Minute)) + } + ts = append(ts, settle(d, Facts{Now: now})...) + if d.State == Held && d.HeldWhy == "" { + d.HeldWhy, d.Unwalked = "merged without a passing check: only a person decides that it goes on", false + } + h.moved(d, ts) + h.closeGroupOf(d) + h.trunkMoved(d, p.Base, p.MergeCommit, now) +} + +// trunkMoved takes a merge into a trunk as that trunk's move (novox/hq ADR 0323 §7): every other open delivery into +// it keeps the trunk's new head, and a ready one whose verdict names the commit before goes back to proposed. A +// group's composed check goes out of date when the merge is not one of its members and a member still open merges +// into that trunk (§6); the group then reads checking. Nothing is asked here: at the same look askFresh asks the check of +// a group's member, or of a head with an Approve asked or given (issue 1047); a lone head's is judged again by a recheck +// or the merge tool (the rest of ADR 0395 §1 is issue 888). +func (h *Holder) trunkMoved(merged *Delivery, base, commit string, now time.Time) { + if base == "" || commit == "" { + return + } + for _, o := range h.deliveries { + if o == merged || o.Repository != merged.Repository || o.Base != base || o.MergedAs != "" || o.State.Final() || + sameCommit(o.TrunkNow, commit) { + continue + } + // A check asked again against the trunk before this move judged that trunk: it is out of date too. + o.TrunkNow, o.AskedAgain = commit, nil + h.settleAndKeep(o) + } + for _, g := range h.groups { + if g.ID == merged.Group || g.Check == nil || g.Check.Verdict == "" || g.Check.OutOfDate != "" { + continue + } + for _, m := range h.membersOf(g) { + if m.Repository != merged.Repository || m.Base != base || m.MergedAs != "" || m.State.Final() { + continue + } + judged := g.Check.Trunks[m.ID] + if judged == "" { + judged = "?" + } + g.Check.OutOfDate = fmt.Sprintf("composed check out of date: %s was judged against trunk %s, the trunk is now %s "+ + "(%s merged)", m.ID, shortOf(judged, 12), shortOf(commit, 12), merged.ID) + g.Check.OutOfDateAt = now + g.Updated = now + h.keepGroup(g) + h.Logf("[mesh-delivery] group %s: %s", g.ID, g.Check.OutOfDate) + break + } + } +} + +// closeGroupOf closes a delivery's group once it reached the trunk: a delivering group takes no new member. +func (h *Holder) closeGroupOf(d *Delivery) { + if d.Group == "" || d.MergedAs == "" { + return + } + if g := h.groups[d.Group]; g != nil && !g.Closed { + g.Closed = true + g.Updated = h.Now() + h.keepGroup(g) + } +} + +// verdictOfState is a commit status's state as a verdict. +func verdictOfState(s string) string { + switch s { + case "success": + return "pass" + case "warning": + return "warning" + case "failure": + return "fail" + case "": + return "" + } + return "error" +} + +// PullClosed takes a pull request closed unmerged: its open deliveries are stopped. +// +// **And what its checks built for its merge goes** (novox/hq ADR 0350 §5): nothing will take it now. Asked of the +// controller once the deliveries are settled, outside the lock; a refusal is said, and the build seat's holders let +// go of it within the week anyway. +func (h *Holder) PullClosed(p PullEvent) { + h.mu.Lock() + h.forgetUnchecked(p.Owner, p.Repo, p.Number) + var over *PullEvent + for _, d := range h.deliveries { + if d.Repository != p.repository() { + continue + } + if d.Number != p.Number { + // The other pull request at this head closed first: it is no longer one to take the record over. + if len(d.Others) > 0 { + if rest := withoutOther(d.Others, p.Number); len(rest) != len(d.Others) { + d.Others = rest + h.keep(d) + } + } + continue + } + if !d.State.Final() && d.MergedAs == "" { + d.ClosedUnmerged = true + h.settleAndKeep(d) + if d.Group != "" { + h.leaveGroup(d) + } + // Another open pull request carries this head (issue 1114): the record starts over for it at once, as a first + // announcement of it would, so it gets its check, its review record and its Approve ask. + if len(d.Others) > 0 { + next := d.Others[0] + over = &next + } + } + } + if over != nil { + h.pullUpdated(*over) + } + h.mu.Unlock() + if h.Controller == nil || p.Owner == "" || p.Repo == "" || p.Number <= 0 { + return + } + if err := h.Controller.ForgetChecks(p.Owner, p.Repo, p.Number); err != nil { + h.mu.Lock() + h.refuse(fmt.Errorf("%s#%d closed, and its check builds stay until the week's sweep: %w", p.repository(), + p.Number, err)) + h.mu.Unlock() + } +} + +// CheckedEvent is the controller's `checked`. +type CheckedEvent struct { + Owner string `json:"owner"` + Repo string `json:"repo"` + Number int `json:"number,omitempty"` + Commit string `json:"commit"` + Verdict string `json:"verdict"` + Summary string `json:"summary"` + ID string `json:"id"` + On string `json:"on,omitempty"` + Report string `json:"report,omitempty"` + Gate *struct { + Verdict string `json:"verdict"` + Summary string `json:"summary"` + Modules []string `json:"modules,omitempty"` + Dependents []string `json:"dependents,omitempty"` + } `json:"gate,omitempty"` + RepoCheck *struct { + Verdict string `json:"verdict"` + Summary string `json:"summary"` + } `json:"repo-check,omitempty"` + Plan *DeliveryPlan `json:"plan,omitempty"` + Group string `json:"group,omitempty"` + Members []struct { + Owner string `json:"owner"` + Repo string `json:"repo"` + Commit string `json:"commit"` + Trunk string `json:"trunk,omitempty"` + } `json:"members,omitempty"` + // Trunk is the trunk commit the head was merged into and judged as, Beside the other repositories' commits the + // check read, and Took how long it ran (novox/hq ADR 0323 §3). + Trunk string `json:"trunk,omitempty"` + Beside map[string]string `json:"beside,omitempty"` + Took string `json:"took,omitempty"` +} + +// Checked takes a verdict: a head's own, or a group's composed check. +func (h *Holder) Checked(c CheckedEvent) { + h.mu.Lock() + defer h.mu.Unlock() + now := h.Now() + if c.Group != "" { + g := h.groups[c.Group] + if g == nil || g.Check == nil || g.Check.Asked != c.ID { + h.Logf("[mesh-delivery] a composed check of %s (%s) that is not the one asked: history", c.Group, c.ID) + return + } + g.Check.Verdict, g.Check.Summary = c.Verdict, c.Summary + for _, m := range h.membersOf(g) { + h.mergeAgainNow(m) // the group's composed check answered: its members' merges are asked at once + } + // Each member's head and the trunk commit it was merged into (novox/hq ADR 0323 §6). + for _, m := range c.Members { + for id := range g.Check.Commits { + if d := h.deliveries[id]; d != nil && strings.EqualFold(d.Repository, m.Owner+"/"+m.Repo) && + sameCommit(d.Commit, m.Commit) && m.Trunk != "" { + if g.Check.Trunks == nil { + g.Check.Trunks = map[string]string{} + } + g.Check.Trunks[id] = m.Trunk + } + } + } + // A composed check that was on its way when a trunk moved lands judged against the old trunk: out of date as it lands + // (novox/hq ADR 0403 §6, issue 1047), and asked again once a member is judged again. + if g.Check.OutOfDate == "" { + for id, judged := range g.Check.Trunks { + if d := h.deliveries[id]; d != nil && d.MergedAs == "" && !d.State.Final() && d.TrunkNow != "" && !sameCommit(judged, d.TrunkNow) { + g.Check.OutOfDate = fmt.Sprintf("composed check out of date: %s was judged against trunk %s, the trunk is now %s", + id, shortOf(judged, 12), shortOf(d.TrunkNow, 12)) + // From when the composed check was asked: a member judged since then, even before this verdict landed, + // counts as judged again. + g.Check.OutOfDateAt = g.Check.At + h.Logf("[mesh-delivery] group %s: %s", g.ID, g.Check.OutOfDate) + break + } + } + } + g.Updated = now + h.keepGroup(g) + h.Logf("[mesh-delivery] group %s composed: %s — %s", g.ID, c.Verdict, c.Summary) + return + } + repository := strings.ToLower(c.Owner + "/" + c.Repo) + id := IDOf(repository, c.Commit) + d := h.deliveries[id] + if d == nil { + if c.Number == 0 { + return // a check of no pull request: nothing to deliver + } + d = &Delivery{ID: id, Repository: repository, Commit: c.Commit, Number: c.Number, Created: now} + t, err := Apply(d, EvAnnounced, false, Facts{Now: now, Why: "its verdict arrived before its head was heard"}) + if err != nil { + h.refuse(err) + return + } + h.moved(d, []Transition{t}) + } + if d.State == Rejected && c.Trunk != "" && d.Check != nil && !sameCommit(d.Check.Trunk, c.Trunk) { + // Judged again against another trunk commit — the merge tool's ask of a verdict out of date, or a check asked + // elsewhere (novox/hq ADR 0323 §5, §7): taken as the recheck it is. + t, err := Apply(d, EvRecheck, true, Facts{Now: now, By: byOwner, Why: fmt.Sprintf("judged again against trunk %s "+ + "(it was rejected against trunk %s)", shortOf(c.Trunk, 12), shortOf(d.Check.Trunk, 12))}) + if err != nil { + h.refuse(err) + return + } + h.moved(d, []Transition{t}) + } + if d.State != Proposed { + h.Logf("[mesh-delivery] %s is %s: a verdict for it now is history (%s)", d.ID, d.State, c.Verdict) + return + } + v := &Verdict{ID: c.ID, On: c.On, At: now, Gate: c.Verdict, Summary: c.Summary, Report: c.Report, + Head: shortOf(c.Commit, 12), Trunk: c.Trunk, Beside: c.Beside, Took: c.Took} + if c.Gate != nil { + v.Gate, v.Summary = c.Gate.Verdict, c.Gate.Summary + v.Modules, v.Dependents = c.Gate.Modules, c.Gate.Dependents + } + if c.RepoCheck != nil { + v.Repo, v.RepoSaid = c.RepoCheck.Verdict, c.RepoCheck.Summary + } + if v.Gate == "" { + v.Gate = "error" + } + d.Check, d.AskedAgain = v, nil + h.mergeAgainNow(d) // a verdict taken: the merge is asked at once, not a minute after a refusal + if c.Plan != nil { + d.Plan = c.Plan + } + if why := d.outOfDate(); why != "" { + // Kept and shown, and it does not count: the delivery stays proposed (ADR 0323 §7). If it passes, its check is + // asked against the trunk's head before the operator is (ADR 0421 §3, askFresh). + d.FreshWanted = d.Check.Passes() + h.Logf("[mesh-delivery] %s: its verdict %s is %s", d.ID, c.ID, why) + } + h.settleAndKeep(d) +} + +// WalkMoved takes a walk's record, from `plan-moved` or the controller's walks: the delivery it walks +// moves with it; a backlog walk's carried builds are credited to the deliveries that published them. +func (h *Holder) WalkMoved(w Walk) { + h.mu.Lock() + defer h.mu.Unlock() + h.walkMoved(w) +} + +func (h *Holder) walkMoved(w Walk) { + for _, d := range h.deliveries { + if d.Group != "" { + h.mergeAgainNow(d) // a walk moved: a group member's merge may be decided by it, asked at once + } + } + if w.Release != nil { + h.creditBacklog(w) + return + } + if old, seen := h.walks[w.ID]; !seen || w.Revision >= old.walk.Revision { + h.walks[w.ID] = seenWalk{walk: w, at: h.Now()} + } + if w.IsBatch() { + return // a batch is shown, never linked, adopted or let go (ADR 0276) + } + links := h.deliveriesWalking(w) + if len(links) == 0 { + if h.accounted(w) { + return // it answers only deliveries that ended or follow a newer walk + } + if _, seen := h.unmatched[w.ID]; !seen { + h.unmatched[w.ID] = unmatched{walk: w, first: h.Now()} + } else { + u := h.unmatched[w.ID] + if w.Revision >= u.walk.Revision { + u.walk = w + h.unmatched[w.ID] = u + } + } + return + } + delete(h.unmatched, w.ID) + moved := map[string]bool{} + for _, a := range links { + d := a.d + if d.Walk != nil && d.Walk.ID == w.ID && w.Revision < d.Walk.Revision { + continue // an older record arriving late + } + h.linkWalk(d, w, a.carried) + h.settleAndKeep(d) + moved[d.ID] = true + } + // What its sends carried of other deliveries' builds. + for _, m := range w.Modules { + if m != nil && m.Gate != nil { + h.creditCarried(m.Gate, w.ID, moved) + } + } + // And how long each delivery it answers took, once its end is known (novox/hq ADR 0282): measured, never + // acted on. + h.timeWalk(w) +} + +// linkWalk records a walk on its delivery: what it is at, the commit it carries for it, and the machine steps +// its record says — each moved only as the step table allows. A delivery that follows a new walk takes its +// steps as that walk's, a new journey; and one held because no walk answered it is lifted. +func (h *Holder) linkWalk(d *Delivery, w Walk, carried string) { + switching := d.Walk != nil && d.Walk.ID != w.ID + if switching { + h.Logf("[mesh-delivery] %s follows the walk %s now (it followed %s, %s)", d.ID, w.ID, d.Walk.ID, d.Walk.State) + d.LetGo, d.LetGoWhy = time.Time{}, "" // a new walk waits for a word of its own + } + d.Walk = w.Seen() + // Stopped without some of its merges (novox/hq ADR 0299): this delivery's merge was left out, maybe for + // containing a stopped one, or is walked again by the batch the record names. + if w.Delivery != nil && (w.Delivery.WalkedAgainBy != "" || len(w.Delivery.LeftOut) > 0) { + if o, out := w.leftOut(d.Repository, d.MergedAs); out { + d.Walk.Contains, d.Walk.Shares = o.Contains, o.Shares + } else { + d.Walk.WalkedAgainBy = w.Delivery.WalkedAgainBy + } + } + if carried != "" { + d.AnsweredBy = carried + } + d.Carrier = "" + if d.carriedLater() { + d.Carrier = fmt.Sprintf("merged as %s, carried by the walk %s", shortOf(d.AnsweredBy, 12), w.ID) + for _, o := range h.deliveries { + if o.ID != d.ID && strings.EqualFold(o.Repository, d.Repository) && sameCommit(o.MergedAs, d.AnsweredBy) { + d.Carrier = fmt.Sprintf("%s (merged as %s into %s, walk %s)", o.ID, shortOf(d.AnsweredBy, 8), orNone(d.Base), w.ID) + } + } + } + if d.State == Held && d.Unwalked && w.Open() { + d.HeldWhy = "" + d.Lifted = fmt.Sprintf("the walk %s, which answers its merge, opened", w.ID) + } + d.Unwalked = false // a walk answers it + for _, s := range StepsOf(w) { + if switching { + h.restep(d, s) + continue + } + h.step(d, s) + } +} + +// restep puts a step of a walk a delivery newly follows, whatever the walk before said of that machine. +func (h *Holder) restep(d *Delivery, s Step) { + for i, old := range d.Steps { + if old.Machine == s.Machine && old.Module == s.Module { + d.Steps[i] = s + return + } + } + d.Steps = append(d.Steps, s) +} + +// step moves one machine's step, or refuses a move the step table does not hold; it says whether it moved. +func (h *Holder) step(d *Delivery, s Step) bool { + for i, old := range d.Steps { + if old.Machine == s.Machine && old.Module == s.Module { + if old.State == s.State { + return false + } + if !StepMay(old.State, s.State) { + h.refuse(fmt.Errorf("%s: %s on %s from %s to %s is refused — the step table holds no such move", + d.ID, s.Module, s.Machine, old.State, s.State)) + return false + } + d.Steps[i] = s + return true + } + } + d.Steps = append(d.Steps, s) + return true +} + +// creditBacklog keeps a backlog walk, and credits what its gate carried. +func (h *Holder) creditBacklog(w Walk) { + replaced := false + for i, b := range h.backlog { + if b.ID == w.ID { + h.backlog[i], replaced = w, true + } + } + if !replaced { + h.backlog = append(h.backlog, w) + if len(h.backlog) > 5 { + h.backlog = h.backlog[len(h.backlog)-5:] + } + } + if w.Release.Gate != nil { + h.creditCarried(w.Release.Gate, w.ID, nil) + } + if w.State == walkDone { + h.creditReleased(w) + } +} + +// creditReleased passes the steps a finished backlog walk carried: its last record keeps no gate, only the +// machines it is done on, and a release ends done only when its gate passed on each of them (novox/hq issue +// 351). A step it failed on, and a step another walk carried, are left as they are. +func (h *Holder) creditReleased(w Walk) { + why := carriedBy + w.ID + for _, d := range h.deliveries { + moved := false + for _, s := range d.Steps { + if s.Why != why || !containsString(w.Release.Done, s.Machine) || s.State != StepSent && s.State != StepJudging { + continue + } + s.State, s.Since = StepPassed, h.Now() + if h.step(d, s) { + moved = true + } + } + if moved { + h.keep(d) + } + } +} + +// carriedBy begins the why of a step another walk's send carried, followed by that walk's id. +const carriedBy = "carried by " + +// askWalk is the ask of one walk's record by its id, at most once an hour; what it answers is taken as said. +// which says, for the log, why it is asked: what the walk is to the delivery that asks. +func (h *Holder) askWalk(id, which string, now time.Time) func() { + if asked, was := h.walkAsked[id]; was && now.Sub(asked) < time.Hour { + h.lookAt(asked.Add(time.Hour), "a walk's record asked of the controller by its id") + return nil + } + h.walkAsked[id] = now + h.lookAt(now.Add(time.Hour), "a walk's record asked of the controller by its id") + return func() { + _, walks, err := h.Controller.Walks(id) + h.mu.Lock() + defer h.mu.Unlock() + if err != nil { + h.Logf("[mesh-delivery] the walk %s, %s, cannot be read yet: %v", id, which, err) + h.walkAsked[id] = h.Now().Add(time.Minute - time.Hour) // asked again a minute on + return + } + for _, w := range walks { + if w.ID == id { + h.walkMoved(w) + } + } + } +} + +// creditCarried gives the delivery that published each carried build its machine step. +func (h *Holder) creditCarried(g *WalkGate, walk string, except map[string]bool) { + state := StepJudging + switch { + case g.Rollback == "rolled-back": + state = StepRolledBack + case g.Verdict == "failed": + state = StepFailed + case g.Verdict == "passed": + state = StepPassed + } + for _, c := range g.Carried { + for _, d := range h.deliveries { + if except[d.ID] || d.MergedAs == "" || !sameCommit(d.MergedAs, c.To) { + continue + } + if h.step(d, Step{Machine: c.Node, Module: c.Module, Build: c.Build, State: state, Since: h.Now(), + Why: carriedBy + walk}) { + h.keep(d) + } + } + } +} + +// --- the controller's walks, read back -------------------------------------------------------------------- + +// Reconcile asks the controller for the walks it keeps and takes each as if said: what was missed while +// this holder was down, or said by a command that says nothing, is found by comparison. +func (h *Holder) Reconcile() error { + held, walks, err := h.Controller.Walks("") + h.mu.Lock() + defer h.mu.Unlock() + if err != nil { + h.unreached = err.Error() + return err + } + h.unreached, h.held, h.reconciled = "", held, h.Now() + for _, w := range walks { + h.walkMoved(w) + } + return nil +} + +// --- the look: groups, turns, adoption -------------------------------------------------------------------- + +// look takes every step the facts allow that needs another's word: a group's composed check, a +// delivery's turn, a walk with no delivery made one. The asking happens outside the lock. +func (h *Holder) look() { + for _, ask := range h.plan() { + ask() + } + for _, ask := range h.leaveStopped() { + ask() + } + h.hearAsks() + h.mu.Lock() + asks := h.readyAsks(h.Now()) + h.mu.Unlock() + for _, ask := range asks { + ask() + } +} + +// plan works out, under the lock, what to ask; each ask applies its own answer. +func (h *Holder) plan() []func() { + h.mu.Lock() + defer h.mu.Unlock() + now := h.Now() + var asks []func() + + h.forgetWalks(now) + + // Walks no merge was heard for: made deliveries of their own once their merge had time to be heard. + for id, u := range h.unmatched { + if links := h.deliveriesWalking(u.walk); len(links) > 0 { + for _, a := range links { + h.linkWalk(a.d, u.walk, a.carried) + h.settleAndKeep(a.d) + } + delete(h.unmatched, id) + continue + } + if h.accounted(u.walk) { + delete(h.unmatched, id) + continue + } + if !u.walk.Open() { + if now.Sub(u.first) > 10*time.Minute { + delete(h.unmatched, id) + } else { + h.lookAt(u.first.Add(10*time.Minute), "a walk no merge was heard for, ended: forgotten after ten minutes") + } + continue + } + if u.walk.Created.After(h.started) && now.Sub(u.first) < matchGrace { + h.lookAt(u.first.Add(matchGrace), "a walk no merge was heard for: made a delivery after the grace") + continue + } + h.adopt(u.walk) + delete(h.unmatched, id) + } + + // Plans not known for a delivery that has its diffset: asked at each look. + for _, d := range h.deliveries { + if d.Plan == nil && !d.State.Final() && len(d.Paths) > 0 && (d.State == Ready || d.State == Held || d.MergedAs != "") { + d := d + id, repo, base, head, paths, dirs, removed := d.ID, d.Repository, d.Base, d.Commit, d.Paths, d.ModuleDirs, d.Removed + asks = append(asks, func() { + plan, err := h.Controller.Plan(repo, base, head, paths, dirs, removed) + h.mu.Lock() + defer h.mu.Unlock() + if err != nil { + h.Logf("[mesh-delivery] %s's plan cannot be asked yet: %v", id, err) + h.again("a delivery's plan that could not be asked") + return + } + if d := h.deliveries[id]; d != nil && d.Plan == nil { + d.Plan = plan + h.settleAndKeep(d) + } + }) + } + } + + // Proposed, with no verdict and no check asked: its check is asked at once (issue 1115), or the verdict that + // stands on its head at the forge is adopted. + if a := h.askChecks(now); a != nil { + asks = append(asks, a) + } + + // Merged with no walk yet: tied to the newest walk heard that answers it — by name, by its own commit, or by + // the forge's word that the walk's commit contains it, asked here (novox/hq issue 362). + for _, d := range h.deliveries { + if h.linkAnswering(d) { + h.settleAndKeep(d) + continue + } + asks = append(asks, h.containsAsks(d, now)...) + } + + // Following a walk that was taken over: the walk that took it over, read as soon as it is heard of, asked for + // by its id when it is not. + for _, d := range h.deliveries { + if d.State.Final() || d.Walk == nil || d.Walk.TakenOverBy == "" { + continue + } + if s, known := h.walks[d.Walk.TakenOverBy]; known { + if c, ok := walkAnswers(s.walk, d); ok { + h.linkWalk(d, s.walk, c) + } else { + // The walk that took its own over does not answer it: superseded, as before ADR 0276. + h.Logf("[mesh-delivery] %s: the walk %s, which took %s over, does not answer its merge", d.ID, + d.Walk.TakenOverBy, d.Walk.ID) + d.Walk.TakenOverBy = "" + } + h.settleAndKeep(d) + continue + } + if a := h.askWalk(d.Walk.TakenOverBy, "which took a delivery's walk over", now); a != nil { + asks = append(asks, a) + } + } + + // Held, or merged with no walk opened for it: ended by what is already true of it, never asked of a person + // (novox/hq issue 319). A plan that moves nothing is delivered; one whose every module a later merge into the + // same trunk delivered is covered. Read again at each look, so a delivery read back after a restart, or held + // before this was known, ends by itself. + // A published delivery whose walk ended done before its word is read the same way (novox/hq issue 474): what + // that walk did not build, a later delivered merge may have; and when the cover test says none did, past the + // grace, it is stopped saying so — never left waiting for a start that cannot come. + // A delivery that failed or was stopped is read for a later finished walk of its repository that delivered its + // change after all: a walk that failed in an outage and was resumed, or the walk of a later merge that contains it + // (novox/hq issue 747). Read at each look, from what the walks and the kept deliveries say, so one stuck before this + // was built clears by itself after a restart. + h.coverAsked = map[string]bool{} + if h.coverAskedSince == nil { + h.coverAskedSince = map[string]time.Time{} + } + pending := map[string]bool{} + for _, d := range h.deliveries { + if d.State != Failed && d.State != Stopped || d.coveredStop() { + continue + } + by, ask := h.walkCover(d) + if len(ask) > 0 && by == "" { + pending[d.ID] = true + since, asked := h.coverAskedSince[d.ID] + if !asked { + since = now + h.coverAskedSince[d.ID] = now + } + // Held back only within the bound: a forge that keeps erroring never holds a walk for good. + h.coverAsked[d.ID] = now.Sub(since) < coverAskBound + if h.coverAsked[d.ID] { + h.lookAt(since.Add(coverAskBound), "a walk held back while the forge is asked what a later walk delivered") + } + } + if by != "" { + d.DeliveredBy = by + h.tellCovered(d, now) + continue + } + for _, in := range ask { + if a := h.askContains(d, in, now); a != nil { + asks = append(asks, a) + } + } + } + + for id := range h.coverAskedSince { + if !pending[id] { + delete(h.coverAskedSince, id) + } + } + + for _, d := range h.deliveries { + ended := d.endedUnstarted() + if d.coveredStop() && !d.CoverTold { + h.tellCovered(d, now) // covered before this holder told it, or before a restart (novox/hq issue 480) + continue + } + if strandedUncovered(d) { + // Stopped and final, but still read for a later merge that delivered its modules: until one did, + // `stalled` keeps listing it (review of novox/hq issue 474). + var ask []string + if d.CoveredBy, ask = h.coverOf(d); d.CoveredBy != "" { + h.tellCovered(d, now) + } + for _, in := range ask { + if a := h.askContains(d, in, now); a != nil { + asks = append(asks, a) + } + } + continue + } + if d.State != Held && !h.unwalkedPast(d, now) && !ended { + continue + } + var ask []string + if d.CoveredBy == "" { + d.CoveredBy, ask = h.coverOf(d) + for _, in := range ask { + if a := h.askContains(d, in, now); a != nil { + asks = append(asks, a) + } + } + } + if ended { + if a := h.strand(d, ask, now); a != nil { + asks = append(asks, a) + } + } + if ts := settle(d, Facts{Now: now}); len(ts) > 0 { + h.moved(d, ts) + h.closeGroupOf(d) + } + } + + // A step a walk carried and left judging: that walk's record is asked for by its id. A walk that ended + // while this holder was down may be older than the ended walks the controller lists back at its start, + // and its end is what passes the step (novox/hq issue 351). + for _, d := range h.deliveries { + for _, s := range d.Steps { + if s.State != StepSent && s.State != StepJudging || !strings.HasPrefix(s.Why, carriedBy) { + continue + } + if a := h.askWalk(strings.TrimPrefix(s.Why, carriedBy), "which carried a step still judging", now); a != nil { + asks = append(asks, a) + } + } + } + + // Merged, and no walk opened for it after the walks were read again: never silent — held, saying so. + for _, d := range h.deliveries { + if h.unwalkedPast(d, now) { + d.HeldWhy, d.Unwalked = h.unwalkedWhy(d), true + h.settleAndKeep(d) + } + } + + // Groups: membership, composed check, and their word. + h.turning = map[string]*turnWant{} + grouped := map[string]bool{} + for _, g := range h.groups { + members := h.membersOf(g) + for _, m := range members { + grouped[m.ID] = true + } + asks = append(asks, h.planGroup(g, members, now)...) + } + + // A lone delivery's turn: published, its walk waiting, nothing before it. + for _, d := range h.deliveries { + if grouped[d.ID] || d.Group != "" && h.groups[d.Group] != nil { + continue + } + h.turn(d, "its turn: a delivery in no group") + } + // A walk that answers the merge of a delivery whose stop lapsed: that delivery's turn (novox/hq issue 753). + h.lapsedTurns() + // The word, once per walk, when every live delivery it carries is at its turn. + asks = append(asks, h.words()...) + + // Waiting walks no live delivery holds: listed by `stalled` (issue 362). + h.seeOrphans(now) + + // Final deliveries past their keeping, once nothing is owed for them. + for id, d := range h.deliveries { + if d.State.Final() && len(d.Owed) == 0 && now.Sub(d.Since) > keptFinal && !strandedUncovered(d) { + if err := h.Store.DeleteDelivery(id); err == nil { + delete(h.deliveries, id) + delete(h.written, id) + } + } + } + return asks +} + +// strandedAfter is how long after its walk ended unstarted a delivery that walk did not deliver waits for a later +// merge to cover it before it is stopped (novox/hq issue 474): within the bound of a published delivery, so it is +// stopped, saying why, before `stalled` would list it. +const strandedAfter = 20 * time.Minute + +// strand says why a delivery whose walk ended done before its word is stopped, once that is so: the walk did not +// build every module it moves, the cover test asked the forge everything it had to and found no later delivered +// merge that carried them, and the walk ended strandedAfter ago. A walk record kept before it named its modules is +// asked for again (askWalk) rather than read as moving nothing. +func (h *Holder) strand(d *Delivery, pending []string, now time.Time) func() { + moved, known := d.walkMovedItsModules() + switch { + case !known && d.Plan != nil: + return h.askWalk(d.Walk.ID, "whose record, kept before it named the modules it built, is read again", now) + case !known, moved, d.CoveredBy != "", len(pending) > 0, d.Stranded != "", h.laterOnItsWay(d) != "": + return nil + } + ended := d.Walk.Updated + if ended.IsZero() { + ended = d.Since + } + if now.Sub(ended) < strandedAfter { + h.lookAt(ended.Add(strandedAfter), "a merge its walk did not build: stopped after the grace for a later merge to cover it") + return nil + } + note := "" + if d.Walk.Note != "" { + note = " (" + d.Walk.Note + ")" + } + d.Stranded = fmt.Sprintf("a merged change never reached the machines: its walk %s ended done before its word "+ + "without building %s%s, and no later merge into %s that contains it delivered them (novox/hq issue 474)", d.Walk.ID, + strings.Join(d.notMovedBy(), ", "), note, orNone(d.Base)) + return nil +} + +// laterOnItsWay is a later merge of the same repository into the same trunk, not yet ended, whose plan moves a +// module this delivery's walk left unbuilt — its walk queued, waiting or running — or empty when there is none +// (review of novox/hq issue 474). A stop is final and nothing moves it to covered, so it waits for that merge: +// once delivered, the cover test reads it; if it ends otherwise, the stop follows. +func (h *Holder) laterOnItsWay(d *Delivery) string { + missing := d.notMovedBy() + for _, o := range h.deliveries { + if o.ID == d.ID || o.State.Final() || o.Plan == nil || o.MergedAs == "" || sameCommit(o.MergedAs, d.MergedAs) || + !strings.EqualFold(o.Repository, d.Repository) || !strings.EqualFold(o.Base, d.Base) || + !o.MergedAt.After(d.MergedAt) { + continue + } + for _, m := range missing { + if containsString(o.Plan.Moved, m) { + return o.ID + } + } + } + return "" +} + +// tellCovered tells the forge that a stop is covered since, once: its status says covered and by which merge, its view +// says so, and its note keeps the line on its head and its merge commit (novox/hq issue 480). No transition: the stop +// stays final, and the bus's transition event is not said again. +func (h *Holder) tellCovered(d *Delivery, now time.Time) { + d.CoverTold = true + line := oneLine(fmt.Sprintf("%s delivery %s: %s, %s", now.UTC().Format(time.RFC3339), d.ID, d.State, d.coveredStopSaid())) + d.Owed = append(d.Owed, Effect{Kind: EffectNote, Commit: d.Commit, Line: line, Since: now}) + if d.MergedAs != "" && !sameCommit(d.MergedAs, d.Commit) { + d.Owed = append(d.Owed, Effect{Kind: EffectNote, Commit: d.MergedAs, Line: line, Since: now}) + } + if d.shownOnItsPullRequest() { + d.owe(Effect{Kind: EffectView, Since: now}) + state, desc := statusOf(d) + d.owe(Effect{Kind: EffectStatus, Commit: d.Commit, Context: "mesh/delivery", State: state, Line: desc, Since: now}) + } + h.Logf("[mesh-delivery] %s: %s, %s", d.ID, d.State, d.coveredStopSaid()) + h.keep(d) +} + +// strandedUncovered is whether a delivery stopped because its ended walk delivered nothing is still uncovered: a +// merged change that never reached the machines, which `stalled` keeps listing until a later delivered merge +// carries its modules (review of novox/hq issue 474). +func strandedUncovered(d *Delivery) bool { + return d.State == Stopped && d.Stranded != "" && d.CoveredBy == "" +} + +// walkCover says which later finished walk delivered the change of a delivery that failed or was stopped, or +// nothing, and which walk commits the forge must still be asked about (novox/hq issue 747). A walk covers it when it +// is of its repository and trunk, done, made after its merge, its commit containing its merge — the merge itself, or +// a later one the forge says contains it — and it built every module the delivery's plan moves, each it did not +// build being one the delivery's own walk said unchanged (issue 611). This is the containment "one walk answers every +// merge it contains" uses (issue 362, ADR 0276), read after the fact: a failed walk resumed to done answers its own +// merge, and a later merge's walk answers every merge it contains. A walk that only said a module unchanged for its +// own merges says nothing of this one's, so it covers no module that way. Walks are read from the controller's records +// this owner holds and from every delivery's kept walk, so a walk long ended still counts after a restart. +func (h *Holder) walkCover(d *Delivery) (string, []string) { + if d.State != Failed && d.State != Stopped || d.MergedAs == "" || d.Plan == nil || len(d.Plan.Moved) == 0 { + return "", nil + } + var ask []string + for _, c := range h.doneWalksOf(d.Repository) { + w := c.walk + if !sameBranch(w.Branch, d.Base) || !w.Created.IsZero() && w.Created.Before(d.MergedAt.Add(-time.Minute)) { + continue + } + var missing []string + for _, m := range d.Plan.Moved { + unchanged := false + if d.Walk != nil { + _, unchanged = d.Walk.Unchanged[m] + } + if !containsString(w.Moved, m) && !unchanged { + missing = append(missing, m) + } + } + if len(missing) > 0 { + continue + } + if !sameCommit(c.commit, d.MergedAs) { + yes, known := d.ContainedIn[c.commit] + if !known && !containsString(ask, c.commit) { + ask = append(ask, c.commit) + } + if !yes { + continue + } + } + return fmt.Sprintf("%s (%s at %s into %s, done, contains its merge %s)", w.ID, d.Repository, shortOf(c.commit, 8), + orNone(d.Base), shortOf(d.MergedAs, 8)), nil + } + return "", ask +} + +// doneWalk is a finished walk and the commit of its repository it carried. +type doneWalk struct { + walk *WalkSeen + commit string +} + +// doneWalksOf are the finished walks of a repository this owner knows, oldest first: the controller's records it +// holds, and each delivery's kept walk with the commit of its repository that walk carried for it. +func (h *Holder) doneWalksOf(repository string) []doneWalk { + byID := map[string]doneWalk{} + for id, s := range h.walks { + w := s.walk + if w.Release != nil || w.IsBatch() || w.State != walkDone || !strings.EqualFold(w.Repository, repository) { + continue + } + byID[id] = doneWalk{walk: w.Seen(), commit: w.Commit} + } + for _, o := range h.deliveries { + if o.Walk == nil || o.Walk.State != walkDone || !strings.EqualFold(o.Repository, repository) { + continue + } + if _, known := byID[o.Walk.ID]; known { + continue + } + commit := o.AnsweredBy + if commit == "" && o.MergedAs != "" && sameCommit(o.Walk.Commit, o.MergedAs) { + commit = o.MergedAs + } + if commit == "" { + continue + } + byID[o.Walk.ID] = doneWalk{walk: o.Walk, commit: commit} + } + out := make([]doneWalk, 0, len(byID)) + for _, c := range byID { + out = append(out, c) + } + sort.Slice(out, func(i, j int) bool { + if !out[i].walk.Created.Equal(out[j].walk.Created) { + return out[i].walk.Created.Before(out[j].walk.Created) + } + return out[i].walk.ID < out[j].walk.ID + }) + return out +} + +// stopLapsed is whether a member its group stopped for another is stopped for nothing any more (see stopLapsed). +func (h *Holder) stopLapsed(d *Delivery) bool { + return stopLapsed(d, func(id string) *Delivery { return h.deliveries[id] }) +} + +// unwalkedPast is whether a merged delivery has gone without a walk past the bound one opens within, read +// again after it: what holds it for a person, unless something already ends it. +func (h *Holder) unwalkedPast(d *Delivery, now time.Time) bool { + if _, batched := h.batchOf(d); batched { + return false // its batch is assembling or queued: a walk is not owed yet (ADR 0276) + } + if d.State != Ready || d.MergedAs == "" || d.Walk != nil || d.MovesNothing() || d.HeldWhy != "" { + return false + } + bound := d.MergedAt.Add(unwalked) + if now.Sub(d.MergedAt) <= unwalked { + h.lookAt(bound, "a merge no walk opened for within ten minutes") + return false + } + if !h.reconciled.After(bound) { + // The walks were not read since the bound: read once now (React), the only read besides the one at the start. + h.rereadWalks = true + return false + } + return true +} + +// coverOf says which later merges cover a merged delivery, or nothing, and which later merge commits the forge +// must still be asked about (novox/hq ADR 0270). A module is covered by a later delivery that carries it (see +// carries) and whose merge commit the forge said contains this one's merge. Merge order is the cheap filter; +// ancestry decides. The forge is asked only once every module has a candidate, once per pair, and its answer +// is kept on the delivery: an answer not known yet is "not yet covered". +func (h *Holder) coverOf(d *Delivery) (string, []string) { + if d.MergedAs == "" || d.MergedAt.IsZero() || d.Base == "" || d.Plan == nil || len(d.Plan.Moved) == 0 { + return "", nil + } + candidates := map[string][]*Delivery{} + for _, module := range d.Plan.Moved { + for _, c := range h.deliveries { + if carries(c, d, module) { + candidates[module] = append(candidates[module], c) + } + } + if len(candidates[module]) == 0 { + return "", nil // nothing later carried it: nothing to ask + } + } + carried := map[string][]string{} + var by []*Delivery + var ask []string + for _, module := range d.Plan.Moved { + var first *Delivery + for _, c := range candidates[module] { + yes, known := d.ContainedIn[c.MergedAs] + if !known && !containsString(ask, c.MergedAs) { + ask = append(ask, c.MergedAs) + } + if yes && (first == nil || c.MergedAt.Before(first.MergedAt) || c.MergedAt.Equal(first.MergedAt) && c.ID < first.ID) { + first = c + } + } + if first == nil { + return "", ask + } + if carried[first.ID] == nil { + by = append(by, first) + } + carried[first.ID] = append(carried[first.ID], module) + } + sort.Slice(by, func(i, j int) bool { return by[i].MergedAt.Before(by[j].MergedAt) }) + var parts []string + for _, c := range by { + mods := carried[c.ID] + sort.Strings(mods) + parts = append(parts, fmt.Sprintf("%s (merged as %s into %s, walk %s): %s", c.ID, shortOf(c.MergedAs, 8), d.Base, + c.Walk.ID, strings.Join(mods, ", "))) + } + return strings.Join(parts, "; "), nil +} + +// carries is whether c delivered a module after d: of the same repository, merged later into the same trunk, +// delivered on its own passing check — a merge held without one is covered only by a later merge that passed +// its own — by its own walk, of its merge commit on that trunk, which sent the module to every machine running +// it. Its machine steps say so in one of two ways: the walk's own record, which sends to every machine at once +// or to the rest after its first machines passed; or, for a build the controller held back and a person +// released, one step per machine credited by the backlog walk that carried it — passed on every machine d's +// plan sends the module to (novox/hq issue 351). Whether c's merge commit contains d's is the forge's to say +// (coverOf). +func carries(c, d *Delivery, module string) bool { + if c.ID == d.ID || c.State != Delivered || !c.Check.Passes() || !strings.EqualFold(c.Repository, d.Repository) || + !strings.EqualFold(c.Base, d.Base) || c.MergedAs == "" || sameCommit(c.MergedAs, d.MergedAs) || + !c.MergedAt.After(d.MergedAt) || c.Walk == nil || c.Walk.State != walkDone || !sameCommit(c.Walk.Commit, c.MergedAs) || + c.Walk.Branch != "" && !strings.EqualFold(c.Walk.Branch, d.Base) { + return false + } + everywhere := false + passed := map[string]bool{} + for _, s := range c.Steps { + if s.Module != module { + continue + } + switch { + case s.State == StepFailed || s.State == StepRolledBack: + return false + case s.State == StepSent && (s.Machine == stepEveryMachine || s.Machine == stepTheRest): + everywhere = true + case s.State == StepPassed: + passed[s.Machine] = true + } + } + if everywhere { + return true + } + machines := d.Plan.receiving(module) + if len(machines) == 0 { + return false + } + for _, m := range machines { + if !passed[m] { + return false + } + } + return true +} + +// askContains is the ask of the forge whether a later merge commit contains a delivery's merge: at most once a +// minute while the forge cannot say, and never again once it said — its answer is kept on the delivery. +func (h *Holder) askContains(d *Delivery, in string, now time.Time) func() { + key := d.ID + " in " + in + if tried, was := h.askTried[key]; was && now.Sub(tried) < time.Minute { + h.lookAt(tried.Add(time.Minute), "the forge asked whether a later merge contains this one") + return nil + } + h.askTried[key] = now + h.lookAt(now.Add(time.Minute), "the forge asked whether a later merge contains this one") + id, owner, repo, merged := d.ID, d.Owner(), d.Repo(), d.MergedAs + return func() { + yes, err := h.Forge.Contains(owner, repo, merged, in) + h.mu.Lock() + defer h.mu.Unlock() + if err != nil { + h.Logf("[mesh-delivery] %s: whether %s contains its merge cannot be read yet (%v): not yet covered", id, + shortOf(in, 12), err) + return + } + delete(h.askTried, key) + d := h.deliveries[id] + if d == nil { + return + } + if d.ContainedIn == nil { + d.ContainedIn = map[string]bool{} + } + d.ContainedIn[in] = yes + h.keep(d) + } +} + +// heldForGroup is whether a member is held only because its group's composed check had not passed. +func heldForGroup(d *Delivery, g *Group) bool { + return d.State == Held && strings.HasPrefix(d.HeldWhy, fmt.Sprintf("its group %s's composed check", g.ID)) +} + +// standingVerdict is the passing verdict an open pull request's head holds at the forge, when every merge check and +// every context its base requires stands as success or warning; nil otherwise, a missing context included (issue 1115). +func standingVerdict(p OpenPull, now time.Time) *Verdict { + if p.Checks == nil { + return nil + } + contexts := slices.Clone(mergeChecks) + for _, c := range p.Required { + if !slices.Contains(contexts, c) { + contexts = append(contexts, c) + } + } + for _, c := range contexts { + if st := p.Checks[c].State; st != "success" && st != "warning" { + return nil + } + } + return &Verdict{At: now, Gate: verdictOfState(p.Checks["mesh/merge-gate"].State), + Summary: "the head's mesh/merge-gate, as the forge holds it", Repo: verdictOfState(p.Checks["mesh/repo-check"].State)} +} + +// askedCheck is whether a proposed delivery's own check stands asked: a kept ask with no id is none. +func askedCheck(d *Delivery) bool { return d.CheckAsk != nil && d.CheckAsk.ID != "" } + +// askChecks are the asks of proposed deliveries with no verdict, at each look, at once (issue 1115): the forge's open +// pull requests are read once, a passing verdict standing on a head is adopted (the delivery goes on to ready with no +// new check), and a head with no check asked is asked its check. Under the lock; the asks run outside it. +func (h *Holder) askChecks(now time.Time) func() { + type want struct { + id, owner, repo, head string + number int + member Member + ask bool + why string + } + var wants []want + for _, d := range h.deliveries { + if d.State != Proposed || d.Check != nil || d.MergedAs != "" || d.Number == 0 { + continue + } + w := want{id: d.ID, head: d.Commit, number: d.Number, member: MemberOf(d)} + w.owner, w.repo, _ = strings.Cut(d.Repository, "/") + if !askedCheck(d) { + if tried, was := h.askTried[d.ID]; was && now.Sub(tried) < time.Minute { + h.lookAt(tried.Add(time.Minute), "a proposed delivery's check that could not be asked") + } else { + w.ask = true + h.askTried[d.ID] = now + h.lookAt(now.Add(time.Minute), "a proposed delivery's check that could not be asked") + w.why = fmt.Sprintf("proposed for %s with no verdict and no check asked", now.Sub(d.Since).Round(time.Second)) + } + } + wants = append(wants, w) + } + if len(wants) == 0 { + return nil + } + return func() { + open, err := h.Forge.OpenPulls() + for _, w := range wants { + var standing *Verdict + if err == nil { + for _, p := range open { + if p.Number == w.number && strings.EqualFold(p.Owner, w.owner) && strings.EqualFold(p.Repo, w.repo) && + sameCommit(p.HeadSHA, w.head) { + standing = standingVerdict(p, h.Now()) + } + } + } + if standing != nil { + h.mu.Lock() + if d := h.deliveries[w.id]; d != nil && d.State == Proposed && d.Check == nil { + delete(h.askTried, w.id) + d.Check = standing + h.Logf("[mesh-delivery] %s: a passing verdict already stands on its head at the forge; adopted, no check asked", w.id) + h.settleAndKeep(d) + } + h.mu.Unlock() + continue + } + if w.ask { + h.askOwnCheck(w.id, w.member, w.why) + } + } + } +} + +// askOwnCheck asks the controller a proposed delivery's own check and keeps the ask. +func (h *Holder) askOwnCheck(id string, m Member, why string) { + asked, err := h.Controller.Check("", []Member{m}) + h.mu.Lock() + defer h.mu.Unlock() + if err != nil { + h.Logf("[mesh-delivery] %s's check cannot be asked yet: %v", id, err) + return + } + if asked == "" { + h.Logf("[mesh-delivery] %s's check was asked and the controller's answer held no id: taken as refused, asked again in a minute at most", id) + return + } + delete(h.askTried, id) + d := h.deliveries[id] + if d == nil || d.State != Proposed || d.Check != nil || askedCheck(d) { + return + } + d.CheckAsk = &CheckAsk{ID: asked, At: h.Now(), By: byOwner, Why: why} + h.Logf("[mesh-delivery] %s: its check asked of the controller as %s — %s", id, asked, why) + d.Owed = append(d.Owed, Effect{Kind: EffectNote, Commit: d.Commit, Since: h.Now(), + Line: oneLine(fmt.Sprintf("%s delivery %s: its check asked by %s as %s: %s", h.Now().UTC().Format(time.RFC3339), + id, byOwner, asked, why))}) + h.keep(d) +} + +// adopt makes a delivery of a walk no pull request is known for: running already — at the switch, or on the +// controller's own path — it is delivering; waiting for a word, it waits for a person. +func (h *Holder) adopt(w Walk) { + now := h.Now() + d := &Delivery{ID: IDOf(w.Repository, w.Commit), Repository: strings.ToLower(w.Repository), Commit: w.Commit, + Base: w.Branch, MergedAs: w.Commit, Created: now} + if old := h.deliveries[d.ID]; old != nil { + if old.State.Final() { + // Never revived: an ended delivery is not the walk's; the walk, held by nothing, is listed by + // `stalled` (issue 362). + h.Logf("[mesh-delivery] the walk %s is at %s, whose delivery has ended (%s): not adopted by it", w.ID, + shortOf(w.Commit, 12), old.State) + return + } + h.linkWalk(old, w, w.Commit) + h.settleAndKeep(old) + return + } + h.linkWalk(d, w, w.Commit) + var t Transition + var err error + if d.Walk.Waits { + d.HeldWhy = "a commit on the trunk with no pull request this owner heard of — merged before it held the " + + "seat, or pushed to the trunk — waits for a person's release" + t, err = Apply(d, EvAppeared, false, Facts{Now: now, Why: d.HeldWhy}) + } else { + t, err = Apply(d, EvAdopted, false, Facts{Now: now, Why: "a walk already running when this owner first heard " + + "of it (" + w.ID + "), adopted as its delivery"}) + } + if err != nil { + h.refuse(err) + return + } + h.moved(d, []Transition{t}) + h.settleAndKeep(d) +} + +// turn says a published delivery's waiting walk may start as far as that delivery goes; nil when it is not its +// turn. The word itself is asked once per walk by words, when every live delivery the walk carries is at its turn +// (ADR 0276 decision 5). What it answers is only whether this delivery's turn came. +func (h *Holder) turn(d *Delivery, why string) func() { + if d.Walk == nil || !d.Walk.Waits || !d.LetGo.IsZero() { + return nil + } + if d.State != Published && d.State != Delivering { + return nil + } + if d.Walk.State != walkBuilding && d.Walk.State != walkRolling { + return nil // a walk superseded, done or failed is no walk to start + } + if h.turning == nil { + h.turning = map[string]*turnWant{} + } + want := h.turning[d.Walk.ID] + if want == nil { + want = &turnWant{} + h.turning[d.Walk.ID] = want + } + if !containsString(want.ids, d.ID) { + want.ids = append(want.ids, d.ID) + want.whys = append(want.whys, d.ID+": "+why) + } + return func() {} +} + +// membersOf are a group's current members, by id order. +func (h *Holder) membersOf(g *Group) []*Delivery { + var out []*Delivery + for _, id := range g.Members { + if d := h.deliveries[id]; d != nil { + out = append(out, d) + } + } + return out +} + +// planGroup is what a group needs asked, and its turns. +func (h *Holder) planGroup(g *Group, members []*Delivery, now time.Time) []func() { + var asks []func() + if len(members) == 0 { + return nil + } + if len(members) == 1 && !g.Closed { + // A group of one is no group: its member is a lone delivery again. + members[0].Group = "" + h.keep(members[0]) + g.Members = nil + h.keepGroup(g) + return nil + } + commits := commitsOf(members) + // The composed check, once every member is ready. Asked again for new heads, even once the + // group is closed: a member rebased or pushed is a new head, and the group waits for its composed check rather + // than being held for a person (novox/hq ADR 0323 §6, issue 406). One out of date is asked again once a member + // was judged again after it went out of date: a recheck, or the merge tool's ask. + allReady := true + for _, d := range members { + // In a closed group a member on the trunk counts — merged and on its way, or already delivered (reached). + if d.State != Ready && !(g.Closed && d.MergedAs != "" && (!d.State.Final() || d.State.Reached())) { + allReady = false + } + } + again := g.Check != nil && (!sameMembers(g.Check.Commits, commits) || g.Check.OutOfDate != "" && rejudgedSince(members, g.Check.OutOfDateAt)) + // A closed group's member whose merge the tool refused, after the composed check answered, is judged again at once + // (novox/hq ADR 0418 §4, issue 1082): the refusal is a sibling's merge having moved what the verdict stands on, and + // nothing else asks the composed check again. Not while a check is on its way (its verdict is empty then). + // **A refusal that names the member's OWN pending status** (mesh/repo-check, with the gate not pending: the composed + // check set it) is not that: the composed check sets no repository check, so asking it again changes nothing. The + // member's own check is asked, once for the head (issue 1049); its verdict sets the status. + if !again && g.Closed && g.Check != nil && g.Check.Verdict != "" { + for _, d := range members { + if !refusedAfter(d, g.Check.At) { + continue + } + if ownStatusPending(d.Ready.Answer) { + if a := h.askOwnAfterRefusal(d); a != nil { + asks = append(asks, a) + } + continue + } + again = true + h.Logf("[mesh-delivery] group %s: the merge of %s was refused (%s), so its composed check is asked again", g.ID, + d.ID, d.Ready.Answer) + break + } + } + if allReady && (g.Check == nil && !g.Closed || again) { + ms := membersOfAsk(members) + id := g.ID + asks = append(asks, func() { + asked, err := h.Controller.Check(id, ms) + h.mu.Lock() + defer h.mu.Unlock() + g := h.groups[id] + if g == nil { + return + } + if err != nil { + h.Logf("[mesh-delivery] group %s's composed check cannot be asked yet: %v", id, err) + h.again("a group's composed check that could not be asked") + return + } + g.Check = &GroupCheck{Asked: asked, At: h.Now(), Commits: commits} + g.Updated = h.Now() + h.keepGroup(g) + }) + } + // A superseded member of a closed group: whether a later delivered walk on its trunk carried every module it + // moves, ADR 0270's covered test, kept on it as CoveredBy. Its own state stays superseded — a final state never + // moves — and its group counts it as reached (novox/hq issue 442). + if g.Closed { + for _, d := range members { + if d.State != Superseded || d.CoveredBy != "" { + continue + } + var ask []string + if d.CoveredBy, ask = h.coverOf(d); d.CoveredBy != "" { + h.keep(d) + } + for _, in := range ask { + if a := h.askContains(d, in, now); a != nil { + asks = append(asks, a) + } + } + } + } + // Its state, said when it changes. + state, why := GroupState(g, members) + status := groupStatusState(state) + if state == "delivering" && !(g.Check != nil && (g.Check.Verdict == "pass" || g.Check.Verdict == "warning") && + sameMembers(g.Check.Commits, commitsOf(members)) && g.Check.OutOfDate == "") { + // Success there lets the merge tool merge its members: only for a composed check that passed for these heads. + status = "pending" + } + if state != g.Said || status != g.SaidStatus { + g.Said, g.SaidStatus = state, status + for _, d := range members { + g.Owed = append(g.Owed, Effect{Kind: EffectStatus, Commit: d.Commit, Context: "mesh/delivery-group", + State: status, Line: clip(fmt.Sprintf("group %s %s: %s", g.ID, state, why), 140), + Since: now, Last: d.Repository + "#" + fmt.Sprint(d.Number) + " " + d.HTMLURL}) + } + g.Owed = append(g.Owed, Effect{Kind: EffectEmit, Line: string(mustJSON(map[string]any{"group": g.ID, + "state": state, "why": why, "members": g.Members})), Since: now}) + g.Updated = now + h.keepGroup(g) + } + // Each member's mesh/merge-gate is the composed check's (ADR 0403 §1, §6): pending, saying why, while no composed + // verdict stands at these heads. The verdict itself sets it, success or failure, in the forge module. + if why := composedPending(g, members); why != "" { + if g.SaidGate == nil { + g.SaidGate = map[string]string{} + } + for _, d := range members { + if d.MergedAs != "" || d.State.Final() || g.SaidGate[d.ID] == why { + continue + } + g.SaidGate[d.ID] = why + g.Owed = append(g.Owed, Effect{Kind: EffectStatus, Commit: d.Commit, Context: "mesh/merge-gate", State: "pending", + Line: clip("group "+g.ID+": "+why, 140), Since: now, Last: d.Repository + "#" + fmt.Sprint(d.Number) + " " + d.HTMLURL}) + g.Updated = now + h.keepGroup(g) + } + } else if len(g.SaidGate) > 0 { + g.SaidGate = nil + h.keepGroup(g) + } + // Its word, once, when every member is merged (ADR 0418 §2). + if g.Closed { + asks = append(asks, h.groupWord(g, members)...) + } + return asks +} + +// composedPending is why a group member's gate stands pending: the composed check has given no verdict at the members' +// heads as they are, or its verdict is out of date; empty once one stands, which the forge module then sets as every +// member's gate (novox/hq ADR 0403 §6). +func composedPending(g *Group, members []*Delivery) string { + c := g.Check + switch { + case c == nil || c.Verdict == "": + return "its group's composed check has not given a verdict at the members' heads yet" + case !sameMembers(c.Commits, commitsOf(members)): + return "composed again for the group's new heads" + case c.OutOfDate != "": + return c.OutOfDate + } + return "" +} + +// membersOfAsk are a group's members as the controller is asked about them: by id. No order is asked for or kept; the +// plan's tiers take it from the module graph (novox/hq ADR 0418 §2, ADR 0419). +func membersOfAsk(members []*Delivery) []Member { + sorted := append([]*Delivery(nil), members...) + sort.Slice(sorted, func(i, j int) bool { return sorted[i].ID < sorted[j].ID }) + out := make([]Member, 0, len(sorted)) + for _, d := range sorted { + out = append(out, MemberOf(d)) + } + return out +} + +// groupWord gives a closed group's deliveries their turn for the plan's word, once every member is merged (novox/hq +// ADR 0418 §2, issue 978): no member has a turn of its own, and none waits for another's delivery. The word itself is +// asked once per plan by words, when every live delivery the plan carries is at its turn. A member that failed or was +// stopped stops no other: a group's plan that fails fails as every plan does (ADR 0418 §5), and a stop is a person's. +func (h *Holder) groupWord(g *Group, members []*Delivery) []func() { + for _, d := range members { + if d.MergedAs == "" && !d.State.Final() { + return nil // a member not yet on the trunk: the plan waits for it, nothing is let go + } + } + // Composed for these very heads: a verdict on another set of heads is no verdict on these. + composed := g.Check != nil && (g.Check.Verdict == "pass" || g.Check.Verdict == "warning") && + sameMembers(g.Check.Commits, commitsOf(members)) && g.Check.OutOfDate == "" + sorted := append([]*Delivery(nil), members...) + sort.Slice(sorted, func(i, j int) bool { return sorted[i].ID < sorted[j].ID }) + var asks []func() + for _, d := range sorted { + switch { + case memberReached(d), h.stopLapsed(d): + continue + case d.State == Held: + // Held for a composed check that has since passed for the heads that merged: lifted, published again + // (novox/hq issue 316). Any other hold is a person's. + if composed && heldForGroup(d, g) && walkCanGoOn(d) { + d.HeldWhy = "" + d.Lifted = fmt.Sprintf("its group %s's composed check said %s for the heads that merged: %s", g.ID, + g.Check.Verdict, g.Check.Summary) + h.settleAndKeep(d) + if d.State == Published { + if a := h.turn(d, "its group "+g.ID+": every member is merged"); a != nil { + asks = append(asks, a) + } + } + } + case d.State == Published: + // Asked and not answered is not "did not pass": it waits, published, for the answer (issue 316). Nor is a + // composed check of other heads, or one out of date: it runs again for these heads, and the member waits + // for it (novox/hq ADR 0323 §6, issue 406). + if !composed && d.HeldWhy == "" && g.Check != nil && (g.Check.Verdict == "" || g.Check.OutOfDate != "" || + !sameMembers(g.Check.Commits, commitsOf(members))) { + continue + } + if !composed && d.HeldWhy == "" { + d.HeldWhy = fmt.Sprintf("its group %s's composed check did not pass for the heads that merged; a person "+ + "decides that it goes on", g.ID) + d.Unwalked = false + h.settleAndKeep(d) + continue + } + if a := h.turn(d, "its group "+g.ID+": every member is merged"); a != nil { + asks = append(asks, a) + } + } + } + return asks +} + +// stopLeaving asks the controller to stop a walk or batch without merges (ADR 0299) and says what came of it. A +// refusal — an older controller takes no `without`, or the leave-out could not be made — falls back to stopping the +// whole walk: the stopped change is held back, and every delivery on it ends stopped with the reason (review of +// #221). Outside the lock. +func (h *Holder) stopLeaving(walk, why, by string, without []string) string { + h.mu.Lock() + h.askedToLeave(walk) + h.mu.Unlock() + a, err := h.Controller.StopWithout(walk, why, by, without) + if err == nil { + return fmt.Sprintf("the walk %s is stopped without %s; %s", walk, strings.Join(without, ", "), againWords(a)) + } + refused := err + if err := h.Controller.Stop(walk, fmt.Sprintf("%s, and it could not be left out alone (%v)", why, refused), by); err != nil { + h.mu.Lock() + if h.leaveRefused == nil { + h.leaveRefused = map[string]string{} + } + h.leaveRefused[walk] = fmt.Sprintf("leaving it out (%v), and stopping the walk whole (%v)", refused, err) + h.mu.Unlock() + return fmt.Sprintf("the walk %s could not be stopped without %s (%v), nor whole (%v): `plans stop %s --why …` "+ + "ends it by hand", walk, strings.Join(without, ", "), refused, err, walk) + } + return fmt.Sprintf("the walk %s could not be stopped without %s (%v), so it was stopped whole: every delivery on it "+ + "ends stopped", walk, strings.Join(without, ", "), refused) +} + +// carriesStopped are the merges a walk or batch carries whose deliveries were stopped, as `@`, +// and why the first was: a record that names a stopped delivery's merge and has not left it out would deliver a +// stopped change (review of #221). Under the lock. +func (h *Holder) carriesStopped(walk string) ([]string, string) { + s, known := h.walks[walk] + if !known || !(s.walk.Open() || s.walk.IsBatch()) || s.walk.Delivery == nil || len(s.walk.Delivery.Merges) == 0 { + return nil, "" + } + var out []string + why := "" + ids := make([]string, 0, len(h.deliveries)) + for id := range h.deliveries { + ids = append(ids, id) + } + slices.Sort(ids) + for _, id := range ids { + d := h.deliveries[id] + // Stopped by a person or by its group, the acts that hold a change back: never one ended by what its walk + // did (the observed rows, which name nobody), such as a merge stranded after its walk (issue 474). + if d.State != Stopped || d.MergedAs == "" || len(d.Transitions) == 0 { + continue + } + if last := d.Transitions[len(d.Transitions)-1]; last.Event != EvStop || last.By == "" { + continue + } + // A stop whose change a later walk delivered since, or one its group made for a member delivered since, + // holds nothing back: the group stops a walk only for a member whose change is still undelivered (novox/hq + // issue 747). + if d.coveredStop() || h.stopLapsed(d) { + continue + } + // By the merges it names alone: its own commit may be another's merge it no longer answers. + if !slices.ContainsFunc(s.walk.Delivery.Merges, func(m WalkMerge) bool { + return strings.EqualFold(m.Repository, d.Repository) && sameCommit(m.Commit, d.MergedAs) + }) { + continue + } + if _, out := s.walk.leftOut(d.Repository, d.MergedAs); out { + continue + } + out = append(out, d.Repository+"@"+d.MergedAs) + if why == "" { + why = fmt.Sprintf("%s was stopped: %s", d.ID, lastWhyOf(d)) + } + } + return out, why +} + +// coverAwaited is whether a stopped merge a walk carries, given as `@`, waits for the forge's +// word on whether a later finished walk delivered it or the member its group stopped it for (novox/hq issue 747). +// It says since when the forge was first asked, the earliest of them. Held only within coverAskBound of the first +// ask: past it the forge's silence or error lets the walk be asked to leave the merge out. Under the lock. +func (h *Holder) coverAwaited(merges []string) (time.Time, bool) { + var since time.Time + awaited := false + for _, d := range h.deliveries { + if d.State != Stopped || d.MergedAs == "" || !containsString(merges, d.Repository+"@"+d.MergedAs) { + continue + } + for _, id := range []string{d.ID, d.stoppedFor()} { + if id == "" || !h.coverAsked[id] { + continue + } + if at := h.coverAskedSince[id]; !awaited || at.Before(since) { + since = at + } + awaited = true + } + } + return since, awaited +} + +// coverHeld says why a walk is held back while the forge is asked whether a later finished walk delivered a stopped +// merge it carries (novox/hq issue 747), or nothing when it is not. Said by `waits`, `stalled` and `stalled walks`, so +// a forge that cannot answer is loud. Under the lock. +func (h *Holder) coverHeld(walk string) string { + without, _ := h.carriesStopped(walk) + if len(without) == 0 { + return "" + } + since, awaited := h.coverAwaited(without) + if !awaited { + return "" + } + return fmt.Sprintf("it carries %s, whose delivery was stopped, and the forge is asked since %s whether a later "+ + "finished walk delivered it; it is asked to leave the merge out if no answer comes by %s (novox/hq issue 747)", + strings.Join(without, ", "), since.Local().Format("15:04:05"), since.Add(coverAskBound).Local().Format("15:04:05")) +} + +// coverAskBound is how long a walk carrying a stopped merge is held back while the forge is asked whether a later +// finished walk delivered it (novox/hq issue 747); past it, the walk is asked to leave the merge out. +const coverAskBound = 10 * time.Minute + +// lastWhyOf is the reason of a delivery's last transition. +func lastWhyOf(d *Delivery) string { + if len(d.Transitions) == 0 { + return "" + } + return d.Transitions[len(d.Transitions)-1].Why +} + +// leaveAsk is when a walk was last asked to leave merges out, and at which revision of its record. +type leaveAsk struct { + at time.Time + revision int64 +} + +// askedToLeave notes an ask to leave merges out of a walk, at its record's revision: under the lock. +func (h *Holder) askedToLeave(walk string) { + if h.leaveAsked == nil { + h.leaveAsked = map[string]leaveAsk{} + } + h.leaveAsked[walk] = leaveAsk{at: h.Now(), revision: h.walks[walk].walk.Revision} + h.lookAt(h.Now().Add(5*time.Minute), "a walk asked to leave a stopped merge out, asked again if it went nowhere") +} + +// leaveStopped asks, for every open walk or batch that carries a stopped delivery's merge, that the merge be left +// out (ADR 0299, review of #221): a merge walked again and then stopped, or stopped while its batch assembled, never +// ships. Asked once per record of it while it is carried; such a walk is never given its word (words). +func (h *Holder) leaveStopped() []func() { + h.mu.Lock() + defer h.mu.Unlock() + now := h.Now() + var asks []func() + held := map[string]string{} + for id := range h.walks { + if without, _ := h.carriesStopped(id); len(without) > 0 { + said := fmt.Sprintf("its walk %s is not let go: it carries %s, whose delivery was stopped, and is asked to "+ + "leave it out (novox/hq ADR 0299)", id, strings.Join(without, ", ")) + if hold := h.coverHeld(id); hold != "" { + said = "its walk " + id + " is held back: " + hold + } + if r := h.leaveRefused[id]; r != "" { + said += "; the controller refused: " + r + } + held[id] = said + } + } + // What the waiting deliveries say: `waits`, and `stalled` past their bound. + for _, d := range h.deliveries { + said := "" + if d.Walk != nil && !d.State.Final() { + said = held[d.Walk.ID] + } + if d.HeldBack != said { + d.HeldBack = said + h.keep(d) + } + } + for id, seen := range h.walks { + without, why := h.carriesStopped(id) + if len(without) == 0 { + delete(h.leaveRefused, id) + continue + } + if _, awaited := h.coverAwaited(without); awaited { + continue // held back, not stopped, until the forge says whether a later walk delivered it (issue 747) + } + // Once per record of it: the record that answers an ask is newer. Asked again after five minutes, should + // the ask have gone nowhere. + if a, asked := h.leaveAsked[id]; asked && seen.walk.Revision <= a.revision && now.Sub(a.at) < 5*time.Minute { + h.lookAt(a.at.Add(5*time.Minute), "a walk asked to leave a stopped merge out, asked again if it went nowhere") + continue + } + walk := id + asks = append(asks, func() { h.Logf("[mesh-delivery] %s", h.stopLeaving(walk, why, byOwner, without)) }) + } + return asks +} + +// leave gathers a delivery's merge to leave out of a walk (ADR 0299), for a caller holding the lock; true for the +// first of that walk, whose ask asks for all. +func (h *Holder) leave(walk string, d *Delivery) bool { + if h.leaving == nil { + h.leaving = map[string][]string{} + } + first := len(h.leaving[walk]) == 0 + if m := d.Repository + "@" + d.MergedAs; d.MergedAs != "" && !containsString(h.leaving[walk], m) { + h.leaving[walk] = append(h.leaving[walk], m) + } + return first +} + +// leaving459 takes the merges gathered to leave out of a walk, and whether the walk carries anything else: another +// merge it names, or a delivery not ended that follows it. Taken under the lock, outside any. +func (h *Holder) leaving459(walk string) ([]string, bool) { + h.mu.Lock() + defer h.mu.Unlock() + without := h.leaving[walk] + delete(h.leaving, walk) + return without, len(h.othersOn(walk, without)) > 0 +} + +// othersOn are what a walk carries besides the merges named: each delivery not ended that follows it, by id, and +// each merge it names that no such delivery holds, as `@`. +func (h *Holder) othersOn(walk string, without []string) []string { + out := map[string]bool{} + held := map[string]bool{} + for _, d := range h.deliveries { + if d.Walk == nil || d.Walk.ID != walk || d.MergedAs == "" { + continue + } + m := d.Repository + "@" + d.MergedAs + held[strings.ToLower(m)] = true + if !d.State.Final() && !containsString(without, m) { + out[d.ID] = true + } + } + if s, known := h.walks[walk]; known && s.walk.Delivery != nil { + for _, m := range s.walk.Delivery.Merges { + key := m.Repository + "@" + m.Commit + if !held[strings.ToLower(key)] && !slices.ContainsFunc(without, func(w string) bool { return strings.EqualFold(w, key) }) { + out[key] = true + } + } + } + keys := make([]string, 0, len(out)) + for k := range out { + keys = append(keys, k) + } + slices.Sort(keys) + return keys +} + +// againWords is what a stop that left merges out did, in a person's words: the walk that walks the others again, +// what it left out (the controller's word on what contains a stopped merge), what stays on the machines, and what +// will still deliver a stopped change (ADR 0299). +func againWords(a StopAnswer) string { + var left []string + for _, o := range a.LeftOut { + s := o.Repository + "@" + shortOf(o.Commit, 12) + switch { + case len(o.Shares) > 0: + s += " (moves " + strings.Join(o.Shares, ", ") + ", as " + shortOf(o.Contains, 12) + " does)" + case o.Contains != "": + s += " (contains " + shortOf(o.Contains, 12) + ")" + } + left = append(left, s) + } + var out string + switch { + case a.WalkedAgainBy != "": + out = fmt.Sprintf("%s walks its other merges again: %s; left out: %s", a.WalkedAgainBy, + strings.Join(a.WalkedAgain, ", "), orNone(strings.Join(left, ", "))) + case len(left) > 0: + out = "left out: " + strings.Join(left, ", ") + "; nothing else to walk" + default: + out = "nothing else to walk" + } + if len(a.Withheld) > 0 { + out += "; its walk withholds " + strings.Join(a.Withheld, ", ") + } + if len(a.Kept) > 0 { + var kept []string + for _, k := range a.Kept { + kept = append(kept, k.Module+" on "+strings.Join(k.Machines, ", ")) + } + out += "; what the stopped walk already sent stays on the machines: " + strings.Join(kept, "; ") + } + if len(a.StillCarriedBy) > 0 { + out += "; still to deliver the stopped change when walked, a revert's to hold back: " + + strings.Join(a.StillCarriedBy, ", ") + } + return out +} + +// GroupState is a group's state, derived from its members and its composed check — never set apart (novox/hq +// ADR 0239 decision 4, which lists checking, rejected, ready, delivering, delivered, failed and stopped). +// +// It extends that list for a superseded member, which ADR 0239 does not name (novox/hq issue 442: five closed +// groups read "delivering, 1 of 2 delivered" for days with nothing in progress). A superseded member is an end. +// It counts as reached when a later delivered walk on its trunk carried every module it moves (CoveredBy, ADR +// 0270's covered test); a closed group whose members all ended is then delivered, and otherwise stopped, naming +// the member. A group with nothing in progress never reads delivering. +// +// groupWord agrees: a group's plan is let go once, when every member is merged (novox/hq ADR 0418 §2). +func GroupState(g *Group, members []*Delivery) (string, string) { + count := map[State]int{} + reached, ended := 0, 0 + var uncovered, lapsed []string + byID := map[string]*Delivery{} + for _, d := range members { + byID[d.ID] = d + } + for _, d := range members { + if d.State.Final() { + ended++ + } + switch { + case memberReached(d): + // Reached, whatever its final state: a failure or stop a later walk delivered since is no failure of the + // group (novox/hq issue 747). + reached++ + continue + case stopLapsed(d, func(id string) *Delivery { return byID[id] }): + lapsed = append(lapsed, d.ID) + continue + case d.State == Superseded: + uncovered = append(uncovered, d.ID) + } + count[d.State]++ + } + n := len(members) + switch { + case count[Failed] > 0: + return "failed", "a member failed" + case count[Stopped] > 0 && g.Closed: + return "stopped", "a member was stopped" + case n > 0 && reached == n: + return "delivered", "every member is delivered" + case g.Closed && len(lapsed) > 0 && reached+len(lapsed) == n: + // Stopped for a member delivered since: its own change waits for a later walk of its repository, which then + // covers it (novox/hq issue 747). + return "delivering", fmt.Sprintf("%d of %d delivered; %s, stopped for a member delivered since, waits for a "+ + "later walk of its repository", reached, n, strings.Join(lapsed, ", ")) + case g.Closed && n > 0 && ended == n: + return "stopped", supersededWhy + strings.Join(uncovered, ", ") + case g.Closed && g.Check != nil && (g.Check.OutOfDate != "" || g.Check.Verdict == "" || + !sameMembers(g.Check.Commits, commitsOf(members))): + // A member rebased or pushed once the group was delivering, or its composed check out of date: it is composed + // again for these heads, and the group is checking until then, never held (novox/hq ADR 0323 §6, issue 406). + if g.Check.OutOfDate != "" { + return "checking", g.Check.OutOfDate + } + return "checking", "composed again for its members' new heads" + case g.Closed && g.Check != nil && g.Check.Verdict != "pass" && g.Check.Verdict != "warning": + // Composed again for new heads once it was delivering, and those do not compose: its members wait (issue 902). + return "rejected", "composed together: " + g.Check.Summary + case g.Closed: + return "delivering", fmt.Sprintf("%d of %d delivered", reached, n) + case count[Rejected] > 0: + return "rejected", "a member's own check failed" + case g.Check != nil && g.Check.OutOfDate != "": + // Out of date counts as outstanding (ADR 0239 §18, novox/hq ADR 0323 §7): checking, saying why. + return "checking", g.Check.OutOfDate + case g.Check != nil && g.Check.Verdict != "" && g.Check.Verdict != "pass" && g.Check.Verdict != "warning": + return "rejected", "composed together: " + g.Check.Summary + case count[Ready] == n && g.Check != nil && sameMembers(g.Check.Commits, commitsOf(members)) && + (g.Check.Verdict == "pass" || g.Check.Verdict == "warning"): + return "ready", "every member ready, and composed together they pass" + } + return "checking", fmt.Sprintf("%d of %d member(s) ready", count[Ready], n) +} + +// supersededWhy begins why a group whose members all ended is stopped. It is kept short: the forge's status line, +// "group stopped: ", is clipped at 140 characters, and the member's id must fit in it whole. +const supersededWhy = "superseded, not carried by a later delivery: " + +// groupStatusState is the forge status mesh/delivery-group takes for a group's state. A delivering group passed its composed check for these very heads (GroupState reads checking while it is out of +// date), so it is a success too: its members after the first merge only once mesh/delivery-group is one at their +// heads (novox/hq ADR 0382), and a pending one there would hold them for ever (novox/hq issue 902). +func groupStatusState(s string) string { + switch s { + case "ready", "delivering", "delivered": + return "success" + case "rejected", "failed": + return "failure" + case "stopped": + return "error" + } + return "pending" +} + +func clip(s string, n int) string { + if len(s) > n { + return s[:n-1] + "…" + } + return s +} + +func containsString(xs []string, x string) bool { + for _, y := range xs { + if y == x { + return true + } + } + return false +} + +// refusedAfter is whether a member's merge, not yet merged, was refused by the merge tool in a try made at or after at. +func refusedAfter(d *Delivery, at time.Time) bool { + r := d.Ready + return r != nil && r.Merged == nil && d.MergedAs == "" && r.RefusedSince != nil && !r.LastTry.Before(at) +} + +// ownStatusPending is whether the merge tool refused a merge because a status of the member's own merge check is +// pending or missing (issue 1276) while its gate is not: the forge's "not a success: mesh/repo-check (pending)" (gitea merge.ts, mergeableUnder). +// A refusal that also names mesh/merge-gate is the group's composed check's to answer, as is a verdict out of date. +func ownStatusPending(answer string) bool { + _, listed, ok := strings.Cut(answer, "not a success:") + return ok && (strings.Contains(listed, "mesh/repo-check (pending)") || strings.Contains(listed, "mesh/repo-check (missing)")) && + !strings.Contains(listed, "mesh/merge-gate") +} + +// askOwnAfterRefusal asks a member's own check once for its head, after the merge tool refused it for a pending status of +// its own. Under the lock; the ask is made outside it. Nil when it was asked already for this head. +func (h *Holder) askOwnAfterRefusal(d *Delivery) func() { + r := d.Ready + if !r.OwnAskedAt.IsZero() { + return nil + } + r.OwnAskedAt = h.Now() + h.keep(d) + id, m := d.ID, MemberOf(d) + return func() { + asked, err := h.Controller.Check("", []Member{m}) + h.mu.Lock() + defer h.mu.Unlock() + d := h.deliveries[id] + if err != nil { + h.Logf("[mesh-delivery] %s's own check cannot be asked yet: %v", id, err) + if d != nil && d.Ready != nil { + d.Ready.OwnAskedAt = time.Time{} + h.keep(d) + } + h.again("a member's own check that could not be asked") + return + } + h.Logf("[mesh-delivery] %s: the merge tool refused it for a pending status of its own, so its own check is asked as %s", + id, asked) + } +} + +func rejudgedSince(members []*Delivery, at time.Time) bool { + for _, d := range members { + if d.Check != nil && d.Check.At.After(at) { + return true + } + } + return false +} diff --git a/modules/mesh-delivery/cmd/mesh-delivery/holder_test.go b/modules/mesh-delivery/cmd/mesh-delivery/holder_test.go new file mode 100644 index 00000000..3f1c841b --- /dev/null +++ b/modules/mesh-delivery/cmd/mesh-delivery/holder_test.go @@ -0,0 +1,631 @@ +package main + +import ( + "encoding/json" + "strings" + "testing" + "time" +) + +// The holder, end to end over fakes (novox/hq ADR 0239 *How it is checked*): a pull request's head proposed, +// checked and ready; merged, published, let go, delivering and delivered; a gate that fails, put back and +// failed; a newer head superseding; a restart resuming; the trunk rule; and every transition kept before +// it is said, noted on the commit and shown on the pull request. + +const ( + head = "aaaaaaaaaaaa1111" + merge = "mmmmmmmmmmmm9999" +) + +func TestAPullRequestsHeadIsProposedCheckedAndReady(t *testing.T) { + w := newWorld(t) + w.h.PullUpdated(pr("novox/app", 7, head, "feat/x", "src/a.go")) + id := IDOf("novox/app", head) + if w.state(id) != Proposed { + t.Fatalf("a head is %s", w.state(id)) + } + w.h.Checked(verdict("novox/app", 7, head, "pass")) + d := w.delivery(id) + if d.State != Ready || d.Plan == nil || d.Plan.Summary != "builds app → anchor" { + t.Fatalf("a passing verdict left it %s with plan %+v", d.State, d.Plan) + } + var path []State + for _, tr := range d.Transitions { + path = append(path, tr.To) + } + if strings.Join(statesOf(path), ",") != "proposed,checked,ready" { + t.Fatalf("its transitions are %v", path) + } + w.settleAll() + notes := w.forge.notesOn(head) + if len(notes) != 3 || !strings.Contains(notes[2], "checked -> ready") { + t.Fatalf("the head's note is %q", notes) + } + if !strings.Contains(w.forge.statuses["novox/app@"+head+" mesh/delivery"], "success ready") { + t.Fatalf("the head's status is %q", w.forge.statuses) + } + if v := w.forge.views["novox/app#7"]; !strings.Contains(v, viewMarker) || !strings.Contains(v, "**ready**") { + t.Fatalf("the pull request's view is %q", v) + } + // A failing verdict for another head rejects it, and a newer head supersedes the older. + w.h.PullUpdated(pr("novox/app", 7, "bbbbbbbbbbbb2222", "feat/x")) + if w.state(id) != Superseded { + t.Fatalf("a newer head left the older %s", w.state(id)) + } + w.h.Checked(verdict("novox/app", 7, "bbbbbbbbbbbb2222", "fail")) + if s := w.state(IDOf("novox/app", "bbbbbbbbbbbb2222")); s != Rejected { + t.Fatalf("a failing verdict left it %s", s) + } +} + +func statesOf(ss []State) []string { + var out []string + for _, s := range ss { + out = append(out, string(s)) + } + return out +} + +// A merge publishes; the walk waits for this owner's word; the word is given; the walk's record moves the +// delivery through its machines to delivered — and the note on the merge commit says what was executed. +func TestAMergeIsPublishedLetGoDeliveringAndDelivered(t *testing.T) { + w := newWorld(t) + id := IDOf("novox/app", head) + w.h.PullUpdated(pr("novox/app", 7, head, "feat/x", "src/a.go")) + w.h.Checked(verdict("novox/app", 7, head, "pass")) + // The walk is heard before the merge: it waits for its delivery. + walk := w.ctl.put(aWalk("plan-1", "novox/app", merge, true)) + w.h.WalkMoved(walk) + if w.state(id) != Ready { + t.Fatalf("a walk for a commit no merge was heard of moved the delivery to %s", w.state(id)) + } + w.h.PullMerged(merged("novox/app", 7, head, merge)) + if w.state(id) != Published { + t.Fatalf("a merge with its walk open left it %s", w.state(id)) + } + w.settleAll() + if len(w.ctl.delivers) != 1 || w.ctl.delivers[0] != "plan-1" { + t.Fatalf("its walk was let go %v", w.ctl.delivers) + } + if w.state(id) != Delivering { + t.Fatalf("let go, it is %s", w.state(id)) + } + // The first machine sent, judged, passed; the rest sent; the walk done. + sent := time.Date(2026, 10, 6, 12, 40, 0, 0, time.UTC) + walk = w.ctl.walk("plan-1") + walk.State = walkRolling + walk.Modules["app"] = &WalkModule{State: "built", First: []string{"anchor"}, FirstAt: &sent, Build: "build-1", + Gate: &WalkGate{Machines: []string{"anchor"}, Since: &sent, Passes: 1}} + w.h.WalkMoved(w.ctl.put(walk)) + if d := w.delivery(id); len(d.Steps) != 1 || d.Steps[0].State != StepJudging { + t.Fatalf("judging on its first machine reads %+v", d.Steps) + } + judged := sent.Add(3 * time.Minute) + walk.Modules["app"].Gate.Verdict, walk.Modules["app"].Gate.JudgedAt = "passed", &judged + walk.Modules["app"].SentAt = &judged + walk.State = walkDone + w.h.WalkMoved(w.ctl.put(walk)) + d := w.delivery(id) + if d.State != Delivered { + t.Fatalf("a walk done left it %s", d.State) + } + if len(d.Steps) != 2 || d.Steps[0].State != StepPassed || d.Steps[1].Machine != "the rest running it" { + t.Fatalf("its machine steps are %+v", d.Steps) + } + w.settleAll() + onMerge := w.forge.notesOn(merge) + if len(onMerge) == 0 || !strings.Contains(strings.Join(onMerge, "\n"), "executed (walk plan-1): app on anchor passed") { + t.Fatalf("the merge commit's note is %q", onMerge) + } + // Said on the bus, kept first: every transition emitted once. + n := 0 + for _, s := range w.said { + if s["id"] == id { + n++ + } + } + if n != len(d.Transitions) { + t.Fatalf("%d transitions, %d said", len(d.Transitions), n) + } +} + +// A first machine's gate fails: what it carried is put back (the controller's), the step is rolled back, +// and the delivery failed — never delivered. +func TestAFailedGateIsRolledBackAndTheDeliveryFails(t *testing.T) { + w := newWorld(t) + id := IDOf("novox/app", head) + w.h.PullUpdated(pr("novox/app", 7, head, "feat/x", "src/a.go")) + w.h.Checked(verdict("novox/app", 7, head, "pass")) + w.h.PullMerged(merged("novox/app", 7, head, merge)) + w.h.WalkMoved(w.ctl.put(aWalk("plan-1", "novox/app", merge, true))) + w.settleAll() + sent := time.Date(2026, 10, 6, 12, 40, 0, 0, time.UTC) + walk := w.ctl.walk("plan-1") + walk.State = walkRolling + walk.Modules["app"] = &WalkModule{State: "built", First: []string{"anchor"}, FirstAt: &sent, + Gate: &WalkGate{Machines: []string{"anchor"}, Since: &sent}} + w.h.WalkMoved(w.ctl.put(walk)) + at := sent.Add(10 * time.Minute) + walk.Modules["app"].Gate.Verdict, walk.Modules["app"].Gate.Why, walk.Modules["app"].Gate.JudgedAt = + "failed", "its tools were not served", &at + walk.Modules["app"].Gate.Rollback = "rolled-back" + walk.State = walkFailed + walk.Note = "app failed its gate on anchor" + w.h.WalkMoved(w.ctl.put(walk)) + d := w.delivery(id) + if d.State != Failed || len(d.Steps) != 1 || d.Steps[0].State != StepRolledBack { + t.Fatalf("a failed gate left it %s, steps %+v", d.State, d.Steps) + } + // An older record arriving late changes nothing; a step the table does not hold is refused. + w.h.WalkMoved(aWalk("plan-1", "novox/app", merge, false)) + if w.state(id) != Failed { + t.Fatal("an older record of the walk moved a final delivery") + } +} + +// A walk superseded that names no walk taking it over (a record from before ADR 0276): superseded, from +// published or delivering. One that names its successor is followed (TestADeliveryFollowsTheWalkThatTookItsOver). +func TestANewerWalkSupersedesAnOlderDelivery(t *testing.T) { + w := newWorld(t) + id := IDOf("novox/app", head) + w.h.PullUpdated(pr("novox/app", 7, head, "feat/x", "src/a.go")) + w.h.Checked(verdict("novox/app", 7, head, "pass")) + w.h.PullMerged(merged("novox/app", 7, head, merge)) + walk := aWalk("plan-1", "novox/app", merge, true) + w.h.WalkMoved(w.ctl.put(walk)) + walk.State = walkSuperseded + walk.Note = "superseded" + w.ctl.down = true // its word could not be given meanwhile + w.settleAll() + w.h.WalkMoved(w.ctl.put(walk)) + if s := w.state(id); s != Superseded { + t.Fatalf("its walk superseded left it %s", s) + } +} + +// The trunk rule: a ready delivery whose commit never reached the trunk is never published, whatever walks +// are heard of; and a walk for a commit no delivery landed as is not taken as its. +func TestACommitOffTheTrunkIsNeverPublished(t *testing.T) { + w := newWorld(t) + id := IDOf("novox/app", head) + w.h.PullUpdated(pr("novox/app", 7, head, "feat/x", "src/a.go")) + w.h.Checked(verdict("novox/app", 7, head, "pass")) + w.h.WalkMoved(w.ctl.put(aWalk("plan-9", "novox/app", head, true))) // a walk of the head itself: no merge + w.settleAll() + if s := w.state(id); s != Ready { + t.Fatalf("an off-trunk head is %s", s) + } + if len(w.ctl.delivers) != 0 { + t.Fatalf("a walk was let go for a commit off the trunk: %v", w.ctl.delivers) + } +} + +// A restart resumes: a new holder over the same state stands every delivery where it was, reads the walks it +// missed back from the controller, and goes on. +func TestARestartResumesFromTheState(t *testing.T) { + w := newWorld(t) + id := IDOf("novox/app", head) + w.h.PullUpdated(pr("novox/app", 7, head, "feat/x", "src/a.go")) + w.h.Checked(verdict("novox/app", 7, head, "pass")) + w.h.PullMerged(merged("novox/app", 7, head, merge)) + w.ctl.put(aWalk("plan-1", "novox/app", merge, true)) + if err := w.h.Reconcile(); err != nil { + t.Fatal(err) + } + w.settleAll() + if s := w.state(id); s != Delivering { + t.Fatalf("before the restart it is %s", s) + } + // While it is down, the walk finishes and nobody says so. + walk := w.ctl.walk("plan-1") + walk.State = walkDone + w.ctl.put(walk) + + again := newWorldOver(t, w.store, w.ctl, w.forge, w.now.Add(time.Hour)) + if s := again.state(id); s != Delivering { + t.Fatalf("read back, it is %s", s) + } + if err := again.h.Reconcile(); err != nil { + t.Fatal(err) + } + if s := again.state(id); s != Delivered { + t.Fatalf("the walk it missed was not read back: %s", s) + } + if len(w.ctl.delivers) != 1 { + t.Fatalf("a restarted holder let the walk go again: %v", w.ctl.delivers) + } +} + +// Kept before it is said: with the state unreachable, nothing is emitted or noted; once it is kept, all of it. +func TestATransitionIsKeptBeforeItIsSaid(t *testing.T) { + w := newWorld(t) + w.store.failing = true + w.h.PullUpdated(pr("novox/app", 7, head, "feat/x", "src/a.go")) + w.settleAll() + if len(w.said) != 0 || len(w.forge.notesOn(head)) != 0 { + t.Fatalf("said before it was kept: %v %v", w.said, w.forge.notesOn(head)) + } + w.store.failing = false + w.settleAll() + if len(w.said) != 1 || len(w.forge.notesOn(head)) != 2 { + t.Fatalf("once kept, said %d and noted %d", len(w.said), len(w.forge.notesOn(head))) + } + // A forge that is away is asked again; a note is appended once however often it is asked. + w.forge.down = true + w.h.Checked(verdict("novox/app", 7, head, "pass")) + w.settleAll() + if len(w.forge.notesOn(head)) != 2 { + t.Fatal("noted while the forge was away") + } + w.forge.down = false + w.later(time.Hour) // each failure put its next try later: past every one of them + w.settleAll() + w.settleAll() + if notes := w.forge.notesOn(head); len(notes) != 4 { + t.Fatalf("after the forge came back the note has %d line(s): %q", len(notes), notes) + } + if d := w.delivery(IDOf("novox/app", head)); len(d.Owed) != 0 { + t.Fatalf("still owed: %+v", d.Owed) + } +} + +// At the switch: a walk already running is adopted as a delivery that is delivering; a waiting walk whose pull +// request this owner never heard of waits for a person, and a person's release lets it go. +func TestTheSwitchAdoptsRunningWalksAndHoldsUnknownWaitingOnes(t *testing.T) { + w := newWorld(t) + running := aWalk("plan-old", "novox/mesh-catalog", "cccccccccccc3333", false) + running.Created = w.now.Add(-time.Hour) + running.State = walkRolling + w.ctl.put(running) + waiting := aWalk("plan-new", "novox/app", "dddddddddddd4444", true) + waiting.Created = w.now.Add(time.Minute) + w.ctl.put(waiting) + if err := w.h.Reconcile(); err != nil { + t.Fatal(err) + } + w.h.React() + if s := w.state(IDOf("novox/mesh-catalog", "cccccccccccc3333")); s != Delivering { + t.Fatalf("a running walk at the switch was adopted as %s", s) + } + if _, made := w.h.deliveries[IDOf("novox/app", "dddddddddddd4444")]; made { + t.Fatal("a new walk was made a delivery before its merge had time to be heard") + } + w.later(3 * time.Minute) + w.h.React() + held := IDOf("novox/app", "dddddddddddd4444") + if s := w.state(held); s != Held { + t.Fatalf("a waiting walk with no pull request is %s", s) + } + if _, err := w.h.Release(held, "", "jochen"); err == nil { + t.Fatal("released without why") + } + if _, err := w.h.Release(held, "checked by hand", "jochen"); err != nil { + t.Fatal(err) + } + if s := w.state(held); s != Delivering || len(w.ctl.delivers) != 1 { + t.Fatalf("released, it is %s and let go %v", s, w.ctl.delivers) + } +} + +// mesh-delivery's controller down: nothing is lost — its word is asked again on the next tick. +func TestAControllerThatDoesNotAnswerIsAskedAgain(t *testing.T) { + w := newWorld(t) + id := IDOf("novox/app", head) + w.h.PullUpdated(pr("novox/app", 7, head, "feat/x", "src/a.go")) + w.h.Checked(verdict("novox/app", 7, head, "pass")) + w.h.PullMerged(merged("novox/app", 7, head, merge)) + w.h.WalkMoved(w.ctl.put(aWalk("plan-1", "novox/app", merge, true))) + w.ctl.down = true + w.settleAll() + if s := w.state(id); s != Published { + t.Fatalf("with the controller down it is %s", s) + } + w.ctl.down = false + w.settleAll() + if s := w.state(id); s != Delivering { + t.Fatalf("with the controller back it is %s", s) + } +} + +// A merge before the check passed is held for a person; a merge this owner never heard the head of is made +// from the forge's word and published when the forge says its gate passed. +func TestAMergeWithoutAPassingCheckIsHeld(t *testing.T) { + w := newWorld(t) + w.h.PullUpdated(pr("novox/app", 7, head, "feat/x", "src/a.go")) + w.h.Checked(verdict("novox/app", 7, head, "fail")) + w.h.PullMerged(merged("novox/app", 7, head, merge)) + if s := w.state(IDOf("novox/app", head)); s != Held { + t.Fatalf("a rejected head merged is %s", s) + } + // Never heard: the forge's statuses on its head decide. + m := merged("novox/lab", 3, "eeeeeeeeeeee5555", "ffffffffffff6666") + m.HeadChecks = map[string]string{"mesh/merge-gate": "success", "mesh/repo-check": "warning"} + m.Paths = []string{"README.md"} + w.h.PullMerged(m) + w.settleAll() + d := w.delivery(IDOf("novox/lab", "eeeeeeeeeeee5555")) + // The fake planner says it moves app; with no walk yet it waits published. + if d.State != Ready && d.State != Published { + t.Fatalf("a merge whose head passed by the forge's word is %s", d.State) + } +} + +// A delivery group (ADR 0239 decision 4, ADR 0418): two heads sharing a branch name across repositories, checked +// together, ready only when both are and their composition passed — and let go for delivery once, when every member is +// merged, whatever order they merged in. No member waits for another's delivery. +func TestAGroupIsCheckedTogetherAndLetGoOnceEveryMemberIsMerged(t *testing.T) { + w := newWorld(t) + const ctlHead, catHead = "c1c1c1c1c1c1c1c1", "d2d2d2d2d2d2d2d2" + ctl, cat := IDOf("novox/mesh-controller", ctlHead), IDOf("novox/mesh-catalog", catHead) + w.h.PullUpdated(pr("novox/mesh-controller", 1, ctlHead, "feat/x", "internal/a.go")) + w.h.PullUpdated(pr("novox/mesh-catalog", 2, catHead, "feat/x", "modules/app/module.json")) + if w.delivery(ctl).Group != "feat/x" || w.delivery(cat).Group != "feat/x" { + t.Fatalf("one branch name in two repositories made no group: %q %q", w.delivery(ctl).Group, w.delivery(cat).Group) + } + w.h.Checked(verdict("novox/mesh-controller", 1, ctlHead, "pass")) + w.settleAll() + if len(composed(w.ctl.checks)) != 0 { + t.Fatal("the group was checked composed before every member was ready") + } + w.h.Checked(verdict("novox/mesh-catalog", 2, catHead, "pass")) + w.settleAll() + if c := composed(w.ctl.checks); len(c) != 1 || c[0] != "feat/x:"+cat+","+ctl { + t.Fatalf("the composed check asked %v, wanted the members by id", w.ctl.checks) + } + if state, _ := GroupState(w.h.groups["feat/x"], w.h.membersOf(w.h.groups["feat/x"])); state != "checking" { + t.Fatalf("before its composed verdict the group is %s", state) + } + w.h.Checked(CheckedEvent{Group: "feat/x", ID: w.h.groups["feat/x"].Check.Asked, Verdict: "pass", Summary: "both compose"}) + w.settleAll() + if state, _ := GroupState(w.h.groups["feat/x"], w.h.membersOf(w.h.groups["feat/x"])); state != "ready" { + t.Fatalf("composed and ready, the group is %s", state) + } + if !strings.HasPrefix(w.forge.statuses["novox/mesh-catalog@"+catHead+" mesh/delivery-group"], "success") { + t.Fatalf("the group's status on a member's head: %v", w.forge.statuses) + } + + // The catalogue merges first; its walk waits — the controller's member is not merged yet, so nothing is let go. + w.h.PullMerged(merged("novox/mesh-catalog", 2, catHead, "m2m2m2m2m2m2")) + w.h.WalkMoved(w.ctl.put(aWalk("plan-cat", "novox/mesh-catalog", "m2m2m2m2m2m2", true))) + w.settleAll() + if w.state(cat) != Published || len(w.ctl.delivers) != 0 { + t.Fatalf("a member was let go before every member merged: %s %v", w.state(cat), w.ctl.delivers) + } + // The controller merges: its walk is on the controller's own path, started at the merge. Every member is merged, + // so the catalogue's walk is let go at once, without waiting for the controller's delivery. + w.h.PullMerged(merged("novox/mesh-controller", 1, ctlHead, "m1m1m1m1m1m1")) + ctlWalk := w.ctl.put(aWalk("plan-ctl", "novox/mesh-controller", "m1m1m1m1m1m1", false)) + w.h.WalkMoved(ctlWalk) + w.settleAll() + if w.state(ctl) != Delivering || len(w.ctl.delivers) != 1 || w.ctl.delivers[0] != "plan-cat" { + t.Fatalf("every member merged, the catalogue was not let go once: %s %v", w.state(ctl), w.ctl.delivers) + } + if w.state(cat) != Delivering { + t.Fatalf("the catalogue's member is %s", w.state(cat)) + } + w.settleAll() + if len(w.ctl.delivers) != 1 { + t.Fatalf("the word was given more than once: %v", w.ctl.delivers) + } +} + +// A member that fails stops no other (ADR 0418 §5): the plan that fails fails as every plan does, and what was delivered +// stays. Nothing is stopped member by member. +func TestAFailedMemberStopsNoOtherMember(t *testing.T) { + w := newWorld(t) + heads := map[string]string{"novox/a": "a1a1a1a1a1a1a1", "novox/b": "b2b2b2b2b2b2b2", "novox/c": "c3c3c3c3c3c3c3"} + ids := map[string]string{} + for i, repo := range []string{"novox/a", "novox/b", "novox/c"} { + ids[repo] = IDOf(repo, heads[repo]) + w.h.PullUpdated(pr(repo, i+1, heads[repo], "feat/y", "x")) + w.h.Checked(verdict(repo, i+1, heads[repo], "pass")) + } + w.settleAll() + w.h.Checked(CheckedEvent{Group: "feat/y", ID: "check-1", Verdict: "pass", Summary: "compose"}) + for i, repo := range []string{"novox/a", "novox/b", "novox/c"} { + mergeCommit := strings.Repeat(string(rune('p'+i)), 12) + w.h.PullMerged(merged(repo, i+1, heads[repo], mergeCommit)) + w.h.WalkMoved(w.ctl.put(aWalk("plan-"+repo[6:], repo, mergeCommit, true))) + } + w.settleAll() + if len(w.ctl.delivers) != 3 { + t.Fatalf("every member merged, yet the walks were let go %v", w.ctl.delivers) + } + a := w.ctl.walk("plan-a") + a.State = walkDone + w.h.WalkMoved(w.ctl.put(a)) + b := w.ctl.walk("plan-b") + b.State = walkFailed + b.Note = "b failed its gate on anchor" + w.h.WalkMoved(w.ctl.put(b)) + w.settleAll() + if w.state(ids["novox/a"]) != Delivered || w.state(ids["novox/b"]) != Failed || w.state(ids["novox/c"]) == Stopped { + t.Fatalf("a %s, b %s, c %s", w.state(ids["novox/a"]), w.state(ids["novox/b"]), w.state(ids["novox/c"])) + } + if len(w.ctl.stops) != 0 || len(w.ctl.withouts) != 0 { + t.Fatalf("a walk was stopped because another failed: %v %v", w.ctl.stops, w.ctl.withouts) + } + if state, _ := GroupState(w.h.groups["feat/y"], w.h.membersOf(w.h.groups["feat/y"])); state != "failed" { + t.Fatalf("the group is %s", state) + } +} + +// ADR 0419 §2, issue 978: a description's `after:` line is not read, and says so once on its pull request. The +// composed check is asked as for any other group: no order is asked of the controller first, and none gates it. +func TestAnAfterLineIsNotReadAndSaidOnceAndNothingWaitsForAnOrder(t *testing.T) { + w := newWorld(t) + w.op.auto = false + pa := pr("novox/a", 1, "a1a1a1a1a1a1a1", "feat/z") + pa.Body = "Goes after the other.\n\nafter: novox/b\n" + w.h.PullUpdated(pa) + w.h.PullUpdated(pr("novox/b", 2, "b2b2b2b2b2b2b2", "feat/z")) + w.h.Checked(verdict("novox/a", 1, "a1a1a1a1a1a1a1", "pass")) + w.h.Checked(verdict("novox/b", 2, "b2b2b2b2b2b2b2", "pass")) + w.settleAll() + w.later(time.Minute) + w.settleAll() + if len(w.ctl.checks) != 1 { + t.Fatalf("the composed check was not asked once: %v", w.ctl.checks) + } + if got := w.ctl.checks[0]; got != "feat/z:"+IDOf("novox/a", "a1a1a1a1a1a1a1")+","+IDOf("novox/b", "b2b2b2b2b2b2b2") { + t.Fatalf("the members asked: %s", got) + } + said := 0 + for _, c := range w.forge.comments { + if strings.Contains(c, "not read") && strings.Contains(c, "novox/a#1") { + said++ + } + } + if said != 1 { + t.Fatalf("the after: line is not said once on its pull request: %v", w.forge.comments) + } + // A new head of the same pull request, with the line still there, is not told again. + pa2 := pr("novox/a", 1, "a9a9a9a9a9a9a9", "feat/z") + pa2.Body = pa.Body + w.h.PullUpdated(pa2) + w.settleAll() + said = 0 + for _, c := range w.forge.comments { + if strings.Contains(c, "not read") { + said++ + } + } + if said != 1 { + t.Fatalf("said again for a new head: %v", w.forge.comments) + } +} + +// A group's members have their mesh/merge-gate said pending, saying why, until a composed verdict stands at their heads +// (ADR 0403 §1, §6); the verdict then sets the gate in the forge module. +func TestAGroupMembersGateIsPendingUntilItsComposedVerdictStands(t *testing.T) { + w := newWorld(t) + w.op.auto = false + const ha, hb = "a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1", "b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2" + w.h.PullUpdated(pr("novox/a", 1, ha, "feat/z")) + w.h.PullUpdated(pr("novox/b", 2, hb, "feat/z")) + w.h.Checked(verdict("novox/a", 1, ha, "pass")) + w.h.Checked(verdict("novox/b", 2, hb, "pass")) + w.settleAll() + for _, at := range []string{"novox/a@" + ha, "novox/b@" + hb} { + if s := w.forge.statuses[at+" mesh/merge-gate"]; !strings.HasPrefix(s, "pending") || !strings.Contains(s, "composed check") { + t.Fatalf("the member's gate on %s is %q, wanted pending with its reason", at, s) + } + } + // Said once while the reason stands. + w.later(time.Minute) + before := w.forge.statusWrites + w.settleAll() + if w.forge.statusWrites != before { + t.Fatalf("the pending gate was written again with no change") + } + // A composed verdict stands: mesh-delivery says nothing more of the gate (the forge module sets it). + w.h.Checked(CheckedEvent{Group: "feat/z", ID: "check-1", Verdict: "pass", Summary: "compose"}) + w.settleAll() + if g := w.h.groups["feat/z"]; len(g.SaidGate) != 0 { + t.Fatalf("a standing verdict left a gate pending: %v", g.SaidGate) + } + // A member's push puts it back to pending. + w.h.PullUpdated(pr("novox/a", 1, "a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9", "feat/z")) + w.settleAll() + if s := w.forge.statuses["novox/b@"+hb+" mesh/merge-gate"]; !strings.HasPrefix(s, "pending") { + t.Fatalf("the sibling's gate after a push is %q", s) + } +} + +// Stalled: a delivery past its state's bound is listed with what H2 may do; close takes only that. +func TestStalledAndCloseWorkFromTheTable(t *testing.T) { + w := newWorld(t) + id := IDOf("novox/app", head) + w.h.PullUpdated(pr("novox/app", 7, head, "feat/x", "src/a.go")) + w.h.Checked(verdict("novox/app", 7, head, "pass")) + w.h.PullMerged(merged("novox/app", 7, head, merge)) + walk := w.ctl.put(aWalk("plan-1", "novox/app", merge, false)) + w.h.WalkMoved(walk) + if w.state(id) != Delivering { + t.Fatalf("it is %s", w.state(id)) + } + if _, err := w.h.Close(id, "too early"); err == nil { + t.Fatal("closed within its bound") + } + w.later(3 * time.Hour) + stalled := w.h.Stalled() + if len(stalled) != 1 || stalled[0].ID != id || !strings.Contains(stalled[0].H2, "close") { + t.Fatalf("stalled %+v", stalled) + } + if _, err := w.h.Close(id, "delivery stalled"); err == nil { + t.Fatal("closed with nothing in the walk's record saying it moved") + } + walk.State = walkDone + w.ctl.put(walk) // finished, and the event was missed + said, err := w.h.Close(id, "delivery stalled") + if err != nil || w.state(id) != Delivered { + t.Fatalf("H2's close left it %s: %v", w.state(id), err) + } + if !strings.Contains(said, "delivered") { + t.Fatalf("close says %q", said) + } + // Held is the operator's: H2 has no transition there. + w.h.PullUpdated(pr("novox/app", 8, "ffffffffffff0000", "feat/q")) + w.h.PullMerged(merged("novox/app", 8, "ffffffffffff0000", "eeeeeeeeeeee0000")) + w.later(48 * time.Hour) + if _, err := w.h.Close(IDOf("novox/app", "ffffffffffff0000"), "x"); err == nil || !strings.Contains(err.Error(), "operator's") { + t.Fatalf("H2 closed a held delivery: %v", err) + } +} + +// novox/hq issue 867, ADR 0382 §1 (contract A): each member of a group in `groups` also says its pull +// request number, full head, base branch and whether it is open; `members` keeps its text form. +func TestGroupsNamesEachMembersPullRequestHeadBaseAndOpen(t *testing.T) { + w := newWorld(t) + headA := "a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1" + headB := "b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2" + pb := pr("novox/b", 2, headB, "feat/z") + pb.Base = "release/1" + w.h.PullUpdated(pr("novox/a", 1, headA, "feat/z")) + w.h.PullUpdated(pb) + lines := w.h.Groups(false) + if len(lines) != 1 || len(lines[0].Details) != 2 || len(lines[0].Members) != 2 { + t.Fatalf("groups: %+v", lines) + } + got := map[string]MemberLine{} + for _, m := range lines[0].Details { + got[m.Repository] = m + } + a, b := got["novox/a"], got["novox/b"] + if a.Number != 1 || a.Head != headA || a.Base != "main" || !a.Open || a.ID == "" { + t.Fatalf("member a: %+v", a) + } + if b.Number != 2 || b.Head != headB || b.Base != "release/1" || !b.Open { + t.Fatalf("member b: %+v", b) + } + if !strings.Contains(lines[0].Members[0], "(") { + t.Fatalf("members lost its text form: %v", lines[0].Members) + } + // Closed unmerged: still listed while the group is, but not open. + w.h.PullClosed(pr("novox/b", 2, headB, "feat/z")) + for _, l := range w.h.Groups(true) { + for _, m := range l.Details { + if m.Repository == "novox/b" && m.Open { + t.Fatalf("a closed pull request is open: %+v", m) + } + } + } + // JSON keys are the contract's. + raw, _ := json.Marshal(a) + for _, k := range []string{`"number":1`, `"head":"` + headA + `"`, `"base":"main"`, `"open":true`} { + if !strings.Contains(string(raw), k) { + t.Fatalf("%s lacks %s", raw, k) + } + } +} + +// composed are the asks of a group's composed check among the checks asked: a member's own check has no group before its colon. +func composed(checks []string) []string { + var out []string + for _, c := range checks { + if !strings.HasPrefix(c, ":") { + out = append(out, c) + } + } + return out +} diff --git a/modules/mesh-delivery/cmd/mesh-delivery/issue1079_test.go b/modules/mesh-delivery/cmd/mesh-delivery/issue1079_test.go new file mode 100644 index 00000000..c5cb6cec --- /dev/null +++ b/modules/mesh-delivery/cmd/mesh-delivery/issue1079_test.go @@ -0,0 +1,101 @@ +package main + +import ( + "strings" + "testing" +) + +// novox/hq issue 1079: mesh-catalog #354's description was edited ten seconds after it was opened to add its +// `Reviewed-by` line, but the line was read only when the head was announced, so the Approve's ask said "not reviewed at +// this head". The forge's edit event is followed; an Approve ask not yet answered is withdrawn and asked again with the +// record read (the operator agreed, 2026-10-18). The review record is never a gate (ADR 0421 §5). + +// askedUnreviewed is a green pull request whose Approve is asked while its description holds no Reviewed-by line. +func askedUnreviewed(t *testing.T) *world { + w := newWorld(t) + w.op.auto = false + p := pr("novox/app", 7, readyHead, "feat/x", "src/a.go") + p.Title = "Fix the apostrophe (issue 948)" + w.h.PullUpdated(p) + w.h.Checked(judgedAt("novox/app", 7, readyHead, "pass", "t1t1t1t1t1t1")) + w.forge.open = []OpenPull{openPull("novox/app", 7, readyHead)} + w.h.SeeOpenPulls() + w.settleAll() + if open := w.op.open(); len(open) != 1 || !strings.Contains(w.op.ask(open[0]).Explanation, "not reviewed at this head") { + t.Fatalf("setup: the ask is %v", w.op.open()) + } + return w +} + +func edited(body string) PullEvent { + p := pr("novox/app", 7, readyHead, "feat/x") + p.Body = body + return p +} + +func TestAReviewedByLineWrittenAfterTheAskWithdrawsTheUnansweredAskAndAsksAgain(t *testing.T) { + w := askedUnreviewed(t) + first := w.op.open()[0] + w.h.PullEdited(edited("Reviewed-by: reviewer@shanks at " + readyHead)) + w.settleAll() + open := w.op.open() + if len(open) != 1 || open[0] == first || w.op.count() != 2 { + t.Fatalf("after the edit the open asks are %v (first %s), %d asked; wanted the first withdrawn and one new", open, first, w.op.count()) + } + if len(w.op.cancelled) != 1 || w.op.cancelled[0] != first { + t.Fatalf("the first ask was not withdrawn: %v", w.op.cancelled) + } + if e := w.op.ask(open[0]).Explanation; !strings.Contains(e, "reviewed by reviewer on shanks at "+shortOf(readyHead, 12)) || + strings.Contains(e, "not reviewed") { + t.Fatalf("the new ask does not say the record: %s", e) + } + if len(w.forge.comments) != 0 { + t.Fatalf("a withdrawn ask is not an ended Approve, so nothing is said on the pull request: %v", w.forge.comments) + } + // The new ask is the one the Approve answers. + w.op.answer(open[0], OptionApprove) + w.settleAll() + if len(w.forge.merges) != 1 { + t.Fatalf("the new ask's Approve did not merge: %v", w.forge.merges) + } +} + +func TestAnEditThatChangesNothingTheAskSaysAsksNothingAndNoEditChangesAnAnsweredAsk(t *testing.T) { + w := askedUnreviewed(t) + // The same words: no description with a line, then a description without one. + w.h.PullEdited(edited("A new sentence in the description.")) + w.settleAll() + if w.op.count() != 1 || len(w.op.cancelled) != 0 { + t.Fatalf("an edit that changes no word of the ask asked again: %d asks, cancelled %v", w.op.count(), w.op.cancelled) + } + // An edit that brings a line for another head changes the words: the ask is made again, still saying not reviewed. + w.h.PullEdited(edited("Reviewed-by: reviewer@shanks at " + strings.Repeat("a", 40))) + w.settleAll() + if w.op.count() != 2 || len(w.op.open()) != 1 { + t.Fatalf("a line for another head: %d asks, open %v", w.op.count(), w.op.open()) + } + // Answered: a later edit changes nothing. + w.op.answer(w.op.open()[0], OptionApprove) + w.settleAll() + asked := w.op.count() + w.h.PullEdited(edited("Reviewed-by: reviewer@shanks at " + readyHead)) + w.settleAll() + if w.op.count() != asked || len(w.op.open()) != 0 { + t.Fatalf("an edit after the Approve asked again: %d asks, open %v", w.op.count(), w.op.open()) + } + if d := w.delivery(IDOf("novox/app", readyHead)); d.ReviewedBy != "reviewer@shanks" { + t.Fatalf("the record is still read: %q", d.ReviewedBy) + } +} + +func TestAnEditOfAnUnknownOrMergedOrEmptyPullRequestChangesNothing(t *testing.T) { + w := askedUnreviewed(t) + other := pr("novox/other", 3, readyHead, "feat/y") + other.Body = "Reviewed-by: a@b at " + readyHead + w.h.PullEdited(other) // no delivery: nothing made + w.h.PullEdited(edited("")) + w.settleAll() + if w.op.count() != 1 || len(w.h.deliveries) != 1 { + t.Fatalf("asks %d, deliveries %d", w.op.count(), len(w.h.deliveries)) + } +} diff --git a/modules/mesh-delivery/cmd/mesh-delivery/issue1083_test.go b/modules/mesh-delivery/cmd/mesh-delivery/issue1083_test.go new file mode 100644 index 00000000..194f7146 --- /dev/null +++ b/modules/mesh-delivery/cmd/mesh-delivery/issue1083_test.go @@ -0,0 +1,89 @@ +package main + +import ( + "strings" + "testing" +) + +// novox/hq issue 1083: mesh-controller #375's delivery was stopped by a person, then #375 was closed and the same commit +// opened as #376 from another branch. The delivery is named repository@commit; it was Stopped before its pull request +// closed, so it was not ClosedUnmerged, and #376 got no delivery of its own: its Approve was never asked. A new pull +// request at a commit whose delivery is Stopped and unmerged, under another number, takes that record over (the key stays +// repository@commit); the take-over is said in its transitions, with who stopped it and why. + +const ( + head1083 = "e375e375e375e375" + branch1083 = "feat/backup-before-send" +) + +func TestAPullRequestAtACommitWhoseDeliveryAPersonStoppedTakesItOver(t *testing.T) { + w := newWorld(t) + id := IDOf(catalog783, head1083) + + w.h.PullUpdated(pr(catalog783, 375, head1083, branch1083, "src/a.go")) + w.h.Checked(verdict(catalog783, 375, head1083, "pass")) + if w.state(id) != Ready { + t.Fatalf("its passing verdict left it %s", w.state(id)) + } + if _, err := w.h.Stop(id, "to break the loop", "jochen"); err != nil { + t.Fatal(err) + } + // The person stopped it first; the pull request closed afterwards: a final state is not moved by the close. + w.h.PullClosed(pr(catalog783, 375, head1083, branch1083)) + if d := w.delivery(id); d.State != Stopped || d.ClosedUnmerged { + t.Fatalf("setup: %s, closed unmerged %v; wanted stopped by the person alone", d.State, d.ClosedUnmerged) + } + + w.h.PullUpdated(pr(catalog783, 376, head1083, branch1083+"-2", "src/a.go")) + d := w.delivery(id) + if d.State != Proposed || d.Number != 376 { + t.Fatalf("#376 at the same commit left it %s, Number %d; wanted proposed under #376", d.State, d.Number) + } + why := d.Transitions[len(d.Transitions)-1].Why + if !strings.Contains(why, "#375 with the same head was stopped by jochen: to break the loop") { + t.Fatalf("the take-over is not said, with who stopped it and why: %q", why) + } + w.h.Checked(verdict(catalog783, 376, head1083, "pass")) + if w.state(id) != Ready { + t.Fatalf("a passing verdict for #376 left it %s", w.state(id)) + } +} + +// A stopped record that was merged is not taken over, and the same pull request announced again stays stopped. +func TestAStoppedRecordThatWasMergedIsNotTakenOverByAnotherPullRequest(t *testing.T) { + w := newWorld(t) + id := IDOf(catalog783, head783c+"1") + w.h.PullUpdated(pr(catalog783, 400, head783c+"1", branch1083, "src/a.go")) + w.h.PullMerged(merged(catalog783, 400, head783c+"1", "f400f400f400f400")) + w.h.PullUpdated(pr(catalog783, 401, head783c+"1", branch1083+"-2", "src/a.go")) + if d := w.delivery(id); d.Number != 400 || d.MergedAs == "" { + t.Fatalf("a merged record was taken over: Number %d, merged as %q", d.Number, d.MergedAs) + } +} + +// The take-over leaves the stopped record's group: the new record joins its own group by its branch. +func TestATakenOverRecordLeavesTheGroupItWasStoppedIn(t *testing.T) { + w := newWorld(t) + w.h.PullUpdated(pr(gCtl, 1, gHc, "feat/g", "internal/a.go")) + w.h.PullUpdated(pr(gCat, 2, gHk, "feat/g", "modules/app/module.json")) + id := IDOf(gCtl, gHc) + if _, err := w.h.Stop(id, "no", "jochen"); err != nil { + t.Fatal(err) + } + w.h.PullClosed(pr(gCtl, 1, gHc, "feat/g")) + w.h.PullUpdated(pr(gCtl, 9, gHc, "feat/other", "internal/a.go")) + w.h.mu.Lock() + defer w.h.mu.Unlock() + for _, g := range w.h.groups { + if g.ID == "feat/g" { + for _, m := range g.Members { + if m == id { + t.Fatalf("group feat/g still lists %s, now #9 on feat/other: %v", id, g.Members) + } + } + } + } + if d := w.h.deliveries[id]; d.Number != 9 || d.State != Proposed || d.Group == "feat/g" { + t.Fatalf("taken over: Number %d, %s, group %q", d.Number, d.State, d.Group) + } +} diff --git a/modules/mesh-delivery/cmd/mesh-delivery/issue1107_test.go b/modules/mesh-delivery/cmd/mesh-delivery/issue1107_test.go new file mode 100644 index 00000000..f9dea49d --- /dev/null +++ b/modules/mesh-delivery/cmd/mesh-delivery/issue1107_test.go @@ -0,0 +1,61 @@ +package main + +import ( + "encoding/json" + "testing" +) + +// Issue 1107: a pull request closed unmerged in a quiet repository is heard from the forge's own event +// (gitea.forge.pull-request.closed), not only from the poll, which skips a repository whose update time did not move. +// Its delivery leaves its group and stops; a merged one is ignored (the webhook says merged for it); a close heard +// twice, from the event and the poll, changes nothing the second time. +func forgeClosed(owner, repo string, number int, merged bool) map[string]any { + return map[string]any{"owner": owner, "repo": repo, "number": number, "merged": merged} +} + +func TestAForgeClosedEventTakesAGroupMembersDeliveryOutOfItsGroupAndStopsIt(t *testing.T) { + w := newWorld(t) + w.h.PullUpdated(pr(gCtl, 1, gHc, "feat/g", "internal/a.go")) + w.h.PullUpdated(pr(gCat, 2, gHk, "feat/g", "modules/app/module.json")) + id := IDOf(gCtl, gHc) + hear(t, w, "gitea.forge.pull-request.closed", forgeClosed("novox", "mesh-controller", 1, false)) + d := w.delivery(id) + if !d.ClosedUnmerged || !d.State.Final() || inGroup(w, "feat/g", id) { + t.Fatalf("closed %v, state %s, in its group %v; wanted closed, stopped, out of its group", d.ClosedUnmerged, d.State, + inGroup(w, "feat/g", id)) + } + // Heard again, from the event and from the poll: nothing changes. + before, _ := json.Marshal(d) + hear(t, w, "gitea.forge.pull-request.closed", forgeClosed("novox", "mesh-controller", 1, false)) + hear(t, w, "gitea.pull.closed", map[string]any{"owner": "novox", "repo": "mesh-controller", "number": 1}) + if after, _ := json.Marshal(w.delivery(id)); string(after) != string(before) { + t.Fatalf("a second close changed it:\n%s\n%s", before, after) + } +} + +func TestAForgeClosedEventOfAMergedPullRequestIsIgnored(t *testing.T) { + w := newWorld(t) + w.h.PullUpdated(pr(gCtl, 1, gHc, "feat/g", "internal/a.go")) + w.h.PullUpdated(pr(gCat, 2, gHk, "feat/g", "modules/app/module.json")) + id := IDOf(gCtl, gHc) + hear(t, w, "gitea.forge.pull-request.closed", forgeClosed("novox", "mesh-controller", 1, true)) + if d := w.delivery(id); d.ClosedUnmerged || !inGroup(w, "feat/g", id) || d.State.Final() { + t.Fatalf("a merged pull request's close stopped it: closed %v, %s", d.ClosedUnmerged, d.State) + } +} + +// inGroup is whether the group still lists the delivery as a member. +func inGroup(w *world, group, id string) bool { + w.h.mu.Lock() + defer w.h.mu.Unlock() + g := w.h.groups[group] + if g == nil { + return false + } + for _, m := range g.Members { + if m == id { + return true + } + } + return false +} diff --git a/modules/mesh-delivery/cmd/mesh-delivery/issue1114_test.go b/modules/mesh-delivery/cmd/mesh-delivery/issue1114_test.go new file mode 100644 index 00000000..6da2cec4 --- /dev/null +++ b/modules/mesh-delivery/cmd/mesh-delivery/issue1114_test.go @@ -0,0 +1,90 @@ +package main + +import ( + "strings" + "testing" +) + +// novox/hq issue 1114: mesh-controller #379 was opened at the head of the still-open #378. Its announcement read as the +// same delivery announced again, so when #378 closed unmerged the shared record went Stopped and #379 had no delivery. +// The other open pull request at the head is kept on the record; a close of the record's own pull request starts the +// record over for it at once (as issue 1083's take-over does for a later announcement), and an edit of it does so too, +// so a case stuck before the fix frees itself. A merged record is never taken over. + +const head1114 = "0149a3d70149a3d70149a3d70149a3d70149a3d7" + +func TestClosingThePullRequestOfASharedHeadStartsTheOtherOnesDeliveryOver(t *testing.T) { + w := newWorld(t) + w.op.auto = false + id := IDOf(gCtl, head1114) + w.h.PullUpdated(pr(gCtl, 378, head1114, "feat/chain", "internal/a.go")) + b := pr(gCtl, 379, head1114, "feat/chain-2", "internal/a.go") + b.Body = "Reviewed-by: reviewer@shanks at " + head1114 + w.h.PullUpdated(b) // announced while #378 is live: the same delivery heard again + if d := w.delivery(id); d.Number != 378 { + t.Fatalf("setup: the record is #%d, wanted #378 still", d.Number) + } + w.h.PullClosed(pr(gCtl, 378, head1114, "feat/chain")) + d := w.delivery(id) + if d.Number != 379 || d.State.Final() || d.ClosedUnmerged { + t.Fatalf("after #378 closed the record is #%d %s (closed %v); wanted #379, proposed", d.Number, d.State, d.ClosedUnmerged) + } + if d.ReviewedBy == "" { + t.Fatalf("#379's review record was not read: %+v", d) + } + w.h.Checked(verdict(gCtl, 379, head1114, "pass")) + w.settleAll() + if open := w.op.open(); w.state(id) != Ready || len(open) != 1 || + !strings.Contains(w.op.ask(open[0]).Explanation, "reviewed by reviewer") { + t.Fatalf("#379's delivery is %s with asks %v; wanted ready and one Approve ask with its review record", w.state(id), w.op.open()) + } +} + +func TestClosingTheOtherPullRequestOfASharedHeadLeavesTheRecordAlone(t *testing.T) { + w := newWorld(t) + id := IDOf(gCtl, head1114) + w.h.PullUpdated(pr(gCtl, 378, head1114, "feat/chain", "internal/a.go")) + w.h.PullUpdated(pr(gCtl, 379, head1114, "feat/chain-2", "internal/a.go")) + w.h.PullClosed(pr(gCtl, 379, head1114, "feat/chain-2")) + w.h.PullClosed(pr(gCtl, 378, head1114, "feat/chain")) + if d := w.delivery(id); d.Number != 378 || !d.ClosedUnmerged || !d.State.Final() { + t.Fatalf("with #379 closed first, #378's close left %d %s; wanted it stopped under #378", d.Number, d.State) + } +} + +func TestAnEditOfAPullRequestWhoseSharedHeadsRecordIsStoppedUnderAnotherTakesItOver(t *testing.T) { + w := newWorld(t) + id := IDOf(gCtl, head1114) + w.h.PullUpdated(pr(gCtl, 378, head1114, "feat/chain", "internal/a.go")) + w.h.PullClosed(pr(gCtl, 378, head1114, "feat/chain")) // #379 was never heard: the stuck case + if d := w.delivery(id); !d.State.Final() { + t.Fatalf("setup: %s", d.State) + } + e := pr(gCtl, 379, head1114, "feat/chain-2") + e.Body = "Reviewed-by: reviewer@shanks at " + head1114 + w.h.PullEdited(e) + d := w.delivery(id) + if d.Number != 379 || d.State != Proposed || d.ReviewedBy == "" || len(d.Paths) == 0 { + t.Fatalf("after #379's edit the record is #%d %s reviewed %q paths %v; wanted #379 proposed, read, with the paths", d.Number, + d.State, d.ReviewedBy, d.Paths) + } + w.h.PullEdited(e) // an edit heard again changes nothing + if again := w.delivery(id); again.Number != 379 || again.State != Proposed { + t.Fatalf("a second edit moved it: #%d %s", again.Number, again.State) + } +} + +func TestAMergedRecordIsNeverTakenOverByTheOtherPullRequestOfItsHead(t *testing.T) { + w := newWorld(t) + id := IDOf(gCtl, head1114) + w.h.PullUpdated(pr(gCtl, 378, head1114, "feat/chain", "internal/a.go")) + w.h.PullUpdated(pr(gCtl, 379, head1114, "feat/chain-2", "internal/a.go")) + w.h.PullMerged(merged(gCtl, 378, head1114, "f378f378f378f378")) + w.h.PullClosed(pr(gCtl, 378, head1114, "feat/chain")) + e := pr(gCtl, 379, head1114, "feat/chain-2") + e.Body = "text" + w.h.PullEdited(e) + if d := w.delivery(id); d.Number != 378 || d.MergedAs == "" { + t.Fatalf("a merged record was taken over: #%d merged as %q", d.Number, d.MergedAs) + } +} diff --git a/modules/mesh-delivery/cmd/mesh-delivery/issue1275_test.go b/modules/mesh-delivery/cmd/mesh-delivery/issue1275_test.go new file mode 100644 index 00000000..4530e52c --- /dev/null +++ b/modules/mesh-delivery/cmd/mesh-delivery/issue1275_test.go @@ -0,0 +1,413 @@ +package main + +import ( + "errors" + "strings" + "testing" + "time" +) + +const time30s = 31 * time.Second + +// Issue 1275 (the 2026-10-19 18:05 group): a group is merged whole or not at all, in its order, and a group that is held or +// half merged is said loudly. Issue 1276/AC1: a refusal naming the member's own repository check as missing asks that check. + +const ( + gate, repoCheck = "mesh/merge-gate", "mesh/repo-check" +) + +// statusOf sets a member head's two merge checks on the forge, and makes both required. +func setChecks(w *world, repository, sha, gateState, repoState string) { + w.forge.required = []string{gate, repoCheck} + for ctx, state := range map[string]string{gate: gateState, repoCheck: repoState} { + key := repository + "@" + sha + " " + ctx + if state == "" { + delete(w.forge.statuses, key) // no status at all: the forge says it is missing + continue + } + w.forge.statuses[key] = state + " said → x" + } +} + +func allGreen(w *world) { + setChecks(w, gCat, gHk, "success", "success") + setChecks(w, gCtl, gHc, "success", "success") +} + +func refusedWords(repository string) error { + return errors.New("the forge refused gitea_merge_pull_request: " + repository + " is not merged: main requires " + + "mesh/merge-gate, mesh/repo-check; not a success: mesh/repo-check (missing). A pull request merges once its checks succeed") +} + +func urgentLines(w *world) []StalledLine { + var out []StalledLine + for _, l := range w.h.Stalled() { + if l.Severity == "urgent" { + out = append(out, l) + } + } + return out +} + +// 1275/AC2: an approved group whose member has a required status that is not a success at its head when the merge starts +// is merged not at all, though the first member in order is fine; the group says which member and which status. +func TestAnApprovedGroupWithAMemberWhoseRequiredStatusIsNotASuccessMergesNothing(t *testing.T) { + w := approvedGroup(t) + allGreen(w) + w.op.answer(w.op.open()[0], OptionApprove) + setChecks(w, gCtl, gHc, "success", "") // the second member's repository check went missing after the ask + w.settleAll() + if len(w.forge.merges) != 0 { + t.Fatalf("merged with a required status missing: %v", w.forge.merges) + } + lines := urgentLines(w) + if len(lines) != 1 || !strings.Contains(lines[0].Says, "feat/g") || !strings.Contains(lines[0].Says, IDOf(gCtl, gHc)) || + !strings.Contains(lines[0].Says, "mesh/repo-check (missing)") || !strings.Contains(lines[0].Says, "Needed") { + t.Fatalf("the held group is not said as one urgent line naming group, member, status and need: %+v", lines) + } + // The status turns success: the whole group merges. + setChecks(w, gCtl, gHc, "success", "success") + w.h.CommitStatus(ForgeStatusEvent{Owner: "novox", Repo: "mesh-controller", SHA: gHc, Context: repoCheck, State: "success"}) + w.later(time30s) + w.settleAll() + if len(w.forge.merges) != 2 || len(urgentLines(w)) != 0 { + t.Fatalf("merges %v, lines %+v; wanted both merged and no line", w.forge.merges, urgentLines(w)) + } +} + +// 1275/AC2 with 1276/AC1: a member whose own repository check is the missing status has that check asked, once. +func TestAHeldGroupWhoseMemberLacksItsOwnRepositoryCheckAsksThatCheck(t *testing.T) { + w := approvedGroup(t) + allGreen(w) + w.op.answer(w.op.open()[0], OptionApprove) + setChecks(w, gCtl, gHc, "success", "") + asked := len(w.ctl.checks) + w.settleAll() + w.later(time30s) + w.settleAll() + own, composed := ownChecks(w, asked) + if len(composed) != 0 || len(own) != 1 || own[0] != ":"+IDOf(gCtl, gHc) { + t.Fatalf("own checks %v, composed %v; wanted the member's own check once", own, composed) + } +} + +// 1275/AC3: the merge tool refuses the first member in order; the second is not merged, and the group is said urgent, +// naming the group, the refused member and the forge's words. +func TestARefusedFirstMemberStopsTheGroupAndIsSaidUrgent(t *testing.T) { + w := approvedGroup(t) + allGreen(w) + w.forge.refuseRepo = map[string]error{gCat: refusedWords("novox/mesh-catalog#2")} + w.op.answer(w.op.open()[0], OptionApprove) + w.settleAll() + if len(w.forge.merges) != 1 || !strings.Contains(w.forge.merges[0], "mesh-catalog#2") { + t.Fatalf("wanted only the first member asked: %v", w.forge.merges) + } + lines := urgentLines(w) + if len(lines) != 1 || lines[0].ID != IDOf(gCat, gHk) || !strings.Contains(lines[0].Says, "feat/g") || + !strings.Contains(lines[0].Says, "mesh/repo-check (missing)") || !strings.Contains(lines[0].Says, "Needed") { + t.Fatalf("not said as one urgent line: %+v", lines) + } +} + +// 1275/AC4: a group half merged (the second member refused after the first merged) stays raised, pass after pass, until +// the second is merged; then it goes. +func TestAHalfMergedGroupStaysRaisedUntilEveryMemberIsMerged(t *testing.T) { + w := approvedGroup(t) + allGreen(w) + w.forge.refuseRepo = map[string]error{gCtl: refusedWords("novox/mesh-controller#1")} + w.op.answer(w.op.open()[0], OptionApprove) + w.settleAll() + w.h.PullMerged(merged(gCat, 2, gHk, gT1)) + for range 3 { + w.later(time30s) + w.settleAll() + lines := urgentLines(w) + if len(lines) != 1 || lines[0].ID != IDOf(gCtl, gHc) || !strings.Contains(lines[0].Says, "feat/g") { + t.Fatalf("the half merged group is not raised: %+v", lines) + } + } + // A half merged group with no refusal on record (the pass died between members) is raised too. + d := w.delivery(IDOf(gCtl, gHc)) + d.Ready.RefusedSince = nil + if lines := urgentLines(w); len(lines) != 1 || !strings.Contains(lines[0].Says, "half merged") { + t.Fatalf("a half merged group without a refusal is not raised: %+v", lines) + } + w.forge.refuseRepo = nil + setChecks(w, gCtl, gHc, "success", "success") + w.h.PullMerged(merged(gCtl, 1, gHc, gT1)) + w.settleAll() + if lines := urgentLines(w); len(lines) != 0 { + t.Fatalf("the group is whole and still raised: %+v", lines) + } +} + +// 1275/AC1: a group with a member whose required status is not a success is not asked its Approve; the group says which +// member and status; it is asked once every member may merge. +func TestAGroupIsNotAskedWhileAMembersRequiredStatusIsNotASuccess(t *testing.T) { + w := newWorld(t) + w.op.auto = false + w.h.PullUpdated(pr(gCtl, 1, gHc, "feat/g", "internal/a.go")) + w.h.PullUpdated(pr(gCat, 2, gHk, "feat/g", "modules/app/module.json")) + w.h.Checked(judged(gCtl, 1, gHc, "pass", gTrunk0)) + w.h.Checked(judged(gCat, 2, gHk, "pass", gTrunk0)) + w.settleAll() + g := w.h.groups["feat/g"] + setChecks(w, gCat, gHk, "success", "success") // the composed verdict set every member's gate + setChecks(w, gCtl, gHc, "success", "pending") + w.h.Checked(CheckedEvent{Group: "feat/g", ID: g.Check.Asked, Verdict: "pass", Summary: "composed", + Members: []struct { + Owner string `json:"owner"` + Repo string `json:"repo"` + Commit string `json:"commit"` + Trunk string `json:"trunk,omitempty"` + }{{Owner: "novox", Repo: "mesh-controller", Commit: gHc, Trunk: gTrunk0[:12]}, + {Owner: "novox", Repo: "mesh-catalog", Commit: gHk, Trunk: gTrunk0[:12]}}}) + w.settleAll() + if w.op.count() != 0 { + t.Fatalf("the operator was asked with a required status pending: %d asks", w.op.count()) + } + if g.Hold == nil || g.Hold.Kind != "unasked" || g.Hold.Member != IDOf(gCtl, gHc) || !strings.Contains(g.Hold.Says, "mesh/repo-check (pending)") { + t.Fatalf("the group does not say which member and status it waits for: %+v", g.Hold) + } + setChecks(w, gCtl, gHc, "success", "success") + w.later(time30s) + w.settleAll() + if w.op.count() != 1 || g.Hold != nil { + t.Fatalf("asks %d, hold %+v; wanted the group asked once its statuses are green", w.op.count(), g.Hold) + } +} + +// 1275/AC5: a member's required status turning not-success while the ask waits withdraws the ask at the router; the +// answer to the withdrawn ask merges nothing; the group is asked again once every member may merge. +func TestAStatusThatTurnsWhileTheAskWaitsWithdrawsItAndItsAnswerMergesNothing(t *testing.T) { + w := approvedGroup(t) + allGreen(w) + ask := w.op.open()[0] + w.h.CommitStatus(ForgeStatusEvent{Owner: "novox", Repo: "mesh-catalog", SHA: gHk, Context: repoCheck, State: "pending"}) + setChecks(w, gCat, gHk, "success", "pending") + w.settleAll() + if len(w.op.cancelled) != 1 || w.op.cancelled[0] != ask { + t.Fatalf("the ask was not withdrawn at the router: %v", w.op.cancelled) + } + w.op.answer(ask, OptionApprove) // an answer that arrives late + w.settleAll() + if len(w.forge.merges) != 0 { + t.Fatalf("the answer to a withdrawn ask merged: %v", w.forge.merges) + } + if w.op.count() != 1 { + t.Fatalf("asked again while a status was still pending: %d", w.op.count()) + } + setChecks(w, gCat, gHk, "success", "success") + w.later(time30s) + w.settleAll() + if w.op.count() != 2 { + t.Fatalf("not asked again once every member may merge: %d", w.op.count()) + } +} + +// 1276/AC1: given a closed group whose member's merge was refused with "not a success: mesh/repo-check (missing)", then +// the member's own check is asked, not the composed check. +func TestAClosedGroupsMemberRefusedForItsOwnMissingStatusAsksItsOwnCheck(t *testing.T) { + w := approvedGroup(t) + allGreen(w) + w.forge.refuseRepo = map[string]error{gCtl: refusedWords("novox/mesh-controller#1")} + w.op.answer(w.op.open()[0], OptionApprove) + w.settleAll() + w.h.PullMerged(merged(gCat, 2, gHk, gT1)) + asked := len(w.ctl.checks) + w.settleAll() + own, composed := ownChecks(w, asked) + if len(composed) != 0 || len(own) != 1 || own[0] != ":"+IDOf(gCtl, gHc) { + t.Fatalf("own checks %v, composed checks %v; wanted the member's own check once and no composed check", own, composed) + } +} + +// 1275/AC3 (review round 1): a member that is both refused and held for a required status is said by one stalled line, +// not two with the same id. +func TestAMemberBothRefusedAndHeldForAStatusIsOneStalledLine(t *testing.T) { + w := approvedGroup(t) + allGreen(w) + w.forge.refuseRepo = map[string]error{gCat: refusedWords("novox/mesh-catalog#2")} + w.op.answer(w.op.open()[0], OptionApprove) + w.settleAll() + setChecks(w, gCat, gHk, "success", "") // now its status is missing too; the next pass is held + w.h.CommitStatus(ForgeStatusEvent{Owner: "novox", Repo: "mesh-catalog", SHA: gHk, Context: repoCheck, State: "success"}) + w.h.mergeAgainNow(w.delivery(IDOf(gCat, gHk))) + w.later(time30s) + w.settleAll() + if g := w.h.groups["feat/g"]; g.Hold == nil { + t.Fatalf("the group was not held for the missing status") + } + n := 0 + for _, l := range w.h.Stalled() { + if l.ID == IDOf(gCat, gHk) { + n++ + } + } + if n != 1 { + t.Fatalf("%d stalled lines for one member: %+v", n, w.h.Stalled()) + } +} + +// 1275/AC6: a member refused because its verdict is out of date (the trunk moved; its fresh check runs) is a warning at +// once, and urgent only when the group is not merged within ten minutes; the pass stops at the refusal all the same. +func TestAnOutOfDateRefusalIsAWarningAtOnceAndUrgentAfterTenMinutes(t *testing.T) { + w := approvedGroup(t) + allGreen(w) + w.forge.refuse = errAskedAgain + w.op.answer(w.op.open()[0], OptionApprove) + w.settleAll() + if len(w.forge.merges) != 1 { + t.Fatalf("the pass did not stop at the refusal: %v", w.forge.merges) + } + sev := func() []string { + var out []string + for _, l := range w.h.Stalled() { + out = append(out, l.Severity) + } + return out + } + if got := sev(); len(got) != 1 || got[0] != "warning" { + t.Fatalf("an out of date refusal is %v at once; wanted one warning", got) + } + w.later(9 * time.Minute) + if got := sev(); len(got) != 1 || got[0] != "warning" { + t.Fatalf("at nine minutes it is %v; wanted still a warning", got) + } + w.later(2 * time.Minute) + if got := sev(); len(got) != 1 || got[0] != "urgent" { + t.Fatalf("after ten minutes it is %v; wanted urgent", got) + } +} + +// 1275/AC6: every other refusal (here a missing status) is urgent at once. +func TestAMissingStatusRefusalIsUrgentAtOnce(t *testing.T) { + w := approvedGroup(t) + allGreen(w) + w.forge.refuseRepo = map[string]error{gCat: refusedWords("novox/mesh-catalog#2")} + w.op.answer(w.op.open()[0], OptionApprove) + w.settleAll() + if lines := w.h.Stalled(); len(lines) != 1 || lines[0].Severity != "urgent" { + t.Fatalf("a missing status refusal is not urgent at once: %+v", lines) + } +} + +func severities(w *world) []string { + var out []string + for _, l := range w.h.Stalled() { + out = append(out, l.Severity) + } + return out +} + +// 1275/AC6 (b): the same out of date refusal on a LATER member, after the earlier one merged, is a warning at once and +// urgent after ten minutes: the half merged group does not say it urgent at once. +func TestAnOutOfDateRefusalOnALaterMemberIsAWarningThenUrgent(t *testing.T) { + w := approvedGroup(t) + allGreen(w) + w.forge.refuseRepo = map[string]error{gCtl: errAskedAgain} + w.op.answer(w.op.open()[0], OptionApprove) + w.settleAll() + w.h.PullMerged(merged(gCat, 2, gHk, gT1)) + w.settleAll() + if got := severities(w); len(got) != 1 || got[0] != "warning" { + t.Fatalf("a later member's out of date refusal is %v at once; wanted one warning", got) + } + w.later(11 * time.Minute) + if got := severities(w); len(got) != 1 || got[0] != "urgent" { + t.Fatalf("after ten minutes it is %v; wanted urgent", got) + } +} + +// 1275/AC6 (c): words naming an out of date check and also a missing status are urgent at once. +func TestARefusalNamingOutOfDateAndAMissingStatusIsUrgentAtOnce(t *testing.T) { + w := approvedGroup(t) + allGreen(w) + w.forge.refuse = errors.New("the forge refused gitea_merge_pull_request: novox/mesh-catalog#2 is not merged: its verdict was " + + "judged against trunk aaaaaaaaaaaa, and main is now bbbbbbbbbbbb. The check of head cccc merged into trunk bbbb was " + + "asked. main requires mesh/merge-gate, mesh/repo-check; not a success: mesh/repo-check (missing)") + w.op.answer(w.op.open()[0], OptionApprove) + w.settleAll() + if got := severities(w); len(got) != 1 || got[0] != "urgent" { + t.Fatalf("a refusal with two causes is %v; wanted urgent at once", got) + } +} + +// 1275/AC6 (d): words mesh-delivery cannot classify are urgent at once. +func TestARefusalInUnknownWordsIsUrgentAtOnce(t *testing.T) { + w := approvedGroup(t) + allGreen(w) + w.forge.refuse = errors.New("the forge refused gitea_merge_pull_request: something nobody foresaw") + w.op.answer(w.op.open()[0], OptionApprove) + w.settleAll() + if got := severities(w); len(got) != 1 || got[0] != "urgent" { + t.Fatalf("an unclassified refusal is %v; wanted urgent at once", got) + } +} + +// The merge tool's refusals, copied from gitea merge.ts with their line, `${...}` filled in. +var mergeToolRefusals = []struct { + line string + words string + outOfDated bool +}{ + {"merge.ts:121", "novox/app#7 is not merged: its verdict was judged against trunk aaaaaaaaaaaa, and main is now bbbbbbbbbbbb. " + + "Its check is already asked and not yet answered (building), so it is not asked again; took 3s. Ask again at cccccccccccc once " + + "both statuses pass: this tool never merges by itself (novox/hq ADR 0323). A check that never answers is asked again with " + + "mesh-delivery.recheck. Nothing was merged", true}, + {"merge.ts:147", "novox/app#7 is not merged: its verdict was judged against trunk aaaaaaaaaaaa, and main is now bbbbbbbbbbbb. " + + "The check of head cccccccccccc merged into trunk bbbbbbbbbbbb was asked; took 3s. Ask again at cccccccccccc once both " + + "statuses pass: this tool never merges by itself (novox/hq ADR 0323). In a busy repository another merge may move the trunk " + + "again first; this is then refused and asked once more. A check that never answers is asked again with " + + "mesh-delivery.recheck. Nothing was merged", true}, + {"merge.ts:144", "novox/app#7 is not merged: its verdict was judged against trunk aaaaaaaaaaaa, and main is now bbbbbbbbbbbb, " + + "and its check could not be asked: the forge is away. Nothing was merged", false}, + {"merge.ts:156", "novox/app#7 is not merged: main requires mesh/merge-gate, mesh/repo-check; not a success: mesh/repo-check " + + "(pending). A pull request merges once its checks succeed", false}, + {"merge.ts:165", "novox/app#7 is not merged: its verdict read siblings, and which ones is not held here, so the group cannot " + + "be weighed. Ask its check again at cccccccccccc. Nothing was merged", false}, + {"merge.ts:172", "novox/app#7 is not merged: its verdict was judged beside 1 sibling(s), and a sibling is not ready. A member " + + "judged beside its siblings merges once its group passed at the heads that verdict read (novox/hq ADR 0382, 0421). " + + "Nothing was merged", false}, +} + +// 1275/AC6 (e): only the merge tool's two out of date refusals (:121, :147) are warnings; the one saying its check could +// not be asked (:144) and every other is urgent at once. +func TestOnlyTheMergeToolsTwoOutOfDateRefusalsAreWarnings(t *testing.T) { + for _, c := range mergeToolRefusals { + if got := outOfDateRefusal(c.words); got != c.outOfDated { + t.Errorf("%s: outOfDateRefusal = %v, wanted %v", c.line, got, c.outOfDated) + } + w := approvedGroup(t) + allGreen(w) + w.forge.refuse = errors.New("the forge refused gitea_merge_pull_request: " + c.words) + w.op.answer(w.op.open()[0], OptionApprove) + w.settleAll() + want := "urgent" + if c.outOfDated { + want = "warning" + } + if got := severities(w); len(got) != 1 || got[0] != want { + t.Errorf("%s: severity %v at once, wanted %s", c.line, got, want) + } + } +} + +// 1275/AC6 (f): the severity never steps down: a missing status refusal, then an out of date one, stays urgent. +func TestASeverityOnceUrgentNeverStepsDownToAWarning(t *testing.T) { + w := approvedGroup(t) + allGreen(w) + w.forge.refuseRepo = map[string]error{gCat: refusedWords("novox/mesh-catalog#2")} + w.op.answer(w.op.open()[0], OptionApprove) + w.settleAll() + w.forge.refuseRepo = map[string]error{gCat: errAskedAgain} + w.h.mergeAgainNow(w.delivery(IDOf(gCat, gHk))) + w.later(time30s) + w.settleAll() + if d := w.delivery(IDOf(gCat, gHk)); !outOfDateRefusal(d.Ready.Answer) { + t.Fatalf("the second refusal was not the out of date one: %q", d.Ready.Answer) + } + if got := severities(w); len(got) != 1 || got[0] != "urgent" { + t.Fatalf("severity %v after an out of date refusal that followed an urgent one; wanted urgent", got) + } +} diff --git a/modules/mesh-delivery/cmd/mesh-delivery/issue326_test.go b/modules/mesh-delivery/cmd/mesh-delivery/issue326_test.go new file mode 100644 index 00000000..2588dd6d --- /dev/null +++ b/modules/mesh-delivery/cmd/mesh-delivery/issue326_test.go @@ -0,0 +1,89 @@ +package main + +import ( + "testing" + "time" +) + +// novox/hq issue 326: a delivery whose state did not change is not written again — a reconcile pass over +// walks that did not move writes nothing — and one that changed is written, and said. +func TestADeliveryIsWrittenOnlyWhenItChanged(t *testing.T) { + w := newWorld(t) + id := IDOf("novox/app", head) + w.h.PullUpdated(pr("novox/app", 7, head, "feat/x", "src/a.go")) + w.h.Checked(verdict("novox/app", 7, head, "pass")) + w.ctl.put(aWalk("plan-1", "novox/app", merge, true)) + w.h.PullMerged(merged("novox/app", 7, head, merge)) + w.settleAll() + if err := w.h.Reconcile(); err != nil { + t.Fatal(err) + } + w.settleAll() + // The word given moves the walk's record once more: read back, it is a change, and written. + if err := w.h.Reconcile(); err != nil { + t.Fatal(err) + } + w.settleAll() + if w.state(id) != Delivering { + t.Fatalf("let go, it is %s", w.state(id)) + } + + // Passes over the same walks, as main.go's tickers run them: nothing changed, nothing written. + before, said := w.store.puts, len(w.said) + for range 5 { + w.later(30 * time.Second) + if err := w.h.Reconcile(); err != nil { + t.Fatal(err) + } + w.settleAll() + } + if w.store.puts != before { + t.Fatalf("five passes over walks that did not move wrote %d delivery(ies)", w.store.puts-before) + } + if len(w.said) != said { + t.Fatalf("five passes over walks that did not move said %d event(s)", len(w.said)-said) + } + + // The walk moves: the delivery changes, is written, and is said. + walk := w.ctl.walk("plan-1") + walk.State = walkDone + w.ctl.put(walk) + if err := w.h.Reconcile(); err != nil { + t.Fatal(err) + } + w.settleAll() + if w.state(id) != Delivered { + t.Fatalf("a walk done left it %s", w.state(id)) + } + if w.store.puts == before { + t.Fatal("a delivery that changed was not written") + } + back, err := w.store.Deliveries() + if err != nil { + t.Fatal(err) + } + for _, d := range back { + if d.ID == id && d.State != Delivered { + t.Fatalf("the store holds it %s, the holder %s", d.State, Delivered) + } + } + if len(w.said) == said { + t.Fatal("a delivery that changed was not said") + } +} + +// A delivery that could not be written is written again, even though it did not change since. +func TestADeliveryNotWrittenIsWrittenAgainUnchanged(t *testing.T) { + w := newWorld(t) + id := IDOf("novox/app", head) + w.store.failing = true + w.h.PullUpdated(pr("novox/app", 7, head, "feat/x", "src/a.go")) + if _, there := w.store.deliveries[id]; there { + t.Fatal("a failing store kept it") + } + w.store.failing = false + w.h.Flush() + if _, there := w.store.deliveries[id]; !there { + t.Fatal("a delivery that could not be written was not written again") + } +} diff --git a/modules/mesh-delivery/cmd/mesh-delivery/issue341_test.go b/modules/mesh-delivery/cmd/mesh-delivery/issue341_test.go new file mode 100644 index 00000000..afdaea8a --- /dev/null +++ b/modules/mesh-delivery/cmd/mesh-delivery/issue341_test.go @@ -0,0 +1,412 @@ +package main + +import ( + "strings" + "testing" + "time" +) + +// **A verdict holds for the trunk commit it judged** (novox/hq ADR 0323, issue 341). The check judges the head +// merged into a named trunk commit; when another merge moves that trunk, a ready delivery goes back to proposed, +// a rejected one stays rejected, and nothing is asked. A verdict for an old trunk is kept and shown and does not +// count, so the table reaches a fixed point. A group's composed check goes out of date on a merge that is not one of +// its members, and runs again for new heads instead of holding the group. + +const ( + trunk0 = "a2c2c35000000000000000000000000000000000" + merge1 = "d28ca95000000000000000000000000000000000" +) + +// judged is a verdict of a head merged into a trunk commit. +func judged(repo string, number int, sha, gate, trunk string) CheckedEvent { + c := verdict(repo, number, sha, gate) + c.Trunk = shortOf(trunk, 12) + c.ID = "check-of-" + sha + "-at-" + shortOf(trunk, 8) + c.Took = "4m12s" + c.Beside = map[string]string{"mesh-catalog": "0123456789ab"} + plan := *c.Plan + plan.Head, plan.Trunk = sha, c.Trunk + c.Plan = &plan + return c +} + +func TestATrunkMoveSendsAReadyDeliveryBackToProposed(t *testing.T) { + const host = "novox/mesh-host" + const h64, h66, h67 = "6464646464646464", "dfb1415dfb1415df", "6767676767676767" + w := newWorld(t) + for _, p := range []PullEvent{pr(host, 64, h64, "a"), pr(host, 66, h66, "b"), pr(host, 67, h67, "c")} { + w.h.PullUpdated(p) + } + w.h.Checked(judged(host, 64, h64, "pass", trunk0)) + w.h.Checked(judged(host, 66, h66, "pass", trunk0)) + w.h.Checked(judged(host, 67, h67, "fail", trunk0)) + id66, id67 := IDOf(host, h66), IDOf(host, h67) + if s := w.state(id66); s != Ready { + t.Fatalf("#66 checked green against the trunk is %s", s) + } + asked := len(w.ctl.checks) + + // #64 merges: the trunk moves to its merge commit. + w.h.PullMerged(merged(host, 64, h64, merge1)) + w.settleAll() + d := w.delivery(id66) + if d.State != Proposed { + t.Fatalf("#66's verdict names trunk %s, the trunk is now %s, and it is %s", trunk0[:12], merge1[:12], d.State) + } + last := d.Transitions[len(d.Transitions)-1] + if last.Event != EvTrunkMoved || !strings.Contains(last.Why, trunk0[:12]) || !strings.Contains(last.Why, merge1[:12]) { + t.Fatalf("its transition was %s, %q: not trunk moved with both commits", last.Event, last.Why) + } + if d.Check == nil || d.Check.Trunk != trunk0[:12] { + t.Fatalf("its out-of-date verdict was not kept to be shown: %+v", d.Check) + } + if s := w.state(id67); s != Rejected { + t.Fatalf("a rejected delivery went to %s on the trunk's move", s) + } + if len(w.ctl.checks) != asked { + t.Fatalf("the trunk's move asked a check: %v", w.ctl.checks[asked:]) + } + + // Out of date and not asked: no bound, so stalled does not list it, however long it waits. + w.later(5 * time.Hour) + w.settleAll() + for _, l := range w.h.Stalled() { + if l.ID == id66 { + t.Fatalf("stalled lists a proposed delivery whose check is out of date and not asked: %+v", l) + } + } + if len(w.ctl.checks) != asked { + t.Fatalf("an out-of-date check was asked by this owner: %v", w.ctl.checks[asked:]) + } + + // A late verdict for the old trunk is kept, does not count, and moves nothing: the table's fixed point. + before := len(w.delivery(id66).Transitions) + late := judged(host, 66, h66, "pass", trunk0) + late.ID = "check-late" + w.h.Checked(late) + w.settleAll() + d = w.delivery(id66) + if d.State != Proposed || len(d.Transitions) != before || d.Check == nil || d.Check.ID != "check-late" { + t.Fatalf("a late verdict for an old trunk left it %s after %d transition(s), verdict %+v", d.State, + len(d.Transitions)-before, d.Check) + } + if ts := settle(d, Facts{Now: w.now}); len(ts) != 0 { + t.Fatalf("the table took %d more transition(s) on a verdict that does not count", len(ts)) + } + + // The check asked again judges the head merged into the trunk's head: it counts. + w.h.Checked(judged(host, 66, h66, "pass", merge1)) + w.settleAll() + if s := w.state(id66); s != Ready { + t.Fatalf("a verdict judged against the trunk's head left it %s", s) + } +} + +// A rejected delivery judged again against another trunk — the merge tool's ask — is taken as its recheck. +func TestARejectedDeliveryJudgedAgainstANewTrunkIsRecheckedByTheVerdict(t *testing.T) { + const repo = "novox/mesh-lab" + const head = "1a2b3c4d5e6f7a8b" + w := newWorld(t) + w.h.PullUpdated(pr(repo, 9, head, "x")) + w.h.Checked(judged(repo, 9, head, "fail", trunk0)) + id := IDOf(repo, head) + if s := w.state(id); s != Rejected { + t.Fatalf("it is %s", s) + } + w.h.PullMerged(merged(repo, 8, "8888888888888888", merge1)) + w.h.Checked(judged(repo, 9, head, "pass", merge1)) + w.settleAll() + d := w.delivery(id) + if d.State != Ready || d.Check.Trunk != merge1[:12] { + t.Fatalf("judged green against the trunk's head it is %s with %+v", d.State, d.Check) + } +} + +// **A verdict names the trunk commit it judged** in `checks` and `show`, and the kept plan records it (ADR 0323 §3). +func TestChecksAndShowNameTheHeadAndTheTrunkCommit(t *testing.T) { + const repo = "novox/mesh-lab" + const head = "1a2b3c4d5e6f7a8b" + w := newWorld(t) + w.h.PullUpdated(pr(repo, 9, head, "x")) + w.h.Checked(judged(repo, 9, head, "pass", trunk0)) + id := IDOf(repo, head) + + a, err := w.h.Checks(repo, 0, head) + if err != nil { + t.Fatal(err) + } + want := "head " + head[:12] + " merged into trunk " + trunk0[:12] + if a.Judged != want || a.Verdict == nil || a.Verdict.Head != head[:12] || a.Verdict.Trunk != trunk0[:12] { + t.Fatalf("checks says %q, verdict %+v; want %q", a.Judged, a.Verdict, want) + } + if !strings.Contains(a.Says, want) { + t.Fatalf("checks' words do not say what was judged: %s", a.Says) + } + if a.Verdict.Beside["mesh-catalog"] != "0123456789ab" || a.Verdict.Took != "4m12s" { + t.Fatalf("checks does not name the other repositories' commits or how long it took: %+v", a.Verdict) + } + shown, err := w.h.Show(id) + if err != nil { + t.Fatal(err) + } + d := shown.(map[string]any)["delivery"].(*Delivery) + if d.Check.Head != head[:12] || d.Check.Trunk != trunk0[:12] { + t.Fatalf("show gives the verdict %+v", d.Check) + } + if d.Plan == nil || d.Plan.Trunk != trunk0[:12] || d.Plan.Head != head { + t.Fatalf("the kept delivery plan records %+v", d.Plan) + } + + w.h.PullMerged(merged(repo, 8, "8888888888888888", merge1)) + if a, _ = w.h.Checks(repo, 0, head); !strings.Contains(a.OutOfDate, merge1[:12]) { + t.Fatalf("checks does not say the verdict is out of date: %+v", a) + } +} + +// **A delivery group's composed check follows the same rule** (ADR 0323 §6, §7): a merge that is not one of its +// members into a member's trunk puts it out of date and the group reads checking; its own members merging in its +// order do not; and it runs again once a member is judged again. +func TestAGroupsComposedCheckGoesOutOfDateOnAMergeThatIsNotItsMember(t *testing.T) { + const ctl, cat = "novox/mesh-controller", "novox/mesh-catalog" + const hc, hk = "c1c1c1c1c1c1c1c1", "k1k1k1k1k1k1k1k1" + idc, idk := IDOf(ctl, hc), IDOf(cat, hk) + start := func(t *testing.T) *world { + w := newWorld(t) + w.h.PullUpdated(pr(ctl, 1, hc, "feat/g")) + w.h.PullUpdated(pr(cat, 2, hk, "feat/g")) + w.h.Checked(judged(ctl, 1, hc, "pass", trunk0)) + w.h.Checked(judged(cat, 2, hk, "pass", trunk0)) + w.settleAll() + g := w.h.groups["feat/g"] + if g == nil || g.Check == nil { + t.Fatalf("no composed check was asked: %+v", g) + } + w.h.Checked(CheckedEvent{Group: "feat/g", ID: g.Check.Asked, Verdict: "pass", Summary: "composed", + Members: []struct { + Owner string `json:"owner"` + Repo string `json:"repo"` + Commit string `json:"commit"` + Trunk string `json:"trunk,omitempty"` + }{{Owner: "novox", Repo: "mesh-controller", Commit: hc, Trunk: trunk0[:12]}, + {Owner: "novox", Repo: "mesh-catalog", Commit: hk, Trunk: trunk0[:12]}}}) + w.settleAll() + if state, _ := GroupState(g, w.h.membersOf(g)); state != "ready" { + t.Fatalf("the group composed green is %s", state) + } + if g.Check.Trunks[idc] != trunk0[:12] { + t.Fatalf("the composed check names its members' trunks %v", g.Check.Trunks) + } + return w + } + + t.Run("a merge that is not a member", func(t *testing.T) { + w := start(t) + asked := len(w.ctl.checks) + w.h.PullMerged(merged(ctl, 5, "5555555555555555", merge1)) + w.settleAll() + g := w.h.groups["feat/g"] + state, why := GroupState(g, w.h.membersOf(g)) + if state != "checking" || g.Check.OutOfDate == "" { + t.Fatalf("the group is %s (%s) after a merge that is not its member moved a member's trunk", state, why) + } + for _, head := range []string{ctl + "@" + hc, cat + "@" + hk} { + s := w.forge.statuses[head+" "+ctxGroup] + if !strings.HasPrefix(s, "pending ") || !strings.Contains(s, "out of date") { + t.Fatalf("mesh/delivery-group on %s says %q, not pending with the reason", head, s) + } + } + // The group's Approve stays; each member whose verdict went out of date has its check asked at once, against the + // trunk's head (ADR 0421 §3, issue 1047): only the member whose trunk moved, and the composed check is asked + // after it. + if got := w.ctl.checks[asked:]; len(got) != 1 || got[0] != ":"+idc { + t.Fatalf("the trunk's move asked %v; wanted the moved member's own check once", got) + } + asked++ + // The controller's member judged again against the trunk's head: the composed check runs again. + w.later(time.Minute) + w.h.Checked(judged(ctl, 1, hc, "pass", merge1)) + w.settleAll() + if len(w.ctl.checks) == asked || !strings.HasPrefix(w.ctl.checks[len(w.ctl.checks)-1], "feat/g:") { + t.Fatalf("the composed check was not asked again once a member was judged again: %v", w.ctl.checks) + } + }) + + t.Run("its own member merging", func(t *testing.T) { + w := start(t) + mc := merged(ctl, 1, hc, merge1) + mc.Head = "feat/g" + w.h.PullMerged(mc) + w.settleAll() + g := w.h.groups["feat/g"] + if g.Check.OutOfDate != "" { + t.Fatalf("the group's own member merging put its composed check out of date: %s", g.Check.OutOfDate) + } + if s := w.state(idk); s != Ready { + t.Fatalf("the catalogue's member, in another repository, is %s", s) + } + }) +} + +// **After a member is rebased or pushed, the composed check runs again for the new heads** and the group reads +// checking, not held (ADR 0323 §6, issue 406's second symptom): its first member merged, closing the group, and then +// its second was rebased just before its own merge. +func TestARebasedMemberIsComposedAgainNotHeld(t *testing.T) { + const ctl, cat = "novox/mesh-controller", "novox/mesh-catalog" + const hc, hk, hc2 = "c1c1c1c1c1c1c1c1", "k1k1k1k1k1k1k1k1", "c2c2c2c2c2c2c2c2" + idc, idk, idc2 := IDOf(ctl, hc), IDOf(cat, hk), IDOf(ctl, hc2) + w := newWorld(t) + w.op.auto = false // the merges are heard as events below, as when the forge refused one + w.h.PullUpdated(pr(ctl, 1, hc, "feat/g")) + w.h.PullUpdated(pr(cat, 2, hk, "feat/g")) + w.h.Checked(judged(ctl, 1, hc, "pass", trunk0)) + w.h.Checked(judged(cat, 2, hk, "pass", trunk0)) + w.settleAll() + g := w.h.groups["feat/g"] + w.h.Checked(CheckedEvent{Group: "feat/g", ID: g.Check.Asked, Verdict: "pass", Summary: "composed"}) + w.settleAll() + + // The catalogue's member merges first: the group is closed. + mk := merged(cat, 2, hk, merge1) + mk.Head = "feat/g" + w.h.PullMerged(mk) + w.h.WalkMoved(w.ctl.put(aWalk("plan-cat", cat, merge1, true))) + w.settleAll() + if !g.Closed { + t.Fatal("the group is not closed once a member merged") + } + + // The controller's member is rebased just before its merge: a new head, with its own check. + w.h.PullUpdated(pr(ctl, 1, hc2, "feat/g")) + w.settleAll() + if d := w.delivery(idc2); d.Group != "feat/g" { + t.Fatalf("the rebased head did not take its place in its closed group: %q", d.Group) + } + if containsString(g.Members, idc) || !containsString(g.Members, idc2) || len(g.Members) != 2 { + t.Fatalf("the group's members are %v", g.Members) + } + if state, why := GroupState(g, w.h.membersOf(g)); state != "checking" { + t.Fatalf("the group reads %s (%s), not checking", state, why) + } + w.h.Checked(judged(ctl, 1, hc2, "pass", trunk0)) + w.settleAll() + asked := w.ctl.checks[len(w.ctl.checks)-1] + if !strings.HasPrefix(asked, "feat/g:") || !strings.Contains(asked, idc2) || !strings.Contains(asked, idk) { + t.Fatalf("the composed check was not asked again for the new heads: %v", w.ctl.checks) + } + if state, why := GroupState(g, w.h.membersOf(g)); state != "checking" { + t.Fatalf("the group reads %s (%s), not checking", state, why) + } + for _, id := range []string{idk, idc2} { + if d := w.delivery(id); d.State == Held { + t.Fatalf("%s is held for a person: %s", id, d.HeldWhy) + } + } + // Its verdict for the new heads: the rebased member merges and takes its turn. + w.h.Checked(CheckedEvent{Group: "feat/g", ID: g.Check.Asked, Verdict: "pass", Summary: "composed again"}) + mc := merged(ctl, 1, hc2, "e0e0e0e0e0e0e0e0") + mc.Head = "feat/g" + w.h.PullMerged(mc) + w.h.WalkMoved(w.ctl.put(aWalk("plan-ctl", ctl, "e0e0e0e0e0e0e0e0", true))) + w.settleAll() + if d := w.delivery(idc2); d.State == Held { + t.Fatalf("the rebased member, composed green for the new heads, is held: %s", d.HeldWhy) + } +} + +// A head whose merge check the forge's holder put out of date is not a check asked and not answered: `stalled` does +// not list it, however long it waits — rejected, or with no delivery here — until its check is asked (ADR 0323 §7). +func TestAnOutOfDateCheckOnTheForgeIsNotSaidUnanswered(t *testing.T) { + const repo = "novox/mesh-lab" + const head = "1a2b3c4d5e6f7a8b9c0d" + w := newWorld(t) + w.h.PullUpdated(pr(repo, 9, head, "x")) + w.h.Checked(judged(repo, 9, head, "fail", trunk0)) + words := outOfDateWords(trunk0, merge1) + pending := func(description, at string) map[string]HeadCheck { + return map[string]HeadCheck{ctxGate: {State: "pending", Description: description, UpdatedAt: at}, + ctxRepo: {State: "pending", Description: description, UpdatedAt: at}} + } + w.forge.open = []OpenPull{{Owner: "novox", Repo: "mesh-lab", Number: 9, Base: "main", HeadSHA: head, + Required: []string{ctxGate, ctxRepo}, Statuses: []string{ctxGate, ctxRepo}, Checks: pending(words, "2026-10-06T12:01:00Z")}} + w.h.SeeOpenPulls() + w.later(3 * time.Hour) + w.h.SeeOpenPulls() + for _, l := range w.h.Stalled() { + if strings.HasPrefix(l.ID, repo) { + t.Fatalf("an out-of-date check nobody asked is said stalled: %+v", l) + } + } + // Asked by the merge tool: pending anew, and its bound runs from then. + w.forge.open[0].Checks = pending("waiting in the build queue since 15:01: asked by the merge tool", "2026-10-06T15:01:00Z") + w.h.SeeOpenPulls() + w.later(2 * time.Hour) + w.h.SeeOpenPulls() + said := false + for _, l := range w.h.Stalled() { + said = said || l.State == Unanswered + } + if !said { + t.Fatal("a check the merge tool asked and that never answered is not said") + } +} + +// The merge tool's ask announces an out-of-date head again: its one-hour bound runs from then, and a check that never +// answers is said stalled (ADR 0323 §7). +func TestAnOutOfDateDeliveryAskedAgainIsBoundFromTheAsk(t *testing.T) { + const repo = "novox/mesh-lab" + const head = "1a2b3c4d5e6f7a8b" + w := newWorld(t) + w.h.PullUpdated(pr(repo, 9, head, "x")) + w.h.Checked(judged(repo, 9, head, "pass", trunk0)) + w.h.PullMerged(merged(repo, 8, "8888888888888888", merge1)) + id := IDOf(repo, head) + w.later(3 * time.Hour) + w.h.PullUpdated(pr(repo, 9, head, "x")) // the merge tool asked its check + w.later(30 * time.Minute) + for _, l := range w.h.Stalled() { + if l.ID == id { + t.Fatalf("listed half an hour after its check was asked again: %+v", l) + } + } + w.later(time.Hour) + said := false + for _, l := range w.h.Stalled() { + if l.ID == id { + said = true + if !strings.Contains(l.Says, "asked again") || !strings.Contains(l.H2, "operator") { + t.Fatalf("its stalled line says %+v", l) + } + } + } + if !said { + t.Fatal("a check asked again that never answered is not said") + } + // Its verdict judged against the trunk's head: ready, and nothing bounds it as proposed. + w.h.Checked(judged(repo, 9, head, "pass", merge1)) + if d := w.delivery(id); d.State != Ready || d.AskedAgain != nil { + t.Fatalf("judged again it is %s, asked again %v", d.State, d.AskedAgain) + } +} + +// A check this owner asks — a recheck — sends the base head the forge last announced the head with, so the +// controller's verdict for a head no build judges names its trunk commit (ADR 0323 §3, review of mesh-controller#271). +func TestARecheckSendsTheAnnouncedBaseHead(t *testing.T) { + const repo = "novox/mesh-tools-go" + const head = "1a2b3c4d5e6f7a8b" + w := newWorld(t) + p := pr(repo, 3, head, "x") + p.BaseSHA = trunk0 + w.h.PullUpdated(p) + id := IDOf(repo, head) + if m := MemberOf(w.delivery(id)); m.BaseSHA != trunk0 { + t.Fatalf("the member sends base head %q", m.BaseSHA) + } + p.BaseSHA = merge1 // announced again, by the merge tool + w.h.PullUpdated(p) + if m := MemberOf(w.delivery(id)); m.BaseSHA != merge1 { + t.Fatalf("announced again, the member sends base head %q", m.BaseSHA) + } + if raw := string(mustJSON(MemberOf(w.delivery(id)))); !strings.Contains(raw, `"base_sha":"`+merge1+`"`) { + t.Fatalf("the member as the controller reads it: %s", raw) + } +} diff --git a/modules/mesh-delivery/cmd/mesh-delivery/issue362_test.go b/modules/mesh-delivery/cmd/mesh-delivery/issue362_test.go new file mode 100644 index 00000000..c37f72f3 --- /dev/null +++ b/modules/mesh-delivery/cmd/mesh-delivery/issue362_test.go @@ -0,0 +1,434 @@ +package main + +import ( + "strings" + "testing" + "time" +) + +// novox/hq issue 362, ADR 0276: two merges of one repository 18 seconds apart, walked by one walk at the later +// merge's commit, which superseded the first. Every delivery the walk answers follows it and ends with it; no +// walk is tied to a delivery that ended; and what cannot move is said at once. + +// A controller that names the merges a walk answers (ADR 0276): the forge is not asked, and the earlier merge is +// tied at once. +func TestAWalkThatNamesItsMergesIsFollowedByEachOfThem(t *testing.T) { + w := twoMerges(t) + walk := catWalk("plan-9", mergeDu, atDu.Add(5*time.Second), "claude-code", "dunst") + walk.Delivery.Merges = []WalkMerge{{Repository: cat, Commit: mergeCC, Carried: mergeDu}, {Repository: cat, Commit: mergeDu}} + w.h.WalkMoved(w.ctl.put(walk)) + for _, id := range []string{IDOf(cat, headCC), IDOf(cat, headDu)} { + if d := w.delivery(id); d.State != Published || d.Walk == nil || d.Walk.ID != "plan-9" { + t.Fatalf("%s is %s on %v", id, d.State, d.Walk) + } + } + w.settleAll() + if len(w.forge.containsAsks) != 0 { + t.Fatalf("the forge was asked %v for a walk that names its merges", w.forge.containsAsks) + } + if len(w.ctl.delivers) != 1 { + t.Fatalf("the word was asked %v", w.ctl.delivers) + } +} + +// A walk that failed marks no merge delivered: the merge it carried its own commit for fails, and the earlier +// merge it carried a later commit for waits for a walk of its own. +func TestAFailedWalkMarksNoMergeItCarriedDelivered(t *testing.T) { + w := twoMerges(t) + walk := catWalk("plan-9", mergeDu, atDu.Add(5*time.Second), "claude-code", "dunst") + walk.Delivery.Merges = []WalkMerge{{Repository: cat, Commit: mergeCC, Carried: mergeDu}, {Repository: cat, Commit: mergeDu}} + w.h.WalkMoved(w.ctl.put(walk)) + w.settleAll() + walk = w.ctl.walk("plan-9") + walk.State, walk.Note = walkFailed, "dunst failed its gate on anchor" + w.h.WalkMoved(w.ctl.put(walk)) + w.settleAll() + if s := w.state(IDOf(cat, headDu)); s != Failed { + t.Fatalf("dunst is %s", s) + } + cc := w.delivery(IDOf(cat, headCC)) + if cc.State != Published || cc.State.Reached() { + t.Fatalf("claude-code is %s", cc.State) + } + // Its own walk, when the controller opens one, is followed. + own := catWalk("plan-10", mergeCC, atDu.Add(time.Hour), "claude-code") + w.h.WalkMoved(w.ctl.put(own)) + w.settleAll() + w.h.WalkMoved(w.ctl.put(done(w.ctl.walk("plan-10"), w.now))) + w.settleAll() + if s := w.state(IDOf(cat, headCC)); s != Delivered { + t.Fatalf("claude-code on its own walk is %s", s) + } +} + +// Never revived: a walk at the commit of a delivery that ended is not tied to it, not adopted, and not let go — +// it is listed by `stalled` as a walk no live delivery holds. +func TestAWalkIsNeverTiedToADeliveryThatEnded(t *testing.T) { + w := twoMerges(t) + first := w.ctl.put(catWalk("plan-1", mergeDu, atDu.Add(5*time.Second), "dunst")) + w.h.WalkMoved(first) + w.h.WalkMoved(w.ctl.put(supersededBy(first, ""))) + // A record from before ADR 0276: superseded, naming no walk — the delivery ends superseded. + sup := w.ctl.walk("plan-1") + sup.Note = "superseded" + w.h.WalkMoved(w.ctl.put(sup)) + idDu := IDOf(cat, headDu) + if s := w.state(idDu); s != Superseded { + t.Fatalf("dunst is %s", s) + } + orphan := catWalk("plan-2", mergeDu, atDu.Add(6*time.Second), "dunst") + w.h.WalkMoved(w.ctl.put(orphan)) + w.settleAll() + w.later(3 * time.Minute) + w.settleAll() + if d := w.delivery(idDu); d.State != Superseded || d.Walk.ID != "plan-1" { + t.Fatalf("the ended delivery is %s on %s", d.State, d.Walk.ID) + } + for _, asked := range w.ctl.delivers { + if asked == "plan-2" { + t.Fatal("a walk no live delivery holds was let go") + } + } + var found bool + for _, l := range w.h.Stalled() { + if l.State == Unheld && l.Walk == "plan-2" { + found = true + } + } + if !found { + t.Fatalf("stalled does not list the walk no delivery holds: %+v", w.h.Stalled()) + } +} + +// A merge no walk answers is held after ten minutes with the true reason, and listed by `stalled` at once; a walk +// that answers it later lifts it. +func TestAMergeNoWalkAnswersIsHeldTrulyAndSaidAtOnce(t *testing.T) { + w := newWorldOver(t, newMemStore(), newFakeController(), newFakeForge(), atCC) + cc := pr(cat, 101, headCC, "feat/claude-code", "modules/claude-code/module.json") + w.h.PullUpdated(cc) + w.h.Checked(verdictMoving(cat, 101, headCC, "claude-code")) + w.h.PullMerged(mergedAt(cc, mergeCC, atCC)) + // An open walk of the repository that moves something else: it does not answer this merge. + w.h.WalkMoved(w.ctl.put(catWalk("plan-5", "eeeeeeeeeeee5555", atCC.Add(time.Minute), "dunst"))) + w.later(11 * time.Minute) + if err := w.h.Reconcile(); err != nil { + t.Fatal(err) + } + w.settleAll() + id := IDOf(cat, headCC) + d := w.delivery(id) + if d.State != Held || !strings.Contains(d.HeldWhy, "plan-5") || !strings.Contains(d.HeldWhy, "name no merge") { + t.Fatalf("it is %s: %q", d.State, d.HeldWhy) + } + lines := w.h.Stalled() + if len(lines) != 1 || lines[0].ID != id || lines[0].Bound != "0s" { + t.Fatalf("stalled lists %+v", lines) + } + // The controller opens a walk that names it. + walk := catWalk("plan-6", mergeCC, w.now, "claude-code") + w.h.WalkMoved(w.ctl.put(walk)) + w.settleAll() + if s := w.state(id); s != Delivering || len(w.ctl.delivers) != 1 || w.ctl.delivers[0] != "plan-6" { + t.Fatalf("with its walk it is %s, let go %v", s, w.ctl.delivers) + } +} + +// One word per walk, and none while a delivery it carries is not at its turn. +func TestAWalkIsLetGoOnlyWhenEveryDeliveryItCarriesIsAtItsTurn(t *testing.T) { + w := newWorldOver(t, newMemStore(), newFakeController(), newFakeForge(), atDu.Add(5*time.Second)) + cc := pr(cat, 101, headCC, "feat/claude-code", "modules/claude-code/module.json") + du := pr(cat, 102, headDu, "feat/dunst", "modules/dunst/module.json") + w.h.PullUpdated(cc) + w.h.PullUpdated(du) + w.h.Checked(verdictMoving(cat, 101, headCC, "claude-code")) + // dunst merges unchecked: held for a person. + w.h.PullMerged(mergedAt(cc, mergeCC, atCC)) + w.h.PullMerged(mergedAt(du, mergeDu, atDu)) + walk := catWalk("plan-9", mergeDu, atDu.Add(5*time.Second), "claude-code", "dunst") + walk.Delivery.Merges = []WalkMerge{{Repository: cat, Commit: mergeCC, Carried: mergeDu}, {Repository: cat, Commit: mergeDu}} + w.h.WalkMoved(w.ctl.put(walk)) + w.settleAll() + if s := w.state(IDOf(cat, headDu)); s != Held { + t.Fatalf("dunst is %s", s) + } + if len(w.ctl.delivers) != 0 { + t.Fatalf("the walk was let go %v while it carries a held delivery", w.ctl.delivers) + } + if _, err := w.h.Release(IDOf(cat, headDu), "checked by hand", "jochen"); err != nil { + t.Fatal(err) + } + w.settleAll() + if len(w.ctl.delivers) != 1 { + t.Fatalf("released, the word was asked %v", w.ctl.delivers) + } +} + +// A merge in a batch still assembling is said so, is not held for a person, and nothing is let go. +func TestAMergeInAnAssemblingBatchIsShownAndNotHeld(t *testing.T) { + w := newWorldOver(t, newMemStore(), newFakeController(), newFakeForge(), atCC) + cc := pr(cat, 101, headCC, "feat/claude-code", "modules/claude-code/module.json") + w.h.PullUpdated(cc) + w.h.Checked(verdictMoving(cat, 101, headCC, "claude-code")) + w.h.PullMerged(mergedAt(cc, mergeCC, atCC)) + closes, atMost := atCC.Add(90*time.Second), atCC.Add(10*time.Minute) + batch := Walk{ID: "plan-b1", Repository: cat, Branch: "main", Commit: mergeCC, Created: atCC, State: walkAssembling, + Delivery: &WalkDelivery{Awaits: "mesh-delivery", Merges: []WalkMerge{{Repository: cat, Commit: mergeCC}}, + Batch: &WalkBatch{ClosesAt: &closes, AtMost: &atMost}}} + w.h.WalkMoved(w.ctl.put(batch)) + w.later(48 * time.Second) + w.settleAll() + id := IDOf(cat, headCC) + var line Line + for _, l := range w.h.Deliveries("", "", "", true) { + if l.ID == id { + line = l + } + } + if !strings.Contains(line.Waits, "plan-b1: assembling, 42s left") || !strings.Contains(line.Waits, "plan not yet calculated") { + t.Fatalf("it waits for %q", line.Waits) + } + // Queued behind a walk for longer than a walk opens within: still not held. + batch = w.ctl.walk("plan-b1") + batch.State, batch.Delivery.Batch.Behind = walkQueued, "plan-7" + w.h.WalkMoved(w.ctl.put(batch)) + w.later(20 * time.Minute) + if err := w.h.Reconcile(); err != nil { + t.Fatal(err) + } + w.settleAll() + if d := w.delivery(id); d.State != Ready || d.Walk != nil { + t.Fatalf("a merge in a queued batch is %s on %v", d.State, d.Walk) + } + if len(w.ctl.delivers) != 0 { + t.Fatalf("a batch was let go: %v", w.ctl.delivers) + } + // Cut: the batch is a walk now, under its id. + batch.State, batch.Delivery.Batch = walkBuilding, nil + batch.Tiers, batch.Modules = [][]string{{"claude-code"}}, map[string]*WalkModule{"claude-code": {}} + w.h.WalkMoved(w.ctl.put(batch)) + w.settleAll() + if d := w.delivery(id); d.State != Delivering || d.Walk.ID != "plan-b1" { + t.Fatalf("cut, it is %s on %v", d.State, d.Walk) + } +} + +// An older record of the walk a delivery no longer follows, arriving late, does not take it back. +func TestALateRecordOfAnOlderWalkDoesNotTakeADeliveryBack(t *testing.T) { + w := twoMerges(t) + w.forge.contains[mergeCC+" in "+mergeDu] = true + first := w.ctl.put(catWalk("plan-1", mergeDu, atDu.Add(5*time.Second), "dunst")) + w.h.WalkMoved(first) + w.h.WalkMoved(w.ctl.put(catWalk("plan-2", mergeDu, atDu.Add(6*time.Second), "claude-code", "dunst"))) + w.h.WalkMoved(first) // the older record, again + if d := w.delivery(IDOf(cat, headDu)); d.Walk.ID != "plan-2" { + t.Fatalf("dunst went back to %s", d.Walk.ID) + } +} + +// A walk no merge was heard for, at the commit of a delivery that ended, is not adopted by that delivery: it is +// listed by `stalled` as a walk no live delivery holds. +func TestAnUnheardWalkIsNotAdoptedByADeliveryThatEnded(t *testing.T) { + w := newWorldOver(t, newMemStore(), newFakeController(), newFakeForge(), atCC) + cc := pr(cat, 101, headCC, "feat/claude-code", "modules/claude-code/module.json") + w.h.PullUpdated(cc) + w.h.PullClosed(cc) + id := IDOf(cat, headCC) + if s := w.state(id); s != Stopped { + t.Fatalf("closed unmerged it is %s", s) + } + w.h.WalkMoved(w.ctl.put(catWalk("plan-3", headCC, atCC.Add(time.Minute), "claude-code"))) + w.later(3 * time.Minute) + w.settleAll() + w.later(3 * time.Minute) + w.settleAll() + if d := w.delivery(id); d.State != Stopped || d.Walk != nil { + t.Fatalf("the stopped delivery is %s on %v", d.State, d.Walk) + } + var found bool + for _, l := range w.h.Stalled() { + found = found || l.State == Unheld && l.Walk == "plan-3" + } + if !found || len(w.ctl.delivers) != 0 { + t.Fatalf("stalled lists %+v; let go %v", w.h.Stalled(), w.ctl.delivers) + } +} + +// Two members of one delivery group carried by one walk: let go once, for both (review of #171: the group's turn +// stopped at its first member, and the walk waited for the second for ever). +func TestAWalkCarryingTwoMembersOfAGroupIsLetGoOnce(t *testing.T) { + w := newWorldOver(t, newMemStore(), newFakeController(), newFakeForge(), atDu.Add(5*time.Second)) + a := pr("novox/a", 1, "aaaaaaaaaaaa0001", "feat/y", "x.go") + b := pr("novox/b", 2, "bbbbbbbbbbbb0002", "feat/y", "y.go") + w.h.PullUpdated(a) + w.h.PullUpdated(b) + w.h.Checked(verdictMoving("novox/a", 1, "aaaaaaaaaaaa0001", "app-a")) + w.h.Checked(verdictMoving("novox/b", 2, "bbbbbbbbbbbb0002", "app-b")) + w.settleAll() + g := w.h.groups["feat/y"] + if g == nil || g.Check == nil { + t.Fatalf("the group is %+v", g) + } + w.h.Checked(CheckedEvent{Group: "feat/y", ID: g.Check.Asked, Verdict: "pass", Summary: "composed"}) + w.h.PullMerged(mergedAt(a, "aaaaaaaaaaaa1111", atCC)) + w.h.PullMerged(mergedAt(b, "bbbbbbbbbbbb2222", atDu)) + batch := Walk{ID: "plan-batch", Repository: "novox/a", Branch: "main", Commit: "aaaaaaaaaaaa1111", + Created: atDu.Add(time.Minute), State: walkBuilding, Tiers: [][]string{{"app-a"}, {"app-b"}}, + Modules: map[string]*WalkModule{"app-a": {}, "app-b": {}}, + Delivery: &WalkDelivery{Awaits: "mesh-delivery", Merges: []WalkMerge{ + {Repository: "novox/a", Commit: "aaaaaaaaaaaa1111"}, {Repository: "novox/b", Commit: "bbbbbbbbbbbb2222"}}}} + w.h.WalkMoved(w.ctl.put(batch)) + w.settleAll() + if got := w.ctl.delivers; len(got) != 1 || got[0] != "plan-batch" { + t.Fatalf("the word was asked %v, want once for plan-batch", got) + } + for _, id := range []string{IDOf("novox/a", "aaaaaaaaaaaa0001"), IDOf("novox/b", "bbbbbbbbbbbb0002")} { + if s := w.state(id); s != Delivering { + t.Fatalf("%s is %s", id, s) + } + } +} + +// A walk that names its merges answers only those: a delivery whose walk it took over and which it does not name +// is not tied to it, and ends superseded once that is known. +func TestAWalkThatNamesItsMergesAnswersOnlyThose(t *testing.T) { + w := twoMerges(t) + first := w.ctl.put(catWalk("plan-1", mergeDu, atDu.Add(5*time.Second), "dunst")) + w.h.WalkMoved(first) + other := catWalk("plan-2", "ffffffffffff9999", atDu.Add(6*time.Second), "dunst") + other.Delivery.Merges = []WalkMerge{{Repository: cat, Commit: "ffffffffffff9999"}} + w.h.WalkMoved(w.ctl.put(other)) + w.h.WalkMoved(w.ctl.put(supersededBy(first, "plan-2"))) + w.settleAll() + d := w.delivery(IDOf(cat, headDu)) + if d.State != Superseded || d.Walk.ID != "plan-1" { + t.Fatalf("dunst is %s on %s", d.State, d.Walk.ID) + } + for _, asked := range w.ctl.delivers { + if asked == "plan-2" { + t.Fatal("a walk that does not name dunst was let go for it") + } + } +} + +// The forge's containment ties a merge to a walk only when the walk moves everything the merge moves and was made +// for a later merge. +func TestContainmentAloneDoesNotTieAMerge(t *testing.T) { + for _, c := range []struct { + name string + modules []string + created time.Time + names bool + }{ + {"the later walk moves only its own module", []string{"dunst"}, atDu.Add(5 * time.Second), false}, + {"the walk was made for an earlier merge", []string{"claude-code", "dunst"}, atCC.Add(-time.Minute), false}, + {"the walk names its merges, and not this one", []string{"claude-code", "dunst"}, atDu.Add(5 * time.Second), true}, + } { + t.Run(c.name, func(t *testing.T) { + w := newWorldOver(t, newMemStore(), newFakeController(), newFakeForge(), atDu.Add(5*time.Second)) + cc := pr(cat, 101, headCC, "feat/claude-code", "modules/claude-code/module.json") + w.h.PullUpdated(cc) + w.h.Checked(verdictMoving(cat, 101, headCC, "claude-code")) + w.h.PullMerged(mergedAt(cc, mergeCC, atCC)) + w.forge.contains[mergeCC+" in "+mergeDu] = true + walk := catWalk("plan-2", mergeDu, c.created, c.modules...) + if c.names { + walk.Delivery.Merges = []WalkMerge{{Repository: cat, Commit: mergeDu}} + } + w.h.WalkMoved(w.ctl.put(walk)) + w.settleAll() + if d := w.delivery(IDOf(cat, headCC)); d.Walk != nil || len(w.forge.containsAsks) != 0 { + t.Fatalf("it follows %v; the forge was asked %v", d.Walk, w.forge.containsAsks) + } + }) + } +} + +// After the walk that carried a later commit for it failed, a merge stays published — once — and is said within +// the ten minutes a walk opens within, naming what happened. +func TestAMergeWhoseCarryingWalkFailedWaitsOnceAndIsSaid(t *testing.T) { + w := twoMerges(t) + walk := catWalk("plan-9", mergeDu, atDu.Add(5*time.Second), "claude-code", "dunst") + walk.Delivery.Merges = []WalkMerge{{Repository: cat, Commit: mergeCC, Carried: mergeDu}, {Repository: cat, Commit: mergeDu}} + w.h.WalkMoved(w.ctl.put(walk)) + w.settleAll() + walk = w.ctl.walk("plan-9") + walk.State = walkFailed + w.h.WalkMoved(w.ctl.put(walk)) + w.settleAll() + id := IDOf(cat, headCC) + d := w.delivery(id) + rewalks := 0 + for _, tr := range d.Transitions { + if tr.Event == EvRewalk { + rewalks++ + } + } + if d.State != Published || rewalks != 1 { + t.Fatalf("it is %s after %d rewalk(s)", d.State, rewalks) + } + w.later(11 * time.Minute) + var line *StalledLine + for _, l := range w.h.Stalled() { + if l.ID == id { + l := l + line = &l + } + } + if line == nil || !strings.Contains(line.Says, "no walk of its own commit") { + t.Fatalf("stalled says %+v", line) + } +} + +// A delivery that ended is never tied to a walk after: not one that answers its merge, not the walk that took its +// own over. +func TestADeliveryThatEndedFollowsNoWalk(t *testing.T) { + w := newWorldOver(t, newMemStore(), newFakeController(), newFakeForge(), atCC) + w.ctl.planned = planMoving(cat) + p := pr(cat, 103, "eeeeeeeeeeee0003", "docs/x", "README.md") + w.h.PullUpdated(p) + w.h.PullMerged(mergedAt(p, "eeeeeeeeeeee3333", atCC)) + w.settleAll() + nothing := IDOf(cat, "eeeeeeeeeeee0003") + if s := w.state(nothing); s != Delivered { + t.Fatalf("a merge moving nothing is %s", s) + } + w.h.WalkMoved(w.ctl.put(catWalk("plan-4", "eeeeeeeeeeee3333", atCC.Add(time.Minute), "app"))) + w.settleAll() + if d := w.delivery(nothing); d.Walk != nil { + t.Fatalf("a delivered merge was tied to %s", d.Walk.ID) + } + + w2 := twoMerges(t) + first := w2.ctl.put(catWalk("plan-1", mergeDu, atDu.Add(5*time.Second), "dunst")) + w2.h.WalkMoved(first) + idDu := IDOf(cat, headDu) + if _, err := w2.h.Stop(idDu, "not now", "jochen"); err != nil { + t.Fatal(err) + } + w2.ctl.put(catWalk("plan-2", mergeDu, atDu.Add(6*time.Second), "claude-code", "dunst")) + w2.h.WalkMoved(w2.ctl.put(supersededBy(w2.ctl.walk("plan-1"), "plan-2"))) + w2.settleAll() + if d := w2.delivery(idDu); d.State != Stopped || d.Walk.ID != "plan-1" { + t.Fatalf("the stopped delivery is %s on %s", d.State, d.Walk.ID) + } +} + +// Today's controller, merges A then B minutes apart: B's walk took A's over before it started. A follows it and +// ends covered by B, rather than superseded (ADR 0276 decision 4). +func TestAnEarlierMergeFollowsTheWalkThatTookItsOver(t *testing.T) { + w := twoMerges(t) + first := w.ctl.put(catWalk("plan-1", mergeCC, atCC.Add(5*time.Second), "claude-code")) + w.h.WalkMoved(first) + w.h.WalkMoved(w.ctl.put(catWalk("plan-2", mergeDu, atDu.Add(6*time.Second), "claude-code", "dunst"))) + w.h.WalkMoved(w.ctl.put(supersededBy(w.ctl.walk("plan-1"), "plan-2"))) + w.settleAll() + idCC := IDOf(cat, headCC) + if d := w.delivery(idCC); d.Walk.ID != "plan-2" || len(w.ctl.delivers) != 1 { + t.Fatalf("claude-code follows %s; let go %v", d.Walk.ID, w.ctl.delivers) + } + w.h.WalkMoved(w.ctl.put(done(w.ctl.walk("plan-2"), w.now))) + w.settleAll() + if s := w.state(idCC); s != Covered { + t.Fatalf("claude-code is %s", s) + } +} diff --git a/modules/mesh-delivery/cmd/mesh-delivery/issue403_test.go b/modules/mesh-delivery/cmd/mesh-delivery/issue403_test.go new file mode 100644 index 00000000..f7ae4bf8 --- /dev/null +++ b/modules/mesh-delivery/cmd/mesh-delivery/issue403_test.go @@ -0,0 +1,95 @@ +package main + +import ( + "testing" + "time" +) + +// novox/hq issue 403, ADR 0313: the issue tracker measures a fix's time from the move to fixing to delivered and +// splits it at the pull request's opening and its merge. A transition says what it reads them by: when the pull +// request was first heard (its first head, not a later one), when it merged, and the class of what it moves — +// core, leaf, or none for a change that moves no module. `checks` says the same for a fix the tracker hears of late. + +// transitionsTo are the transition events said for a delivery that moved to a state. +func transitionsTo(w *world, id string, to State) []map[string]any { + var out []map[string]any + for _, e := range w.said { + if e["event-kind"] == "transition" && e["id"] == id && e["to"] == string(to) { + out = append(out, e) + } + } + return out +} + +func TestATransitionSaysWhenItsPullRequestOpenedMergedAndItsClass(t *testing.T) { + w := newWorld(t) + repo := "novox/mesh-catalog" + opened := w.now + first := pr(repo, 9, "head9first", "fix/403-x", "modules/app/a.go") + w.h.PullUpdated(first) + w.later(3 * time.Minute) + second := pr(repo, 9, "head9second", "fix/403-x", "modules/app/a.go") + w.h.PullUpdated(second) + c := verdictMoving(repo, 9, "head9second", "app") + c.Plan.Class = "core" + w.h.Checked(c) + w.later(2 * time.Minute) + mergedWhen := w.now + w.h.PullMerged(mergedAt(second, "merge9aaaaaa", mergedWhen)) + walk := aWalk("plan-merge9", repo, "merge9aaaaaa", false) + walk.Delivery = &WalkDelivery{Merges: []WalkMerge{{Repository: repo, Commit: "merge9aaaaaa"}}} + w.h.WalkMoved(w.ctl.put(walk)) + w.settleAll() + walk = w.ctl.walk(walk.ID) + walk.State, walk.Revision = walkDone, walk.Revision+1 + walk.Modules = map[string]*WalkModule{"app": {State: "built"}} + w.h.WalkMoved(w.ctl.put(walk)) + w.settleAll() + + id := IDOf(repo, "head9second") + if s := w.state(id); s != Delivered { + t.Fatalf("the merge is %s, not delivered", s) + } + said := transitionsTo(w, id, Delivered) + if len(said) == 0 { + t.Fatalf("no transition to delivered was said: %v", w.said) + } + e := said[len(said)-1] + if e["opened"] != opened.UTC().Format(time.RFC3339) { + t.Errorf("opened %v, want the first head's %s (not the second head's)", e["opened"], opened.Format(time.RFC3339)) + } + if e["merged_at"] != mergedWhen.UTC().Format(time.RFC3339) { + t.Errorf("merged_at %v, want %s", e["merged_at"], mergedWhen.Format(time.RFC3339)) + } + if e["class"] != "core" { + t.Errorf("class %v, want the plan's core", e["class"]) + } + + a, err := w.h.Checks(repo, 9, "") + if err != nil { + t.Fatal(err) + } + if a.Delivery == nil || a.Delivery.Opened != opened.UTC().Format(time.RFC3339) || a.Delivery.Class != "core" || + a.Delivery.MergedAt != mergedWhen.UTC().Format(time.RFC3339) { + t.Errorf("checks says %+v", a.Delivery) + } +} + +// A plan that moves no module is class none, so the tracker gives its fix no budget; a plan from a controller that +// says no class falls back to its walk's class once known, and says nothing before. +func TestAFixClassIsNoneWhenItMovesNothing(t *testing.T) { + for _, c := range []struct { + why string + d Delivery + want string + }{ + {"the planner's class", Delivery{Plan: &DeliveryPlan{Moved: []string{"app"}, Class: "leaf"}}, "leaf"}, + {"a plan that moves nothing, no class said", Delivery{Plan: planMoving("novox/x")}, "none"}, + {"no class said, its walk's known", Delivery{Plan: planMoving("novox/x", "app"), Time: &DeliveryTime{Class: "core"}}, "core"}, + {"no class said, nothing known", Delivery{Plan: planMoving("novox/x", "app")}, ""}, + } { + if got := fixClass(&c.d); got != c.want { + t.Errorf("%s: %q, want %q", c.why, got, c.want) + } + } +} diff --git a/modules/mesh-delivery/cmd/mesh-delivery/issue407_test.go b/modules/mesh-delivery/cmd/mesh-delivery/issue407_test.go new file mode 100644 index 00000000..551744ab --- /dev/null +++ b/modules/mesh-delivery/cmd/mesh-delivery/issue407_test.go @@ -0,0 +1,105 @@ +package main + +import ( + "testing" + "time" +) + +// triedSaid is delivery_status's count of owed effects tried (novox/hq issue 407), read as a test reads it. +func triedSaid(t *testing.T, w *world) map[string]any { + t.Helper() + got, ok := w.h.Status("listening")["owed-tried"].(map[string]any) + if !ok { + t.Fatalf("delivery_status says no owed-tried: %v", w.h.Status("listening")) + } + return got +} + +func countOf(t *testing.T, m map[string]any, key string) int { + t.Helper() + n, ok := m[key].(int) + if !ok { + t.Fatalf("owed-tried has no count %q: %v", key, m) + } + return n +} + +// novox/hq issue 407: delivery_status says how many owed effects were tried since this owner started, how +// many were done, failed to be tried again, or were given up, and when the last was tried — so a window's +// writes can be checked against the tries made in it (issue 326's rule) by reading it at both ends. +func TestDeliveryStatusCountsTheOwedEffectsTried(t *testing.T) { + w := newWorld(t) + start := w.now + tried := triedSaid(t, w) + if countOf(t, tried, "tried") != 0 || countOf(t, tried, "done") != 0 || countOf(t, tried, "failed") != 0 || + countOf(t, tried, "given-up") != 0 { + t.Fatalf("a holder that tried nothing says %v", tried) + } + if tried["since"] != start.Format(time.RFC3339) { + t.Fatalf("its count is since %v, the holder started %s", tried["since"], start.Format(time.RFC3339)) + } + if _, there := tried["last"]; there { + t.Fatalf("nothing tried, yet a last try is said: %v", tried) + } + + // A head seen: its event and its note are owed and done. + w.h.PullUpdated(pr("novox/app", 7, head, "feat/x", "src/a.go")) + w.settleAll() + tried = triedSaid(t, w) + done := countOf(t, tried, "done") + if done < len(w.said)+len(w.forge.notesOn(head)) || done == 0 { + t.Fatalf("said %d and noted %d, yet done counts %d", len(w.said), len(w.forge.notesOn(head)), done) + } + if countOf(t, tried, "tried") != done || countOf(t, tried, "failed") != 0 { + t.Fatalf("every try succeeded, yet %v", tried) + } + if tried["last"] != w.now.Format(time.RFC3339) { + t.Fatalf("last try said %v, tried at %s", tried["last"], w.now.Format(time.RFC3339)) + } + + // The forge away: its effects fail and are counted failed; quiet passes try nothing more. + w.forge.down = true + w.later(time.Minute) + w.h.Checked(verdict("novox/app", 7, head, "pass")) + w.h.Flush() + tried = triedSaid(t, w) + failed := countOf(t, tried, "failed") + if failed == 0 { + t.Fatalf("the forge away failed nothing: %v", tried) + } + if countOf(t, tried, "tried") != countOf(t, tried, "done")+failed+countOf(t, tried, "given-up") { + t.Fatalf("tries do not add up: %v", tried) + } + if tried["last"] != w.now.Format(time.RFC3339) { + t.Fatalf("last try said %v, tried at %s", tried["last"], w.now.Format(time.RFC3339)) + } + before := countOf(t, tried, "tried") + w.h.Flush() // before each failure's next try: nothing is tried + if n := countOf(t, triedSaid(t, w), "tried"); n != before { + t.Fatalf("a pass before any next try tried %d", n-before) + } + + // The forge back, past every next try: the failed effects are done. + w.forge.down = false + w.later(time.Hour) + w.h.Flush() + tried = triedSaid(t, w) + if countOf(t, tried, "tried") != before+failed || countOf(t, tried, "done") <= done { + t.Fatalf("the forge back: tried %d (was %d, %d failed), %v", countOf(t, tried, "tried"), before, failed, tried) + } +} + +// The count is this run's: a restarted holder says it counts since its own start, from nothing. +func TestOwedTriedCountsSinceTheStart(t *testing.T) { + w := newWorld(t) + w.h.PullUpdated(pr("novox/app", 7, head, "feat/x", "src/a.go")) + w.settleAll() + if countOf(t, triedSaid(t, w), "tried") == 0 { + t.Fatal("nothing counted before the restart") + } + again := newWorldOver(t, w.store, w.ctl, w.forge, w.now.Add(time.Minute)) + tried := triedSaid(t, again) + if countOf(t, tried, "tried") != 0 || tried["since"] != again.now.Format(time.RFC3339) { + t.Fatalf("a restarted holder says %v", tried) + } +} diff --git a/modules/mesh-delivery/cmd/mesh-delivery/issue438_test.go b/modules/mesh-delivery/cmd/mesh-delivery/issue438_test.go new file mode 100644 index 00000000..9895e6fd --- /dev/null +++ b/modules/mesh-delivery/cmd/mesh-delivery/issue438_test.go @@ -0,0 +1,183 @@ +package main + +import ( + "encoding/json" + "slices" + "strings" + "testing" + "time" +) + +// openPullsOf reads gitea_open_pulls' answer as the forge module gives it, so the tests read the real shape: the +// pull requests below are mesh-controller #212's and #211's heads and mesh-catalog #194's, as the forge said them on +// 2026-10-11 (novox/hq issue 438), with only their numbers and shas as they were. +func openPullsOf(t *testing.T, answer string) []OpenPull { + t.Helper() + var open struct { + Pulls []OpenPull `json:"pulls"` + } + if err := json.Unmarshal([]byte(answer), &open); err != nil { + t.Fatal(err) + } + return open.Pulls +} + +// A head the mesh's check started on and never answered: mesh/merge-gate pending since the controller began mapping +// it, mesh/repo-check never set. Its base requires both. +const startedNeverAnswered = `{"pulls": [{ + "base": "main", "head_sha": "c11222026a3b26a0cc4b277119e1ea059b82a36b", + "html_url": "http://127.0.0.1:20000/novox/mesh-controller/pulls/212", "number": 212, + "owner": "novox", "repo": "mesh-controller", + "required": ["mesh/merge-gate", "mesh/repo-check"], + "statuses": ["mesh/merge-gate", "mesh/delivery"], + "checks": { + "mesh/delivery": {"state": "pending", "description": "checking", "updated_at": "2026-10-11T00:11:46Z"}, + "mesh/merge-gate": {"state": "pending", "description": "the mesh is mapping this head onto its module graph", + "updated_at": "2026-10-11T00:11:39Z"} + } +}]}` + +// **A head whose merge check started and never answered is said** (novox/hq issue 438): pending, or missing a check +// its base requires, for as long as the delivery table gives a check asked and not answered (Bounds[Proposed]) since +// this owner first saw it so, it is a stalled line naming each check that did not answer and since when. Before, +// any merge-check status counted as checked, pending included, and such a head was never said. +func TestAHeadWhoseCheckStartedAndNeverAnsweredIsSaid(t *testing.T) { + w := newWorldOver(t, newMemStore(), newFakeController(), newFakeForge(), time.Date(2026, 10, 11, 0, 12, 5, 0, time.UTC)) + w.forge.open = openPullsOf(t, startedNeverAnswered) + w.h.SeeOpenPulls() + w.later(Bounds[Proposed].For - time.Minute) + w.h.SeeOpenPulls() + if lines := w.h.Stalled(); len(lines) != 0 { + t.Fatalf("a head still within the bound of a check asked and not answered is said: %+v", lines) + } + w.later(2 * time.Minute) + lines := w.h.Stalled() + if len(lines) != 1 { + t.Fatalf("a head whose check started and never answered is not said: %+v", lines) + } + l := lines[0] + if l.ID != IDOf("novox/mesh-controller", "c11222026a3b26a0cc4b277119e1ea059b82a36b") || l.Number != 212 || + l.State != Unanswered || l.Bound != Bounds[Proposed].For.String() || l.For != "1h1m0s" { + t.Fatalf("the line is not this head's, past the table's bound: %+v", l) + } + for _, want := range []string{"mesh/merge-gate", "pending since 2026-10-11T00:11:39Z", "mesh/repo-check", "never set", + "never answered"} { + if !strings.Contains(l.Says, want) { + t.Fatalf("the line does not say %q: %s", want, l.Says) + } + } + if want := []string{"mesh/merge-gate pending since 2026-10-11T00:11:39Z", "mesh/repo-check never set"}; !slices.Equal(l.Waiting, want) { + t.Fatalf("the line's checks waited on are %q, not %q", l.Waiting, want) + } + raw, _ := json.Marshal(l) + t.Logf("the line as the controller reads it: %s", raw) // mesh-controller's test of its words reads this line + if strings.Contains(l.Says, "mesh/delivery") { + t.Fatalf("the line names a status that is not the merge check: %s", l.Says) + } + + // Announced at last: a delivery of its own, with its own bound — no longer this line. + w.h.PullUpdated(pr("novox/mesh-controller", 212, "c11222026a3b26a0cc4b277119e1ea059b82a36b", "fix/402")) + w.h.SeeOpenPulls() + for _, l := range w.h.Stalled() { + if l.State == Unannounced || l.State == Unanswered { + t.Fatalf("an announced head is still said unannounced: %+v", l) + } + } +} + +// **A check that answers does not start the bound again**: merge-gate answers at fifty minutes, repo-check is never +// set, and the head is said on the bound from this owner's first sight of it, not fifty minutes later. +func TestACheckThatAnswersDoesNotStartTheBoundAgain(t *testing.T) { + w := newWorldOver(t, newMemStore(), newFakeController(), newFakeForge(), time.Date(2026, 10, 11, 0, 12, 5, 0, time.UTC)) + w.forge.open = openPullsOf(t, startedNeverAnswered) + w.h.SeeOpenPulls() + w.later(50 * time.Minute) + w.forge.open = openPullsOf(t, strings.Replace(startedNeverAnswered, `"state": "pending", "description": "the mesh is mapping`, + `"state": "success", "description": "pass: the mesh is mapping`, 1)) + w.h.SeeOpenPulls() + w.later(11 * time.Minute) + w.h.SeeOpenPulls() + lines := w.h.Stalled() + if len(lines) != 1 || lines[0].For != "1h1m0s" || !strings.Contains(lines[0].Says, "mesh/repo-check never set") || + strings.Contains(lines[0].Says, "mesh/merge-gate") { + t.Fatalf("a head whose merge-gate answered at fifty minutes is not said at sixty-one on what is left: %+v", lines) + } +} + +// A forge time with an offset is said in UTC, as the module's other lines say their times. +func TestAPendingTimeIsSaidInUTC(t *testing.T) { + w := newWorldOver(t, newMemStore(), newFakeController(), newFakeForge(), time.Date(2026, 10, 11, 0, 12, 5, 0, time.UTC)) + w.forge.open = openPullsOf(t, strings.Replace(startedNeverAnswered, "2026-10-11T00:11:39Z", "2026-10-11T02:11:39+02:00", 1)) + w.h.SeeOpenPulls() + w.later(Bounds[Proposed].For + time.Minute) + if lines := w.h.Stalled(); len(lines) != 1 || !strings.Contains(lines[0].Says, "pending since 2026-10-11T00:11:39Z") { + t.Fatalf("the pending time is not said in UTC: %+v", lines) + } +} + +// A check asked again — its pending status set anew, as a recheck or a new announcement does — starts the bound again: +// the head is said only once that ask too has gone unanswered past it. +func TestACheckAskedAgainStartsItsBoundAgain(t *testing.T) { + w := newWorldOver(t, newMemStore(), newFakeController(), newFakeForge(), time.Date(2026, 10, 11, 0, 12, 5, 0, time.UTC)) + w.forge.open = openPullsOf(t, startedNeverAnswered) + w.h.SeeOpenPulls() + w.later(50 * time.Minute) + again := openPullsOf(t, strings.Replace(startedNeverAnswered, "2026-10-11T00:11:39Z", "2026-10-11T01:02:00Z", 1)) + w.forge.open = again + w.h.SeeOpenPulls() + w.later(20 * time.Minute) + w.h.SeeOpenPulls() + if lines := w.h.Stalled(); len(lines) != 0 { + t.Fatalf("a check asked again twenty minutes ago is said on the first ask's time: %+v", lines) + } + w.later(41 * time.Minute) + if lines := w.h.Stalled(); len(lines) != 1 || !strings.Contains(lines[0].Says, "pending since 2026-10-11T01:02:00Z") { + t.Fatalf("a check asked again and not answered past the bound is not said: %+v", lines) + } +} + +// **A head whose required checks all answered is not said**, whatever they answered: a failure is a verdict, and the +// pull request says it itself. Nor is one whose only pending check is one its base does not require: hq requires +// mesh/repo-check alone. +func TestAHeadWhoseRequiredChecksAnsweredIsNotSaid(t *testing.T) { + w := newWorldOver(t, newMemStore(), newFakeController(), newFakeForge(), time.Date(2026, 10, 11, 0, 12, 5, 0, time.UTC)) + w.forge.open = openPullsOf(t, `{"pulls": [{ + "base": "main", "head_sha": "99df6b999bad77c51dfdd13470cf57b1f38e8c24", "number": 194, "owner": "novox", "repo": "mesh-catalog", + "required": ["mesh/merge-gate", "mesh/repo-check"], + "statuses": ["mesh/merge-gate", "mesh/repo-check", "mesh/delivery"], + "checks": { + "mesh/delivery": {"state": "failure", "description": "rejected", "updated_at": "2026-10-11T00:05:37Z"}, + "mesh/merge-gate": {"state": "success", "description": "pass: builds mounts → shanks; no bus step; every machine composes with the change as it did without (4 of 4 compose)", "updated_at": "2026-10-11T00:05:28Z"}, + "mesh/repo-check": {"state": "failure", "description": "fail: its merge-check.sh failed", "updated_at": "2026-10-11T00:05:28Z"} + } +}, { + "base": "main", "head_sha": "11b719fd5cdec43970e513a10a44a558042599e6", "number": 308, "owner": "novox", "repo": "hq", + "required": ["mesh/repo-check"], + "statuses": ["mesh/merge-gate", "mesh/repo-check", "mesh/delivery"], + "checks": { + "mesh/merge-gate": {"state": "pending", "description": "the mesh is mapping this head onto its module graph", "updated_at": "2026-10-11T00:01:00Z"}, + "mesh/repo-check": {"state": "success", "description": "pass: its merge-check.sh passed", "updated_at": "2026-10-11T00:02:00Z"} + } +}]}`) + w.h.SeeOpenPulls() + w.later(3 * Bounds[Proposed].For) + w.h.SeeOpenPulls() + if lines := w.h.Stalled(); len(lines) != 0 { + t.Fatalf("a head whose required checks answered is said: %+v", lines) + } +} + +// A forge module that says only which contexts a head has, not their states — the gitea module before issue 438 — +// is read as before: a head with any merge check is not said, and nothing is guessed of what it does not say. +func TestAForgeThatSaysNoStatesIsReadAsBefore(t *testing.T) { + w := newWorld(t) + w.forge.open = openPullsOf(t, `{"pulls": [{"base": "main", "head_sha": "c11222026a3b26a0cc4b277119e1ea059b82a36b", + "number": 212, "owner": "novox", "repo": "mesh-controller", "required": ["mesh/merge-gate", "mesh/repo-check"], + "statuses": ["mesh/merge-gate", "mesh/delivery"]}]}`) + w.h.SeeOpenPulls() + w.later(3 * Bounds[Proposed].For) + w.h.SeeOpenPulls() + if lines := w.h.Stalled(); len(lines) != 0 { + t.Fatalf("a head is said on states the forge did not give: %+v", lines) + } +} diff --git a/modules/mesh-delivery/cmd/mesh-delivery/issue441_test.go b/modules/mesh-delivery/cmd/mesh-delivery/issue441_test.go new file mode 100644 index 00000000..1b6f0691 --- /dev/null +++ b/modules/mesh-delivery/cmd/mesh-delivery/issue441_test.go @@ -0,0 +1,251 @@ +package main + +import ( + "slices" + "strings" + "testing" + "time" +) + +// A delivery held in ready or rejected whose required merge check reads pending or missing on the forge +// (novox/hq issue 441). The pull requests below are mesh-controller #215's head, ready, and mesh-lab #76's, rejected, +// as gitea_open_pulls said them on 2026-10-11, with only the one status changed that the case is about: a status +// write that lagged or was lost (issue 423), or a check asked again after the verdict. + +const readyHeadSHA = "d2e9dc61590f950b74ae2e93282c354646705188" + +// readyRepoCheckPending is #215 with mesh/repo-check pending since its delivery was ready. +const readyRepoCheckPending = `{"pulls": [{ + "base": "main", "head_sha": "d2e9dc61590f950b74ae2e93282c354646705188", + "html_url": "http://127.0.0.1:20000/novox/mesh-controller/pulls/215", "number": 215, + "owner": "novox", "repo": "mesh-controller", + "required": ["mesh/merge-gate", "mesh/repo-check"], + "statuses": ["mesh/merge-gate", "mesh/repo-check", "mesh/delivery"], + "checks": { + "mesh/delivery": {"state": "success", "description": "ready: it delivers once merged", "updated_at": "2026-10-11T00:42:06Z"}, + "mesh/merge-gate": {"state": "success", "description": "pass: builds build-agent → ace, g14, novox, shanks; no bus step; every machine composes with the change as it did without (4 of 4 compose)", "updated_at": "2026-10-11T00:42:01Z"}, + "mesh/repo-check": {"state": "pending", "description": "the repository's own check runs", "updated_at": "2026-10-11T00:42:01Z"} + } +}]}` + +// readyRepoCheckMissing is #215 whose mesh/repo-check was never written. +const readyRepoCheckMissing = `{"pulls": [{ + "base": "main", "head_sha": "d2e9dc61590f950b74ae2e93282c354646705188", "number": 215, + "owner": "novox", "repo": "mesh-controller", + "required": ["mesh/merge-gate", "mesh/repo-check"], + "statuses": ["mesh/merge-gate", "mesh/delivery"], + "checks": { + "mesh/delivery": {"state": "success", "description": "ready: it delivers once merged", "updated_at": "2026-10-11T00:42:06Z"}, + "mesh/merge-gate": {"state": "success", "description": "pass", "updated_at": "2026-10-11T00:42:01Z"} + } +}]}` + +// readyAnswered is #215 as the forge said it: every required check answered. +const readyAnswered = `{"pulls": [{ + "base": "main", "head_sha": "d2e9dc61590f950b74ae2e93282c354646705188", "number": 215, + "owner": "novox", "repo": "mesh-controller", + "required": ["mesh/merge-gate", "mesh/repo-check"], + "statuses": ["mesh/merge-gate", "mesh/repo-check", "mesh/delivery"], + "checks": { + "mesh/merge-gate": {"state": "success", "description": "pass", "updated_at": "2026-10-11T00:42:01Z"}, + "mesh/repo-check": {"state": "success", "description": "pass: its merge-check.sh passed", "updated_at": "2026-10-11T00:42:01Z"} + } +}]}` + +// heldIn is a delivery of #215's head in ready (a passing verdict) or rejected (a failing one). +func heldIn(t *testing.T, w *world, gate string, want State) string { + t.Helper() + w.h.PullUpdated(pr("novox/mesh-controller", 215, readyHeadSHA, "fix/215")) + w.h.Checked(verdict("novox/mesh-controller", 215, readyHeadSHA, gate)) + id := IDOf("novox/mesh-controller", readyHeadSHA) + if s := w.state(id); s != want { + t.Fatalf("a %s verdict left it %s", gate, s) + } + return id +} + +// **A delivery in ready whose required check reads pending on the forge is said** (novox/hq issue 441) on the same +// rule as a head with no delivery (issue 438): past the table's bound for a check asked and not answered, since this +// owner first saw it so, as an unanswered line naming the check and since when. Before, any head with a delivery was +// skipped, and ready has no bound of its own, so it waited for ever — unable to merge — with nothing said. +func TestADeliveryInReadyWhoseCheckIsPendingIsSaid(t *testing.T) { + w := newWorldOver(t, newMemStore(), newFakeController(), newFakeForge(), time.Date(2026, 10, 11, 0, 43, 0, 0, time.UTC)) + id := heldIn(t, w, "pass", Ready) + w.forge.open = openPullsOf(t, readyRepoCheckPending) + w.h.SeeOpenPulls() + w.later(Bounds[Proposed].For - time.Minute) + w.h.SeeOpenPulls() + if lines := w.h.Stalled(); len(lines) != 0 { + t.Fatalf("a ready delivery still within the bound is said: %+v", lines) + } + w.later(2 * time.Minute) + lines := w.h.Stalled() + if len(lines) != 1 { + t.Fatalf("a ready delivery whose required check is pending past the bound is not said: %+v", lines) + } + l := lines[0] + if l.ID != id || l.Number != 215 || l.State != Unanswered || l.Bound != Bounds[Proposed].For.String() || l.For != "1h1m0s" { + t.Fatalf("the line is not this delivery's, past the table's bound: %+v", l) + } + if want := []string{"mesh/repo-check pending since 2026-10-11T00:42:01Z"}; !slices.Equal(l.Waiting, want) { + t.Fatalf("the line's checks waited on are %q, not %q", l.Waiting, want) + } + for _, want := range []string{"novox/mesh-controller#215", "ready", "mesh/repo-check pending since 2026-10-11T00:42:01Z", "recheck"} { + if !strings.Contains(l.Says, want) { + t.Fatalf("the line does not say %q: %s", want, l.Says) + } + } + if strings.Contains(l.Says, "No delivery of it is held") { + t.Fatalf("the line says no delivery is held of a head whose delivery is ready: %s", l.Says) + } + + // The forge's status catches up: the check answered, and the line is gone. + w.forge.open = openPullsOf(t, readyAnswered) + w.h.SeeOpenPulls() + if lines := w.h.Stalled(); len(lines) != 0 { + t.Fatalf("a ready delivery whose check answered is still said: %+v", lines) + } +} + +// **A delivery in rejected whose required check was never set is said** too: a lost status write leaves the pull +// request with no verdict to show, and nothing here asks again. It was announced — a delivery exists — so it is +// unanswered, never unannounced, though its head has no merge check at all. +func TestADeliveryInRejectedWhoseCheckIsMissingIsSaid(t *testing.T) { + w := newWorldOver(t, newMemStore(), newFakeController(), newFakeForge(), time.Date(2026, 10, 11, 0, 43, 0, 0, time.UTC)) + id := heldIn(t, w, "fail", Rejected) + w.forge.open = openPullsOf(t, strings.Replace(readyRepoCheckMissing, + `"mesh/merge-gate": {"state": "success", "description": "pass", "updated_at": "2026-10-11T00:42:01Z"}`, + `"mesh/merge-gate": {"state": "failure", "description": "fail", "updated_at": "2026-10-11T00:42:01Z"}`, 1)) + w.h.SeeOpenPulls() + w.later(Bounds[Proposed].For + time.Minute) + w.h.SeeOpenPulls() + lines := w.h.Stalled() + if len(lines) != 1 || lines[0].ID != id || lines[0].State != Unanswered || + !slices.Equal(lines[0].Waiting, []string{"mesh/repo-check never set"}) || !strings.Contains(lines[0].Says, "rejected") { + t.Fatalf("a rejected delivery whose required check was never set is not said: %+v", lines) + } + + // No merge check on its head at all — both writes lost — is still a delivery's: unanswered, never unannounced. + w2 := newWorldOver(t, newMemStore(), newFakeController(), newFakeForge(), time.Date(2026, 10, 11, 0, 43, 0, 0, time.UTC)) + heldIn(t, w2, "fail", Rejected) + w2.forge.open = openPullsOf(t, `{"pulls": [{"base": "main", "head_sha": "`+readyHeadSHA+`", "number": 215, + "owner": "novox", "repo": "mesh-controller", "required": ["mesh/merge-gate", "mesh/repo-check"], + "statuses": ["mesh/delivery"], "checks": {"mesh/delivery": {"state": "failure", "description": "rejected"}}}]}`) + w2.h.SeeOpenPulls() + if lines := w2.h.Stalled(); len(lines) != 0 { + t.Fatalf("a delivery's head without a check is said at once, before the table's bound: %+v", lines) + } + w2.later(Bounds[Proposed].For + time.Minute) + w2.h.SeeOpenPulls() + lines = w2.h.Stalled() + if len(lines) != 1 || lines[0].State != Unanswered || + !slices.Equal(lines[0].Waiting, []string{"mesh/merge-gate never set", "mesh/repo-check never set"}) { + t.Fatalf("a rejected delivery whose head has no merge check is not said as unanswered: %+v", lines) + } +} + +// **A delivery in a state with a bound of its own is not said here**: a proposed one waits on its check under +// Bounds[Proposed] and H2's re-ask, and a second line for it would say the same thing twice. +func TestAProposedDeliveryIsSaidByItsOwnBoundOnly(t *testing.T) { + w := newWorldOver(t, newMemStore(), newFakeController(), newFakeForge(), time.Date(2026, 10, 11, 0, 43, 0, 0, time.UTC)) + w.h.PullUpdated(pr("novox/mesh-controller", 215, readyHeadSHA, "fix/215")) + id := IDOf("novox/mesh-controller", readyHeadSHA) + w.forge.open = openPullsOf(t, readyRepoCheckPending) + w.h.SeeOpenPulls() + w.later(3 * Bounds[Proposed].For) + w.h.SeeOpenPulls() + for _, l := range w.h.Stalled() { + if l.State == Unanswered || l.State == Unannounced { + t.Fatalf("a proposed delivery %s is said as a head without a delivery: %+v", id, l) + } + } +} + +// **A ready delivery whose required checks answered is not said**, as the forge said every one on 2026-10-11. +func TestAReadyDeliveryWhoseChecksAnsweredIsNotSaid(t *testing.T) { + w := newWorldOver(t, newMemStore(), newFakeController(), newFakeForge(), time.Date(2026, 10, 11, 0, 43, 0, 0, time.UTC)) + heldIn(t, w, "pass", Ready) + w.forge.open = openPullsOf(t, readyAnswered) + w.h.SeeOpenPulls() + w.later(3 * Bounds[Proposed].For) + w.h.SeeOpenPulls() + if lines := w.h.Stalled(); len(lines) != 0 { + t.Fatalf("a ready delivery whose checks answered is said: %+v", lines) + } +} + +// saidReady is a ready delivery of #215's head whose mesh/repo-check reads pending on the forge, said past the bound. +func saidReady(t *testing.T) (*world, string) { + t.Helper() + w := newWorldOver(t, newMemStore(), newFakeController(), newFakeForge(), time.Date(2026, 10, 11, 0, 43, 0, 0, time.UTC)) + id := heldIn(t, w, "pass", Ready) + w.forge.open = openPullsOf(t, readyRepoCheckPending) + w.h.SeeOpenPulls() + w.later(Bounds[Proposed].For + time.Minute) + if lines := w.h.Stalled(); len(lines) != 1 || lines[0].ID != id || lines[0].State != Unanswered { + t.Fatalf("a ready delivery whose check is pending past the bound is not said: %+v", lines) + } + return w, id +} + +// unansweredOf are the unanswered lines among what `stalled` says. +func unansweredOf(w *world) []StalledLine { + var out []StalledLine + for _, l := range w.h.Stalled() { + if l.State == Unanswered { + out = append(out, l) + } + } + return out +} + +// **A said line clears when its delivery's state changes** (novox/hq issue 441): a recheck proposes it again, under +// the proposed state's own bound, and a merge publishes it, past the pull request. Both clear the line at once — +// before the forge is read again, which may lag — and stay cleared once it is read. +func TestASaidLineClearsWhenItsDeliveryMovesOn(t *testing.T) { + t.Run("a recheck proposes it", func(t *testing.T) { + w, id := saidReady(t) + if _, err := w.h.Recheck(id, "its repo-check status was lost", "a person"); err != nil { + t.Fatal(err) + } + if s := w.state(id); s != Proposed { + t.Fatalf("a recheck left it %s", s) + } + if lines := unansweredOf(w); len(lines) != 0 { + t.Fatalf("a rechecked delivery, proposed again, is still said as unanswered: %+v", lines) + } + w.h.SeeOpenPulls() + if lines := unansweredOf(w); len(lines) != 0 { + t.Fatalf("a rechecked delivery is said again once the forge is read: %+v", lines) + } + }) + t.Run("a merge publishes it", func(t *testing.T) { + w, id := saidReady(t) + w.h.WalkMoved(w.ctl.put(aWalk("plan-1", "novox/mesh-controller", merge, true))) + w.h.PullMerged(merged("novox/mesh-controller", 215, readyHeadSHA, merge)) + if s := w.state(id); s != Published { + t.Fatalf("a merge left it %s", s) + } + if lines := unansweredOf(w); len(lines) != 0 { + t.Fatalf("a published delivery is still said as unanswered: %+v", lines) + } + w.h.SeeOpenPulls() // the forge's open list lags the merge: the head is still on it + if lines := unansweredOf(w); len(lines) != 0 { + t.Fatalf("a published delivery still on a lagging open list is said: %+v", lines) + } + }) +} + +// **A said line clears when its pull request leaves the open list** — closed, its event to this owner lost, so the +// delivery is still ready: the forge no longer lists it, and nothing is said of it. +func TestASaidLineClearsWhenItsPullRequestIsNoLongerOpen(t *testing.T) { + w, id := saidReady(t) + w.forge.open = nil + w.h.SeeOpenPulls() + if s := w.state(id); s != Ready { + t.Fatalf("the delivery moved to %s with no event: the case is not the one meant", s) + } + if lines := unansweredOf(w); len(lines) != 0 { + t.Fatalf("a pull request no longer open is still said: %+v", lines) + } +} diff --git a/modules/mesh-delivery/cmd/mesh-delivery/issue442_test.go b/modules/mesh-delivery/cmd/mesh-delivery/issue442_test.go new file mode 100644 index 00000000..b6f9e899 --- /dev/null +++ b/modules/mesh-delivery/cmd/mesh-delivery/issue442_test.go @@ -0,0 +1,200 @@ +package main + +import ( + "strings" + "testing" + "time" +) + +// novox/hq issue 442: five closed groups read "delivering, 1 of 2 delivered" for days, each beside a member +// whose walk a later walk of the same trunk superseded. Nothing in them was still in progress. A superseded +// member counts as reached when a later delivered walk on the same trunk carried every module it moves (ADR +// 0270's covered test); a closed group whose members all ended reads delivered when every one reached the +// machines, and stopped otherwise — never delivering. +// +// The shapes are the live groups' as mesh-delivery showed them on 2026-10-11: feat/delivery-checks-verb (the +// superseded member last in the order) and fix/assign-says-why-a-module-is-not-there (the superseded member +// first). +func TestAGroupWhoseMembersEndedEnds(t *testing.T) { + const cat, ctl, lab = "novox/mesh-catalog", "novox/mesh-controller", "novox/mesh-lab" + at := time.Date(2026, 10, 7, 19, 4, 10, 0, time.UTC) + + // superseded is a member merged and walked, its walk superseded by a later walk of the same trunk. + superseded := func(repo, sha, merged, group string, modules ...string) *Delivery { + return &Delivery{ID: IDOf(repo, sha), Repository: repo, Commit: sha, Number: 109, Base: "main", + Branch: group, Group: group, State: Superseded, Since: at.Add(time.Minute), Created: at.Add(-time.Hour), + MergedAs: merged, MergedAt: at, Plan: planMoving(repo, modules...), Check: &Verdict{Gate: "pass"}, + Walk: &WalkSeen{ID: "plan-" + merged[:6], Commit: merged, State: walkSuperseded, + Note: "superseded at tier 0 by plan-later (" + repo + " at 68aeee2c); planned there again"}, + Transitions: []Transition{{At: at.Add(time.Minute), From: Delivering, To: Superseded, Event: EvSuperseded, + Why: "a newer delivery to the same trunk took over its walk"}}} + } + // delivered is the other member, delivered by its own walk. + delivered := func(repo, sha, merged, group string) *Delivery { + return &Delivery{ID: IDOf(repo, sha), Repository: repo, Commit: sha, Number: 117, Base: "main", Branch: group, + Group: group, State: Delivered, Since: at.Add(-10 * time.Minute), Created: at.Add(-2 * time.Hour), + MergedAs: merged, MergedAt: at.Add(-time.Hour), Plan: planMoving(repo, "mesh-controller"), + Check: &Verdict{Gate: "pass"}, Walk: &WalkSeen{ID: "plan-" + merged[:6], Commit: merged, State: walkDone}} + } + // later is the later merge of the same repository whose walk took the member's over and finished, sending + // each module given to every machine running it. + later := func(repo, merged string, modules ...string) *Delivery { + d := &Delivery{ID: IDOf(repo, "c0c0c0c0c0c0c0c0"), Repository: repo, Commit: "c0c0c0c0c0c0c0c0", Number: 110, + Base: "main", State: Delivered, Since: at.Add(30 * time.Minute), Created: at.Add(-time.Hour), + MergedAs: merged, MergedAt: at.Add(time.Minute), Plan: planMoving(repo, modules...), Check: &Verdict{Gate: "pass"}, + Walk: &WalkSeen{ID: "plan-later", Commit: merged, State: walkDone}} + for _, m := range modules { + d.Steps = append(d.Steps, Step{Machine: "novox", Module: m, State: StepPassed}, + Step{Machine: "the rest running it", Module: m, State: StepSent}) + } + return d + } + group := func(id string, order ...string) *Group { + return &Group{ID: id, Members: append([]string(nil), order...), Closed: true, Said: "delivering"} + } + + const checks, assign = "feat/delivery-checks-verb", "fix/assign-says-why-a-module-is-not-there" + catID, ctlID := IDOf(cat, "13894744a701fe59"), IDOf(ctl, "1fdff00794e95470") + asCtlID, asLabID := IDOf(ctl, "8adb7f1a051941e9"), IDOf(lab, "33e63ee5b47a7552") + for _, c := range []struct { + name string + kept []*Delivery + group *Group + contains bool // what the forge says of the later merge containing the superseded member's + want string + whySays string + }{ + {"superseded last, a later delivered walk carried every module it moves", + []*Delivery{superseded(cat, "13894744a701fe59", "9d4bbbd2bd271049", checks, "gitea", "mesh-delivery"), + delivered(ctl, "1fdff00794e95470", "85d664438f87f885", checks), + later(cat, "68aeee2c00000000", "gitea", "mesh-delivery")}, + group(checks, ctlID, catID), true, "delivered", "every member"}, + {"superseded last, the later walk carried one of its two modules", + []*Delivery{superseded(cat, "13894744a701fe59", "9d4bbbd2bd271049", checks, "gitea", "mesh-delivery"), + delivered(ctl, "1fdff00794e95470", "85d664438f87f885", checks), + later(cat, "68aeee2c00000000", "mesh-delivery")}, + group(checks, ctlID, catID), true, "stopped", "not carried by a later delivery: "}, + {"superseded last, the forge says the later merge does not contain it", + []*Delivery{superseded(cat, "13894744a701fe59", "9d4bbbd2bd271049", checks, "gitea", "mesh-delivery"), + delivered(ctl, "1fdff00794e95470", "85d664438f87f885", checks), + later(cat, "68aeee2c00000000", "gitea", "mesh-delivery")}, + group(checks, ctlID, catID), false, "stopped", "not carried by a later delivery: "}, + {"superseded first, a later delivered walk carried every module it moves", + []*Delivery{superseded(ctl, "8adb7f1a051941e9", "a5f53ef9ebdf3650", assign, "build-agent", "mesh-controller", "route-proxy"), + delivered(lab, "33e63ee5b47a7552", "77777777aaaaaaaa", assign), + later(ctl, "3f68a49500000000", "build-agent", "mesh-controller", "route-proxy")}, + group(assign, asCtlID, asLabID), true, "delivered", "every member"}, + {"superseded first, nothing later delivered", + []*Delivery{superseded(ctl, "8adb7f1a051941e9", "a5f53ef9ebdf3650", assign, "build-agent", "mesh-controller", "route-proxy"), + delivered(lab, "33e63ee5b47a7552", "77777777aaaaaaaa", assign)}, + group(assign, asCtlID, asLabID), true, "stopped", "not carried by a later delivery: "}, + {"still in progress: a member delivering beside a superseded one", + func() []*Delivery { + d := delivered(ctl, "1fdff00794e95470", "85d664438f87f885", checks) + d.State, d.Walk.State = Delivering, walkRolling + return []*Delivery{superseded(cat, "13894744a701fe59", "9d4bbbd2bd271049", checks, "gitea", "mesh-delivery"), d} + }(), + group(checks, ctlID, catID), true, "delivering", "0 of 2"}, + } { + t.Run(c.name, func(t *testing.T) { + store := newMemStore() + for _, d := range c.kept { + if err := store.PutDelivery(d); err != nil { + t.Fatal(err) + } + } + if err := store.PutGroup(c.group); err != nil { + t.Fatal(err) + } + forge := newFakeForge() + forge.contains[c.kept[0].MergedAs+" in "+"68aeee2c00000000"] = c.contains + forge.contains[c.kept[0].MergedAs+" in "+"3f68a49500000000"] = c.contains + w := newWorldOver(t, store, newFakeController(), forge, at.Add(4*24*time.Hour)) + if err := w.h.Reconcile(); err != nil { + t.Fatal(err) + } + w.settleAll() + g := w.h.groups[c.group.ID] + state, why := GroupState(g, w.h.membersOf(g)) + if state != c.want || !strings.Contains(why, c.whySays) { + t.Fatalf("the group reads %s (%s), want %s saying %q", state, why, c.want, c.whySays) + } + // The forge is asked once per pair, however many ticks follow. + for range 3 { + w.later(2 * time.Minute) + w.settleAll() + } + w.forge.mu.Lock() + asked := map[string]int{} + for _, a := range w.forge.containsAsks { + asked[a]++ + } + w.forge.mu.Unlock() + for pair, n := range asked { + if n != 1 { + t.Fatalf("the forge was asked %d times whether %s", n, pair) + } + } + // What was learned survives a read-back, and the group reads the same with no new ask. + sup := w.delivery(c.kept[0].ID) + again := newWorldOver(t, store, newFakeController(), forge, w.now.Add(time.Hour)) + if err := again.h.Reconcile(); err != nil { + t.Fatal(err) + } + again.settleAll() + back := again.delivery(c.kept[0].ID) + if back.CoveredBy != sup.CoveredBy || len(back.ContainedIn) != len(sup.ContainedIn) { + t.Fatalf("read back, covered by %q and contained in %v; was %q and %v", back.CoveredBy, back.ContainedIn, + sup.CoveredBy, sup.ContainedIn) + } + for k, v := range sup.ContainedIn { + if back.ContainedIn[k] != v { + t.Fatalf("read back, contained in %s is %v, was %v", k, back.ContainedIn[k], v) + } + } + if s2, _ := GroupState(again.h.groups[c.group.ID], again.h.membersOf(again.h.groups[c.group.ID])); s2 != state { + t.Fatalf("read back, the group reads %s, was %s", s2, state) + } + forge.mu.Lock() + n := len(forge.containsAsks) + forge.mu.Unlock() + if want := len(asked); c.want != "delivering" && n != want { + t.Fatalf("read back, the forge was asked again: %d asks, was %d", n, want) + } + // Each member keeps its own end: the group's label changes, no delivery is moved by hand. + for _, d := range c.kept { + if got := w.state(d.ID); got != d.State { + t.Fatalf("%s moved from %s to %s", d.ID, d.State, got) + } + } + // What the group said reached every member's head, once it ended. + if c.want != "delivering" { + w.forge.mu.Lock() + s := w.forge.statuses[c.kept[0].Repository+"@"+c.kept[0].Commit+" mesh/delivery-group"] + w.forge.mu.Unlock() + if !strings.Contains(s, c.want) { + t.Fatalf("mesh/delivery-group on the superseded member's head says %q", s) + } + } + }) + } +} + +// The forge's mesh/delivery-group line, "group stopped: ", is clipped at 140 characters: for each of +// the five live groups of novox/hq issue 442, the superseded member's id fits in it whole. +func TestAStoppedGroupsLineNamesItsSupersededMemberWhole(t *testing.T) { + for group, member := range map[string]string{ + "feat/delivery-checks-verb": "novox/mesh-catalog@13894744a701", + "feat/machine-network-health": "novox/mesh-controller@b8bbf9c79fd8", + "fix/373-one-message-per-condition": "novox/mesh-catalog@8dfc19c9f7c8", + "fix/assign-says-why-a-module-is-not-there": "novox/mesh-controller@8adb7f1a0519", + "fix/needs-you-from-the-console": "novox/mesh-controller@2e6a9b1efc05", + } { + g := &Group{ID: group, Members: []string{member, "novox/hq@39585e3bb2ea"}, Closed: true} + state, why := GroupState(g, []*Delivery{{ID: member, State: Superseded}, {ID: "novox/hq@39585e3bb2ea", State: Delivered}}) + line := clip("group "+group+" "+state+": "+why, 140) + if state != "stopped" || !strings.HasSuffix(line, member) { + t.Fatalf("group %s: the line %q (%d characters) does not end in %s", group, line, len(line), member) + } + } +} diff --git a/modules/mesh-delivery/cmd/mesh-delivery/issue443_test.go b/modules/mesh-delivery/cmd/mesh-delivery/issue443_test.go new file mode 100644 index 00000000..19b6dfa8 --- /dev/null +++ b/modules/mesh-delivery/cmd/mesh-delivery/issue443_test.go @@ -0,0 +1,66 @@ +package main + +import ( + "encoding/json" + "slices" + "strings" + "testing" + "time" +) + +// **An unanswered line gives the checks it waits on as data** (novox/hq issue 443): each check's context, its state +// and, for a pending one, since when in RFC 3339 — so the controller says the time in the operator's own zone. A time +// inside a finished sentence cannot be said again in another zone, and the operator read "since +// 2026-10-11T00:11:39Z". The finished words stay beside the data, for a controller that does not read it yet. +func TestAnUnansweredLineGivesItsChecksAsData(t *testing.T) { + w := newWorldOver(t, newMemStore(), newFakeController(), newFakeForge(), time.Date(2026, 10, 11, 0, 12, 5, 0, time.UTC)) + w.forge.open = openPullsOf(t, strings.Replace(startedNeverAnswered, "2026-10-11T00:11:39Z", "2026-10-11T02:11:39+02:00", 1)) + w.h.SeeOpenPulls() + w.later(Bounds[Proposed].For + time.Minute) + lines := w.h.Stalled() + if len(lines) != 1 { + t.Fatalf("the head is not said: %+v", lines) + } + want := []WaitingCheck{{Context: "mesh/merge-gate", State: CheckPending, Since: "2026-10-11T00:11:39Z"}, + {Context: "mesh/repo-check", State: CheckNeverSet}} + if !slices.Equal(lines[0].Checks, want) { + t.Fatalf("the line's checks are %+v, not %+v", lines[0].Checks, want) + } + if words := []string{"mesh/merge-gate pending since 2026-10-11T00:11:39Z", "mesh/repo-check never set"}; !slices.Equal(lines[0].Waiting, words) { + t.Fatalf("the finished words a controller of before reads are gone: %q", lines[0].Waiting) + } + raw, _ := json.Marshal(lines[0]) + t.Logf("the line as the controller reads it: %s", raw) + if !strings.Contains(string(raw), `"checks":[{"context":"mesh/merge-gate","state":"pending","since":"2026-10-11T00:11:39Z"},{"context":"mesh/repo-check","state":"never-set"}]`) { + t.Fatalf("the line's checks are not in the shape the controller reads: %s", raw) + } +} + +// The data and the words never disagree: for every head the words are made from, the checks give the same contexts +// in the same order, and a pending time the forge did not say, or said unreadably, is given as no time at all. +func TestAnUnansweredLinesChecksAgreeWithItsWords(t *testing.T) { + for _, tc := range []struct { + name, at string + since string + }{{"a time in UTC", "2026-10-11T00:11:39Z", "2026-10-11T00:11:39Z"}, + {"a time with an offset", "2026-10-11T02:11:39+02:00", "2026-10-11T00:11:39Z"}, + {"no time", "", ""}, + {"an unreadable time", "yesterday", ""}} { + t.Run(tc.name, func(t *testing.T) { + p := openPullsOf(t, strings.Replace(startedNeverAnswered, "2026-10-11T00:11:39Z", tc.at, 1))[0] + words, _, _ := p.unanswered() + checks := p.waitingChecks() + if len(words) != len(checks) { + t.Fatalf("words %q, checks %+v", words, checks) + } + for i, c := range checks { + if !strings.HasPrefix(words[i], c.Context+" ") { + t.Fatalf("word %q is not of check %+v", words[i], c) + } + } + if checks[0].State != CheckPending || checks[0].Since != tc.since { + t.Fatalf("the pending check is %+v, not since %q", checks[0], tc.since) + } + }) + } +} diff --git a/modules/mesh-delivery/cmd/mesh-delivery/issue451_test.go b/modules/mesh-delivery/cmd/mesh-delivery/issue451_test.go new file mode 100644 index 00000000..832c53b7 --- /dev/null +++ b/modules/mesh-delivery/cmd/mesh-delivery/issue451_test.go @@ -0,0 +1,48 @@ +package main + +import ( + "encoding/json" + "strings" + "testing" + "time" +) + +// **An unanswered line says the state of the delivery it holds, as data** (novox/hq issue 451): for a delivery in +// ready or rejected, whose check answered once, the controller advises a recheck rather than the build queue, and +// it reads that state from the line, never from its words. A head with no delivery held says none. +func TestAnUnansweredLineNamesTheStateOfItsDelivery(t *testing.T) { + at := time.Date(2026, 10, 11, 0, 43, 0, 0, time.UTC) + for _, c := range []struct { + gate string + want State + }{{"pass", Ready}, {"fail", Rejected}} { + w := newWorldOver(t, newMemStore(), newFakeController(), newFakeForge(), at) + id := heldIn(t, w, c.gate, c.want) + w.forge.open = openPullsOf(t, readyRepoCheckPending) + w.h.SeeOpenPulls() + w.later(Bounds[Proposed].For + time.Minute) + w.h.SeeOpenPulls() + lines := w.h.Stalled() + if len(lines) != 1 || lines[0].ID != id || lines[0].State != Unanswered || lines[0].Held != c.want { + t.Fatalf("a %s delivery's unanswered line does not say its state as data: %+v", c.want, lines) + } + raw, _ := json.Marshal(lines[0]) + if !strings.Contains(string(raw), `"held":"`+string(c.want)+`"`) { + t.Fatalf("the line the controller reads carries no held state: %s", raw) + } + } + + // The same head with no delivery held here: unanswered, and no state to say. + w := newWorldOver(t, newMemStore(), newFakeController(), newFakeForge(), at) + w.forge.open = openPullsOf(t, readyRepoCheckPending) + w.h.SeeOpenPulls() + w.later(Bounds[Proposed].For + time.Minute) + w.h.SeeOpenPulls() + lines := w.h.Stalled() + if len(lines) != 1 || lines[0].State != Unanswered || lines[0].Held != "" { + t.Fatalf("a head with no delivery held says a state: %+v", lines) + } + if raw, _ := json.Marshal(lines[0]); strings.Contains(string(raw), `"held"`) { + t.Fatalf("a line with no delivery held carries a held field: %s", raw) + } +} diff --git a/modules/mesh-delivery/cmd/mesh-delivery/issue459_test.go b/modules/mesh-delivery/cmd/mesh-delivery/issue459_test.go new file mode 100644 index 00000000..568941bb --- /dev/null +++ b/modules/mesh-delivery/cmd/mesh-delivery/issue459_test.go @@ -0,0 +1,338 @@ +package main + +import ( + "fmt" + "strings" + "testing" + "time" +) + +// novox/hq issue 459, ADR 0299: since the merge window (ADR 0276) one walk carries every merge of its batch. A group +// stop asked the controller to end the stopped member's walk, and with it every other delivery on it, which no one +// stopped. Now a group stop asks `delivery-stop` with the stopped members' merges to leave out: the walk ends, and +// the controller walks its other merges again in a fresh batch; a later merge of a stopped merge's repository +// contains it and ends with it, named. A person's stop of a delivery on a shared walk answers first with what +// ending the walk would take down, and acts only on a way given: `again` or `walk`. + +// shape459 is a world of a delivery group feat/g (its members in the repositories group names, in that order) and +// lone deliveries (the repositories alone names), each checked and merged. The group's first member is on a walk of +// its own, plan-a; every other merge is carried by one walk, plan-2, waiting for its word. +type shape459 struct { + *world + ids, heads, merges map[string]string // by repository +} + +func open459(t *testing.T, group, alone []string) *shape459 { + t.Helper() + w := newWorld(t) + s := &shape459{world: w, ids: map[string]string{}, heads: map[string]string{}, merges: map[string]string{}} + n := 0 + add := func(repo, branch string) { + n++ + head := fmt.Sprintf("%02d%s", n, strings.Repeat("h", 14)) + s.ids[repo], s.heads[repo] = IDOf(repo, head), head + s.merges[repo] = fmt.Sprintf("%02d%s", n, strings.Repeat("m", 10)) + w.h.PullUpdated(pr(repo, n, head, branch, "x")) + w.h.Checked(verdict(repo, n, head, "pass")) + } + for _, r := range group { + add(r, "feat/g") + } + for _, r := range alone { + add(r, "fix/"+r[strings.LastIndex(r, "/")+1:]) + } + w.settleAll() + w.h.Checked(CheckedEvent{Group: "feat/g", ID: "check-1", Verdict: "pass", Summary: "compose"}) + w.settleAll() + for i, r := range append(append([]string{}, group...), alone...) { + w.h.PullMerged(merged(r, i+1, s.heads[r], s.merges[r])) + } + first := aWalk("plan-a", group[0], s.merges[group[0]], true) + first.Delivery.Merges = []WalkMerge{{Repository: group[0], Commit: s.merges[group[0]]}} + w.h.WalkMoved(w.ctl.put(first)) + two := aWalk("plan-2", alone[0], s.merges[alone[0]], true) + two.Created = two.Created.Add(time.Minute) + for _, r := range append(append([]string{}, group[1:]...), alone...) { + two.Delivery.Merges = append(two.Delivery.Merges, WalkMerge{Repository: r, Commit: s.merges[r]}) + } + w.h.WalkMoved(w.ctl.put(two)) + w.settleAll() + return s +} + +// failFirst fails the group's first member's walk: the members after it are stopped by their group. +func (s *shape459) failFirst() { + a := s.ctl.walk("plan-a") + a.State, a.Note = walkFailed, "the first member failed its gate on anchor" + s.h.WalkMoved(s.ctl.put(a)) + s.settleAll() +} + +// heard is the controller's records of the stopped walk and the walk that walks its merges again, as plan-moved says +// them, and then a walk done. +func (s *shape459) heard(walks ...string) { + for _, id := range walks { + s.h.WalkMoved(s.ctl.walk(id)) + } + s.settleAll() +} + +// cutAgain is the controller cutting the batch that walks a stopped walk's merges again, as plan-moved says it. +func (s *shape459) cutAgain(id string) { + s.h.WalkMoved(s.ctl.cut(id)) + s.settleAll() +} + +func (s *shape459) done(walk string) { + r := s.ctl.walk(walk) + sent := s.now + for m := range r.Modules { + r.Modules[m] = &WalkModule{State: "built", SentAt: &sent} + } + r.State = walkDone + s.h.WalkMoved(s.ctl.put(r)) + s.settleAll() +} + +func lastWhy(d *Delivery) string { + if len(d.Transitions) == 0 { + return "" + } + return d.Transitions[len(d.Transitions)-1].Why +} + +// retarget makes the delivery of repo `later` a later merge of repository `of`: same repository, carried by plan-2 +// for the merge of `of`. +func (s *shape459) retarget(later, of string) { + s.t.Helper() + s.h.mu.Lock() + d := s.h.deliveries[s.ids[later]] + d.Repository, d.MergedAt = of, d.MergedAt.Add(time.Minute) + s.h.mu.Unlock() + s.ctl.mu.Lock() + s.ctl.mergedAt[s.merges[of]], s.ctl.mergedAt[s.merges[later]] = d.MergedAt.Add(-time.Minute), d.MergedAt + s.ctl.mu.Unlock() + two := s.ctl.walk("plan-2") + merges := []WalkMerge{} + for _, m := range two.Delivery.Merges { + switch m.Repository { + case of: + m.Carried = s.merges[later] + case later: + m.Repository = of + } + merges = append(merges, m) + } + wd := *two.Delivery + wd.Merges = merges + two.Delivery = &wd + s.h.WalkMoved(s.ctl.put(two)) + s.settleAll() +} + +// shared is two lone deliveries, y and z, let go together on one walk, plan-2. +func shared459(t *testing.T) *shape459 { + t.Helper() + s := open459(t, []string{"novox/a"}, []string{"novox/y", "novox/z"}) + for _, r := range []string{"novox/y", "novox/z"} { + if st := s.state(s.ids[r]); st != Delivering { + t.Fatalf("%s is %s, let go %v; want both delivering on plan-2", r, st, s.ctl.delivers) + } + } + return s +} + +// A person's stop of y, which shares its walk with z, does nothing at first: it names z and the two ways. +func TestAPersonsStopOfASharedWalkAsksFirst(t *testing.T) { + s := shared459(t) + said, err := s.h.StopWay(s.ids["novox/y"], "not wanted after all", "jochen", "") + if err == nil { + t.Fatalf("the stop acted at once: %q", said) + } + for _, want := range []string{s.ids["novox/z"], "plan-2", "again", "walk", "not stopped"} { + if !strings.Contains(err.Error(), want) { + t.Fatalf("the stop said %q; want it to name %q", err, want) + } + } + if st := s.state(s.ids["novox/y"]); st != Delivering || len(s.ctl.stops)+len(s.ctl.withouts) != 0 { + t.Fatalf("y is %s, the controller was asked %v %v; want nothing done", st, s.ctl.stops, s.ctl.withouts) + } + if _, err := s.h.StopWay(s.ids["novox/y"], "x", "jochen", "sideways"); err == nil { + t.Fatal("a way that is neither again nor walk was taken") + } +} + +// With `again`, y is stopped, the walk is stopped without y's merge, and z goes back to published and follows the +// walk that walks it again. +func TestAPersonsStopAgainWalksTheOthersAgain(t *testing.T) { + s := shared459(t) + said, err := s.h.StopWay(s.ids["novox/y"], "not wanted after all", "jochen", "again") + if err != nil { + t.Fatal(err) + } + if !strings.Contains(said, "plan-2-again") || !strings.Contains(said, "novox/z@") { + t.Fatalf("the stop said %q; want it to name the walk that walks z again", said) + } + if want := "plan-2 without novox/y@" + s.merges["novox/y"]; !containsString(s.ctl.withouts, want) || + containsString(s.ctl.stops, "plan-2") { + t.Fatalf("the controller was asked %v, ended whole %v; want %q", s.ctl.withouts, s.ctl.stops, want) + } + s.heard("plan-2") + if z := s.delivery(s.ids["novox/z"]); z.State != Published { + t.Fatalf("z is %s once plan-2 was stopped and walked again; want published, waiting for that walk", z.State) + } + s.heard("plan-2-again") + s.cutAgain("plan-2-again") + if z := s.delivery(s.ids["novox/z"]); z.Walk == nil || z.Walk.ID != "plan-2-again" || z.State != Delivering { + t.Fatalf("z is %s on %+v; want it delivering on plan-2-again", z.State, z.Walk) + } + if st := s.state(s.ids["novox/y"]); st != Stopped { + t.Fatalf("y is %s", st) + } +} + +// With `walk`, the whole walk is stopped, and z ends stopped, naming y's stop. +func TestAPersonsStopOfTheWalkStopsEveryDelivery(t *testing.T) { + s := shared459(t) + if _, err := s.h.StopWay(s.ids["novox/y"], "not wanted after all", "jochen", "walk"); err != nil { + t.Fatal(err) + } + if !containsString(s.ctl.stops, "plan-2") || len(s.ctl.withouts) != 0 { + t.Fatalf("the controller was asked %v %v; want plan-2 ended whole", s.ctl.stops, s.ctl.withouts) + } + s.heard("plan-2") + z := s.delivery(s.ids["novox/z"]) + if z.State != Stopped || !strings.Contains(lastWhy(z), s.ids["novox/y"]) { + t.Fatalf("z is %s (%q); want it stopped, naming y's stop", z.State, lastWhy(z)) + } +} + +// Whatever brought it there, a walk that carries a stopped delivery's merge is asked to leave it out, and is never +// given its word while it carries it. +func TestAWalkCarryingAStoppedMergeIsAskedToLeaveItOutAndNeverLetGo(t *testing.T) { + s := shared459(t) + if _, err := s.h.StopWay(s.ids["novox/y"], "not wanted after all", "jochen", "again"); err != nil { + t.Fatal(err) + } + // A walk the controller made that carries y's merge anyway, waiting for its word, with z on it. + x := aWalk("plan-x", "novox/z", s.merges["novox/z"], true) + x.Created = x.Created.Add(10 * time.Minute) + x.Delivery.Merges = []WalkMerge{{Repository: "novox/y", Commit: s.merges["novox/y"]}, + {Repository: "novox/z", Commit: s.merges["novox/z"]}} + s.ctl.refuseWithout = true // it cannot be left out: the walk is stopped whole + s.h.WalkMoved(s.ctl.put(x)) + s.settleAll() + if want := "plan-x without novox/y@" + s.merges["novox/y"]; !containsString(s.ctl.withouts, want) { + t.Fatalf("the controller was asked %v; want %q", s.ctl.withouts, want) + } + if containsString(s.ctl.delivers, "plan-x") { + t.Fatalf("a walk carrying a stopped merge was let go: %v", s.ctl.delivers) + } + if !containsString(s.ctl.stops, "plan-x") { + t.Fatalf("refused the leave-out, the walk was not stopped whole: %v", s.ctl.stops) + } +} + +// A person's stop the controller cannot make leaves the delivery as it was, and says how to go on. +func TestAPersonsStopThatCannotBeMadeStopsNothing(t *testing.T) { + s := shared459(t) + s.ctl.refuseWithout = true + _, err := s.h.StopWay(s.ids["novox/y"], "not wanted after all", "jochen", "again") + if err == nil || !strings.Contains(err.Error(), "is not stopped") || !strings.Contains(err.Error(), "plans stop") { + t.Fatalf("the refused stop said %v", err) + } + if st := s.state(s.ids["novox/y"]); st != Delivering { + t.Fatalf("y is %s; want it delivering, its stop not made", st) + } +} + +// What a stop that left merges out says: the walk that walks the others, what it left out and what each contains, +// as the controller says it, what stays on which machines, and what will still deliver the stopped change. +func TestAStopSaysWhatStaysAndWhatStillCarriesIt(t *testing.T) { + said := againWords(StopAnswer{Stopped: "plan-2", WalkedAgainBy: "plan-3", WalkedAgain: []string{"novox/z@zz"}, + LeftOut: []WalkLeftOut{{Repository: "novox/b", Commit: "b1b1b1b1b1b1"}, + {Repository: "novox/b", Commit: "b2b2b2b2b2b2", Contains: "b1b1b1b1b1b1"}}, + Kept: []WalkKept{{Module: "lab", Machines: []string{"anchor"}}}, + StillCarriedBy: []string{"novox/b@b3b3 (in plan-4)"}}) + for _, want := range []string{"plan-3", "novox/z@zz", "b2b2b2b2b2b2 (contains b1b1b1b1b1b1)", "lab on anchor", + "stays on the machines", "novox/b@b3b3 (in plan-4)", "revert"} { + if !strings.Contains(said, want) { + t.Fatalf("the stop said %q; want %q in it", said, want) + } + } +} + +// A person's first answer names the deliveries that would be left out for moving a module this merge moves, and says +// the controller may leave out more. +func TestAPersonsFirstAnswerNamesWhatSharesItsModules(t *testing.T) { + s := shared459(t) + _, err := s.h.StopWay(s.ids["novox/y"], "x", "jochen", "") + if err == nil || !strings.Contains(err.Error(), "for moving a module this merge moves: "+s.ids["novox/z"]+" (moves app)") || + !strings.Contains(err.Error(), "may leave out more") { + t.Fatalf("the first answer said %v", err) + } +} + +// The safety net holds back only what a person or a group stopped: a delivery ended by what its walk did (the +// observed rows, which name nobody) is not asked out of a walk that carries its merge. +func TestTheSafetyNetLeavesAStrandedStopAlone(t *testing.T) { + s := shared459(t) + s.h.mu.Lock() + y := s.h.deliveries[s.ids["novox/y"]] + y.State = Stopped + y.Transitions = append(y.Transitions, Transition{At: s.now, From: Published, To: Stopped, Event: EvStop, + Why: "its walk ended done before its word"}) + s.h.mu.Unlock() + s.settleAll() + if len(s.ctl.withouts) != 0 { + t.Fatalf("a stop nobody asked for was held back: %v", s.ctl.withouts) + } +} + +// A walk the safety net holds — its leave-out and its whole stop both refused — says why on the deliveries waiting on +// it, in `waits`, and in `stalled` past their bound. +func TestAHeldWalkIsSaidOnItsDeliveriesAndStalls(t *testing.T) { + s := shared459(t) + if _, err := s.h.StopWay(s.ids["novox/y"], "not wanted after all", "jochen", "again"); err != nil { + t.Fatal(err) + } + s.ctl.refuseWithout, s.ctl.refuseStop = true, true + x := aWalk("plan-x", "novox/z", s.merges["novox/z"], true) + x.Created = x.Created.Add(10 * time.Minute) + x.Delivery.Merges = []WalkMerge{{Repository: "novox/y", Commit: s.merges["novox/y"]}, + {Repository: "novox/z", Commit: s.merges["novox/z"]}} + s.h.WalkMoved(s.ctl.put(x)) + s.settleAll() + z := s.delivery(s.ids["novox/z"]) + if !strings.Contains(z.HeldBack, "plan-x") || !strings.Contains(z.HeldBack, "refused") { + t.Fatalf("z says it is held back by %q; want plan-x and the refusal", z.HeldBack) + } + if w := s.h.waitsFor(z); w != z.HeldBack { + t.Fatalf("z waits for %q", w) + } + s.later(Bounds[Published].For + time.Minute) + s.settleAll() + for _, l := range s.h.Stalled() { + if l.ID == z.ID && strings.Contains(l.Says, "plan-x") { + return + } + } + t.Fatalf("z is not listed stalled naming plan-x: %+v", s.h.Stalled()) +} + +// A person's stop of a delivery whose merge waits in a batch asks that batch now to leave it out, and says so. +func TestAPersonsStopOfAMergeInABatchAsksTheBatch(t *testing.T) { + s := shared459(t) + if _, err := s.h.StopWay(s.ids["novox/y"], "not wanted after all", "jochen", "again"); err != nil { + t.Fatal(err) + } + s.heard("plan-2", "plan-2-again") + said, err := s.h.StopWay(s.ids["novox/z"], "this one too", "jochen", "") + if err != nil { + t.Fatal(err) + } + if want := "plan-2-again without novox/z@" + s.merges["novox/z"]; !containsString(s.ctl.withouts, want) || + !strings.Contains(said, "batch plan-2-again") { + t.Fatalf("said %q, asked %v; want the batch asked now, and said", said, s.ctl.withouts) + } +} diff --git a/modules/mesh-delivery/cmd/mesh-delivery/issue474_test.go b/modules/mesh-delivery/cmd/mesh-delivery/issue474_test.go new file mode 100644 index 00000000..797cffe1 --- /dev/null +++ b/modules/mesh-delivery/cmd/mesh-delivery/issue474_test.go @@ -0,0 +1,211 @@ +package main + +import ( + "strings" + "testing" + "time" +) + +// novox/hq issue 474: a merge batched behind a later one of its branch (ADR 0276) whose walk ended done before +// any word was asked — the controller judged the batch's files moved nothing it holds, "nothing to walk" — was +// left published, waiting for "its walk's start", which a done walk never has. Its walk's record settles it: it +// is delivered when that walk moved its modules, covered when a later delivered merge carried them (ADR 0270), +// and otherwise stopped, saying why — never left waiting. +// +// The shape is the night's own (2026-10-11): mesh-controller #215 (merged as fbc7bc97, its check's plan moving +// build-agent) and #217 (merged as 8305e4e3 a minute later, moving nothing) in one batch, walked as +// plan-1791681624611167730 at 8305e4e3: done, no tier, still saying it awaits mesh-delivery. + +const ( + ctl474 = "novox/mesh-controller" + head215 = "d2e9dc61590f950b" + merge215 = "fbc7bc976b3c115b" + head217 = "68fbd99e894443ce" + merge217 = "8305e4e3d126594d" + head219 = "ef551fdfb6b29d4a" + merge219 = "0a2e58c0704ccc09" + batchWalk = "plan-1791681624611167730" + walk219 = "plan-1791686195314774150" + nothingNote = "moved nothing the mesh holds: nothing to walk" +) + +var at215 = time.Date(2026, 10, 11, 1, 20, 19, 0, time.UTC) + +// theNight is #215 and #217 checked and merged, and their batch's walk done with no tier, as the controller said it. +func theNight(t *testing.T) *world { + w := newWorldOver(t, newMemStore(), newFakeController(), newFakeForge(), at215.Add(-time.Minute)) + p215 := pr(ctl474, 215, head215, "fix/431-make-check-runs-what-the-gate-runs", "Makefile") + p217 := pr(ctl474, 217, head217, "fix/432-a-test-reads-what-it-carries", "cmd/mesh-controller/check_test.go") + w.h.PullUpdated(p215) + w.h.PullUpdated(p217) + w.h.Checked(verdictMoving(ctl474, 215, head215, "build-agent")) + w.h.Checked(verdictMoving(ctl474, 217, head217)) + w.later(time.Minute) + w.h.PullMerged(mergedAt(p215, merge215, at215)) + w.later(time.Minute) + w.h.PullMerged(mergedAt(p217, merge217, at215.Add(56*time.Second))) + w.later(90 * time.Second) + cut := w.now + w.h.WalkMoved(w.ctl.put(Walk{ID: batchWalk, Repository: ctl474, Branch: "main", Commit: merge217, Created: cut, + Updated: cut, State: walkDone, Tiers: [][]string{}, Modules: map[string]*WalkModule{}, Note: nothingNote, + Delivery: &WalkDelivery{Awaits: "mesh-delivery", Merges: []WalkMerge{ + {Repository: ctl474, Commit: merge215, Carried: merge217}, {Repository: ctl474, Commit: merge217}}}})) + w.settleAll() + return w +} + +// The live delivery: its walk ended without moving build-agent, and a later merge (#219) delivered build-agent +// everywhere from main holding its merge — covered, from the walk's record and the forge's word, never waiting. +func TestAMergeWhoseBatchWalkEndedIsCoveredByALaterDeliveredMerge(t *testing.T) { + w := theNight(t) + id := IDOf(ctl474, head215) + if s := w.state(IDOf(ctl474, head217)); s != Delivered { + t.Fatalf("#217, which moves nothing, is %s", s) + } + w.forge.contains[merge215+" in "+merge219] = true + deliverLater(w, ctl474, 219, "main", head219, merge219, at215.Add(76*time.Minute), walkDone, "build-agent") + w.later(2 * time.Minute) + w.settleAll() + d := w.delivery(id) + if d.State != Covered { + t.Fatalf("#215, its walk %s done with nothing moved and build-agent delivered later by #219, is %s, waiting "+ + "for %q", batchWalk, d.State, w.h.waitsFor(d)) + } + if why := d.Transitions[len(d.Transitions)-1].Why; !strings.Contains(why, IDOf(ctl474, head219)) { + t.Fatalf("covered, but not by #219: %q", why) + } + if len(w.h.Stalled()) != 0 { + t.Fatalf("stalled still lists %+v", w.h.Stalled()) + } +} + +// With nothing later to carry its modules, it is stopped once the cover test says so and the walk ended past the +// grace — saying which walk ended without moving what — and not before. Stopped, it stays loud: `stalled` lists it +// until a later delivered merge carries its modules. +func TestAMergeWhoseBatchWalkEndedAndNothingCoversItIsStoppedSayingWhy(t *testing.T) { + w := theNight(t) + id := IDOf(ctl474, head215) + w.later(5 * time.Minute) + w.settleAll() + if s := w.state(id); s != Published { + t.Fatalf("within the grace it is %s", s) + } + w.later(strandedAfter) + w.settleAll() + d := w.delivery(id) + if d.State != Stopped { + t.Fatalf("its walk done without build-agent and nothing later carrying it, it is %s, waiting for %q", d.State, + w.h.waitsFor(d)) + } + why := d.Transitions[len(d.Transitions)-1].Why + for _, want := range []string{"never reached the machines", batchWalk, "build-agent", nothingNote} { + if !strings.Contains(why, want) { + t.Fatalf("its stop says %q, not %q", why, want) + } + } + lines := w.h.Stalled() + if len(lines) != 1 || lines[0].ID != id || !strings.Contains(lines[0].Says, "never reached the machines") { + t.Fatalf("stopped, it is not listed loudly: %+v", lines) + } + // A later merge that delivers build-agent ends the listing; the stop itself stays, final. + w.forge.contains[merge215+" in "+merge219] = true + deliverLater(w, ctl474, 219, "main", head219, merge219, w.now, walkDone, "build-agent") + w.later(2 * time.Minute) + w.settleAll() + if d := w.delivery(id); d.State != Stopped || d.CoveredBy == "" { + t.Fatalf("after #219 it is %s, covered by %q", d.State, d.CoveredBy) + } + if lines := w.h.Stalled(); len(lines) != 0 { + t.Fatalf("stalled still lists %+v", lines) + } +} + +// A later merge whose walk is still queued, and whose plan moves what this one's walk left unbuilt, holds the stop +// off: a stop is final, and that merge may yet cover it (review of issue 474). +func TestAMergeWhoseBatchWalkEndedWaitsForALaterMergeOnItsWay(t *testing.T) { + w := theNight(t) + id := IDOf(ctl474, head215) + p := pr(ctl474, 219, head219, "fix/334", "internal/builder/builder.go") + w.h.PullUpdated(p) + w.h.Checked(verdictMoving(ctl474, 219, head219, "build-agent", "mesh-controller")) + w.later(3 * time.Minute) + w.h.PullMerged(mergedAt(p, merge219, w.now)) + w.settleAll() + w.later(strandedAfter) + w.settleAll() + if d := w.delivery(id); d.State != Published { + t.Fatalf("with #219 on its way it is %s (%s)", d.State, d.Stranded) + } + // #219's walk delivers build-agent: covered. + w.forge.contains[merge215+" in "+merge219] = true + walk := aWalk(walk219, ctl474, merge219, true) + walk.Modules = map[string]*WalkModule{"build-agent": {}, "mesh-controller": {}} + w.h.WalkMoved(w.ctl.put(walk)) + w.settleAll() + walk = w.ctl.walk(walk219) + sent := w.now + for m := range walk.Modules { + walk.Modules[m] = &WalkModule{State: "built", SentAt: &sent} + } + walk.State = walkDone + w.h.WalkMoved(w.ctl.put(walk)) + w.later(2 * time.Minute) + w.settleAll() + if s := w.state(IDOf(ctl474, head219)); s != Delivered { + t.Fatalf("#219 is %s", s) + } + if d := w.delivery(id); d.State != Covered { + t.Fatalf("after #219 delivered build-agent it is %s, waiting for %q", d.State, w.h.waitsFor(d)) + } +} + +// A walk that ended before its word built nothing: a module its record names with why it was not sent is not read +// as delivered. A merge whose plan moves nothing, answered by such a walk, is delivered. +func TestAWalkThatEndedBeforeItsWordBuiltNothing(t *testing.T) { + for _, moves := range [][]string{{"build-agent"}, nil} { + w := newWorldOver(t, newMemStore(), newFakeController(), newFakeForge(), at215.Add(-time.Minute)) + p := pr(ctl474, 215, head215, "fix/x", "cmd/mesh-builder/main.go") + w.h.PullUpdated(p) + w.h.Checked(verdictMoving(ctl474, 215, head215, moves...)) + w.later(time.Minute) + w.h.PullMerged(mergedAt(p, merge215, at215)) + w.later(2 * time.Minute) + w.h.WalkMoved(w.ctl.put(Walk{ID: batchWalk, Repository: ctl474, Branch: "main", Commit: merge215, Created: w.now, + Updated: w.now, State: walkDone, Tiers: [][]string{{"build-agent"}}, + Modules: map[string]*WalkModule{"build-agent": {Why: "not sent: its walk never started"}}, + Delivery: &WalkDelivery{Awaits: "mesh-delivery", Merges: []WalkMerge{{Repository: ctl474, Commit: merge215}}}})) + w.settleAll() + d := w.delivery(IDOf(ctl474, head215)) + want := Published + if len(moves) == 0 { + want = Delivered + } + if d.State != want { + t.Fatalf("moving %v: it is %s, waiting for %q, not %s", moves, d.State, w.h.waitsFor(d), want) + } + } +} + +// A walk record kept before it named the modules it built, with tiers, is not read as building nothing: the walk is +// asked for again, and what it answers settles the delivery. +func TestAnOldWalkRecordIsAskedForAgain(t *testing.T) { + w := theNight(t) + id := IDOf(ctl474, head215) + w.h.mu.Lock() + d := w.h.deliveries[id] + d.Walk.Moved, d.Walk.MovedSaid, d.Walk.Tiers = nil, false, 1 + w.h.mu.Unlock() + if got := w.h.waitsFor(w.delivery(id)); !strings.Contains(got, "walk's record") { + t.Fatalf("an old record waits for %q", got) + } + w.later(time.Minute) + w.settleAll() + if d := w.delivery(id); !d.Walk.MovedSaid || d.Walk.Tiers != 0 { + t.Fatalf("the walk was not read again: %+v", d.Walk) + } + w.later(strandedAfter) + w.settleAll() + if s := w.state(id); s != Stopped { + t.Fatalf("read again, it is %s", s) + } +} diff --git a/modules/mesh-delivery/cmd/mesh-delivery/issue480_test.go b/modules/mesh-delivery/cmd/mesh-delivery/issue480_test.go new file mode 100644 index 00000000..b999c478 --- /dev/null +++ b/modules/mesh-delivery/cmd/mesh-delivery/issue480_test.go @@ -0,0 +1,134 @@ +package main + +import ( + "strings" + "testing" + "time" +) + +// novox/hq issue 480: a delivery stopped because its walk ended done before its word without building its modules +// (issue 474) is still read for a later merge that delivers them, and kept CoveredBy once one did. Its state stays +// stopped — a final state never moves, as for a covered superseded member (issue 442) — but the forge's status, its +// view, its note and `show` went on saying "stopped" for a change that reached every machine. Once covered they say +// covered, and by which merge. And while `stalled` lists such a stop, `deliveries` lists it too, past a day. + +// strandedNight is theNight with #215 stopped, its walk past the grace with nothing later carrying build-agent. +func strandedNight(t *testing.T) (*world, string) { + w := theNight(t) + id := IDOf(ctl474, head215) + w.later(strandedAfter + 5*time.Minute) + w.settleAll() + if d := w.delivery(id); d.State != Stopped || d.Stranded == "" { + t.Fatalf("#215 is %s, stranded %q", d.State, d.Stranded) + } + return w, id +} + +func TestAStopALaterMergeCoversReadsCoveredByThatMerge(t *testing.T) { + w, id := strandedNight(t) + if got := statusOn(w, ctl474, head215); !strings.HasPrefix(got, "error stopped: ") { + t.Fatalf("stopped, its status says %q", got) + } + w.forge.contains[merge215+" in "+merge219] = true + by := deliverLater(w, ctl474, 219, "main", head219, merge219, w.now, walkDone, "build-agent") + w.later(2 * time.Minute) + w.settleAll() + d := w.delivery(id) + if d.State != Stopped || !strings.Contains(d.CoveredBy, by) { + t.Fatalf("after #219 it is %s, covered by %q", d.State, d.CoveredBy) + } + got := statusOn(w, ctl474, head215) + if !strings.HasPrefix(got, "success covered") || !strings.Contains(got, by) { + t.Fatalf("covered by %s, its status still says %q", by, got) + } + shown, err := w.h.Show(id) + if err != nil { + t.Fatal(err) + } + if reads, _ := shown.(map[string]any)["reads"].(string); !strings.HasPrefix(reads, "covered") || !strings.Contains(reads, by) { + t.Fatalf("show reads %q", reads) + } + view := w.forge.views[ctl474+"#215"] + if !strings.Contains(view, "Covered") || !strings.Contains(view, by) { + t.Fatalf("its view does not say it was covered by %s:\n%s", by, view) + } + noted := false + for _, l := range w.forge.notesOn(merge215) { + noted = noted || strings.Contains(l, "covered") && strings.Contains(l, by) + } + if !noted { + t.Fatalf("its merge's note does not say it was covered: %v", w.forge.notesOn(merge215)) + } + // Told once: further ticks owe nothing more. + w.later(time.Hour) + w.settleAll() + if d := w.delivery(id); len(d.Owed) != 0 || !d.CoverTold { + t.Fatalf("still owes %+v, told %v", d.Owed, d.CoverTold) + } +} + +// A stop covered before this was built (CoveredBy kept, nothing told) is told after a restart, from what was kept. +func TestACoveredStopKeptBeforeIsToldAfterARestart(t *testing.T) { + w, id := strandedNight(t) + w.h.mu.Lock() + d := w.h.deliveries[id] + d.CoveredBy = IDOf(ctl474, head219) + " (merged as " + merge219[:8] + ")" + if err := w.store.PutDelivery(d); err != nil { + t.Fatal(err) + } + w.h.mu.Unlock() + again := newWorldOver(t, w.store, w.ctl, w.forge, w.now.Add(time.Hour)) + again.settleAll() + if got := statusOn(again, ctl474, head215); !strings.HasPrefix(got, "success covered") || + !strings.Contains(got, IDOf(ctl474, head219)) { + t.Fatalf("after a restart its status says %q", got) + } + if d := again.delivery(id); d.State != Stopped || len(d.Transitions) != len(w.delivery(id).Transitions) { + t.Fatalf("a restart moved it: %s", d.State) + } +} + +// Only a stop for a walk that never built its modules reads covered: a person's stop, or one the cover test has +// not answered, keeps its error. +func TestOnlyACoveredStrandedStopReadsCovered(t *testing.T) { + for _, d := range []*Delivery{ + {State: Stopped, CoveredBy: "x"}, // a person's stop, with a cover found while it was held + {State: Stopped, Stranded: "never reached"}, // stranded, not covered + } { + if state, _ := statusOf(d); state != "error" { + t.Fatalf("%+v reads %s", d, state) + } + if d.coveredStop() { + t.Fatalf("%+v reads as a covered stop", d) + } + } +} + +func TestAStopStalledListsStaysInDeliveries(t *testing.T) { + w, id := strandedNight(t) + w.later(30 * time.Hour) + w.settleAll() + listed := func() *Line { + for _, l := range w.h.Deliveries("", "", "", false) { + if l.ID == id { + return &l + } + } + return nil + } + if len(w.h.Stalled()) != 1 { + t.Fatalf("stalled lists %+v", w.h.Stalled()) + } + l := listed() + if l == nil || !l.Stalled || !strings.Contains(l.Waits, "build-agent") { + t.Fatalf("listed by stalled, deliveries shows %+v", l) + } + // Covered, it ends like any final delivery: a day after its stop. + w.forge.contains[merge215+" in "+merge219] = true + deliverLater(w, ctl474, 219, "main", head219, merge219, w.now, walkDone, "build-agent") + w.later(2 * time.Minute) + w.settleAll() + if len(w.h.Stalled()) != 0 || listed() != nil { + t.Fatalf("covered, stalled lists %+v and deliveries %+v", w.h.Stalled(), listed()) + } +} diff --git a/modules/mesh-delivery/cmd/mesh-delivery/issue529_test.go b/modules/mesh-delivery/cmd/mesh-delivery/issue529_test.go new file mode 100644 index 00000000..f7778a59 --- /dev/null +++ b/modules/mesh-delivery/cmd/mesh-delivery/issue529_test.go @@ -0,0 +1,142 @@ +package main + +import ( + "strings" + "testing" + "time" +) + +// novox/hq issue 362's note of 2026-10-11 (the nzbgeek case) and issue 529: mesh-media-catalog #18 added a module +// whose delivery plan said "sent nowhere". This owner ended its delivery at once, "nothing for a walk to move", but +// the controller opened a walk for the merge that moved modules to a machine and waited for this owner's word. +// With its delivery ended, nothing gave the walk its word: it held 39 minutes and two walks queued behind it. + +const ( + media = "novox/mesh-media-catalog" + headNzb = "abababab00001818" + mergeNzb = "cdcdcdcd00001818" + nzbWalkID = "plan-1791748193836841649" +) + +var atNzb = time.Date(2026, 10, 11, 19, 43, 0, 0, time.UTC) + +// nzbgeekMerged is #18 checked on a plan that builds a new module and sends it nowhere, and merged: its delivery +// ends at once, nothing for a walk to move. +func nzbgeekMerged(t *testing.T) *world { + t.Helper() + w := newWorldOver(t, newMemStore(), newFakeController(), newFakeForge(), atNzb.Add(-time.Minute)) + p := pr(media, 18, headNzb, "feat/nzbgeek", "modules/nzbgeek/module.json") + w.h.PullUpdated(p) + v := verdict(media, 18, headNzb, "pass") + v.Plan = &DeliveryPlan{Repository: media, Base: "main", New: []string{"modules/nzbgeek"}, + Summary: "builds new: modules/nzbgeek, sent nowhere; no bus step"} + w.h.Checked(v) + w.later(time.Minute) + w.h.PullMerged(mergedAt(p, mergeNzb, atNzb)) + w.settleAll() + if d := w.delivery(IDOf(media, headNzb)); d.State != Delivered { + t.Fatalf("#18 is %s, not delivered as nothing for a walk to move", d.State) + } + return w +} + +// nzbWalk is the walk the controller cut for the merge's batch: it names the merge and moves nzbgeek and nzbget. +func nzbWalk(created time.Time, merges ...WalkMerge) Walk { + w := Walk{ID: nzbWalkID, Repository: media, Branch: "main", Commit: mergeNzb, Created: created, State: walkBuilding, + Tiers: [][]string{{"nzbgeek", "nzbget"}}, Modules: map[string]*WalkModule{"nzbgeek": {}, "nzbget": {}}, + Delivery: &WalkDelivery{Awaits: "mesh-delivery", Merges: merges}} + return w +} + +// **A walk answering only merges that ended as nothing for a walk to move is named as held by nothing** (novox/hq +// issues 362 and 529): never let go by this owner — nothing judged what it moves — and said, by `stalled` with +// `walks`, with why, so the controller's `plans` says so instead of pointing nowhere. +func TestAWalkOfNothingToMoveMergesIsNamedAsHeldByNothing(t *testing.T) { + w := nzbgeekMerged(t) + w.later(90 * time.Second) + w.h.WalkMoved(w.ctl.put(nzbWalk(w.now, WalkMerge{Repository: media, Commit: mergeNzb}))) + w.settleAll() + if len(w.ctl.delivers) != 0 { + t.Fatalf("a walk no live delivery holds was let go: %v", w.ctl.delivers) + } + lines := w.h.WalksWaiting() + if len(lines) != 1 || lines[0].Walk != nzbWalkID || lines[0].State != Unheld || + !strings.Contains(lines[0].Says, "ended as nothing-to-move") || !strings.Contains(lines[0].Says, IDOf(media, headNzb)) || + !strings.Contains(lines[0].Says, "nzbget") { + t.Fatalf("the walk is said as %+v", lines) + } +} + +// **The status names each held delivery, why, and the verb that releases it** (novox/hq issue 529): "held: 1" +// alone pointed a reader nowhere. +func TestTheStatusNamesAHeldDeliveryAndWhy(t *testing.T) { + w := newWorldOver(t, newMemStore(), newFakeController(), newFakeForge(), atNzb) + p := pr(media, 22, headNzb, "adr-0307-state-history", "modules/nzbgeek/module.json") + w.h.PullUpdated(p) + w.h.PullMerged(mergedAt(p, mergeNzb, atNzb)) // merged before any verdict: held for a person + w.settleAll() + id := IDOf(media, headNzb) + d := w.delivery(id) + if d.State != Held { + t.Fatalf("#22 is %s", d.State) + } + held, _ := w.h.Status("listening")["held"].([]string) + if len(held) != 1 || !strings.Contains(held[0], id) || !strings.Contains(held[0], d.HeldWhy) || + !strings.Contains(held[0], "mesh-delivery.release") { + t.Fatalf("the status says held %q", held) + } +} + +// **A walk waiting for this owner's word is told why** (novox/hq issue 529): asked with `walks`, `stalled` says of +// each walk waiting for its word what holds it — a held delivery, naming it, its reason and `mesh-delivery.release`, +// or the deliveries not at their turn — so the controller's `plans` points at the person's decision, not at a hand +// start. +func TestAWaitingWalkIsToldWhatHoldsIt(t *testing.T) { + w := newWorldOver(t, newMemStore(), newFakeController(), newFakeForge(), atDu.Add(5*time.Second)) + cc := pr(cat, 101, headCC, "feat/claude-code", "modules/claude-code/module.json") + du := pr(cat, 102, headDu, "feat/dunst", "modules/dunst/module.json") + w.h.PullUpdated(cc) + w.h.PullUpdated(du) + w.h.Checked(verdictMoving(cat, 101, headCC, "claude-code")) + w.h.PullMerged(mergedAt(cc, mergeCC, atCC)) + w.h.PullMerged(mergedAt(du, mergeDu, atDu)) // unchecked: held for a person + walk := catWalk("plan-9", mergeDu, atDu.Add(5*time.Second), "claude-code", "dunst") + walk.Delivery.Merges = []WalkMerge{{Repository: cat, Commit: mergeCC, Carried: mergeDu}, {Repository: cat, Commit: mergeDu}} + w.h.WalkMoved(w.ctl.put(walk)) + w.settleAll() + idDu := IDOf(cat, headDu) + lines := w.h.WalksWaiting() + if len(lines) != 1 || lines[0].Walk != "plan-9" || lines[0].State != Held || lines[0].ID != idDu || + !strings.Contains(lines[0].Says, w.delivery(idDu).HeldWhy) || !strings.Contains(lines[0].Says, "mesh-delivery.release") { + t.Fatalf("the waiting walk is said as %+v", lines) + } + // Not by default: `stalled` stays what the self-check raises conditions for. + for _, l := range w.h.Stalled() { + if l.Walk == "plan-9" { + t.Fatalf("stalled lists the waiting walk without being asked: %+v", l) + } + } + // Released, its word is asked: nothing holds it any more. + if _, err := w.h.Release(idDu, "checked by hand", "jochen"); err != nil { + t.Fatal(err) + } + w.h.WalkMoved(w.ctl.walk("plan-9")) + w.settleAll() + if lines := w.h.WalksWaiting(); len(lines) != 0 { + t.Fatalf("a walk let go is still said waiting: %+v", lines) + } +} + +// A walk waiting while a delivery it carries is not at its turn says so, naming the delivery — what was said only +// in this owner's journal (novox/hq issue 529, note 2). +func TestAWaitingWalkSaysWhichDeliveryIsNotAtItsTurn(t *testing.T) { + w := newWorldOver(t, newMemStore(), newFakeController(), newFakeForge(), atNzb) + w.h.mu.Lock() + w.h.waitsWhy = map[string]string{"plan-7": "novox/x@1 (published) carried by it not at its turn"} + w.h.walks["plan-7"] = seenWalk{walk: aWalk("plan-7", "novox/x", "1111", true), at: atNzb} + w.h.mu.Unlock() + lines := w.h.WalksWaiting() + if len(lines) != 1 || lines[0].Walk != "plan-7" || !strings.Contains(lines[0].Says, "not at its turn") { + t.Fatalf("the waiting walk is said as %+v", lines) + } +} diff --git a/modules/mesh-delivery/cmd/mesh-delivery/issue611_test.go b/modules/mesh-delivery/cmd/mesh-delivery/issue611_test.go new file mode 100644 index 00000000..dd12f090 --- /dev/null +++ b/modules/mesh-delivery/cmd/mesh-delivery/issue611_test.go @@ -0,0 +1,150 @@ +package main + +import ( + "sort" + "strings" + "testing" + "time" +) + +// novox/hq issue 611: mesh-controller #293 changed one test file. Its check's plan, made while another walk built +// build-agent and route-proxy, read them whole and named them; at the cut their recorded build sources held none of +// its files, so its walk was done with no tier, "moved nothing the mesh holds: nothing to walk". The stranded rule +// (issue 474) read that as "ended without building build-agent, route-proxy", stopped it, and the operator was asked +// about a change already on every machine. The controller now says per module that the build matched what runs +// (the walk's delivery.unchanged), and such a delivery is delivered: its merge moves nothing (ADR 0270 §1). A walk +// that says neither — not built, not unchanged — still stops it. + +const ( + head293 = "94c928b4c7e1d2aa" + merge293 = "8f1aec0f2b9d6e11" + walk293 = "plan-1791867648768608552" +) + +var at293 = time.Date(2026, 10, 13, 5, 10, 0, 0, time.UTC) + +// matched is the why the controller gives a module it says unchanged. +const matched = "its build source in novox/mesh-controller holds none of the 1 changed file(s) " + + "(e.g. internal/builder/replay363_test.go): the build the mesh holds is its build" + +// replay293 is #293 checked moving build-agent and route-proxy, merged, and its walk done with no tier, saying the +// modules named unchanged. +func replay293(t *testing.T, unchanged map[string]string) *world { + t.Helper() + w := newWorldOver(t, newMemStore(), newFakeController(), newFakeForge(), at293.Add(-time.Minute)) + p := pr(ctl474, 293, head293, "fix/363-replay", "internal/builder/replay363_test.go") + w.h.PullUpdated(p) + w.h.Checked(verdictMoving(ctl474, 293, head293, "build-agent", "route-proxy")) + w.later(time.Minute) + w.h.PullMerged(mergedAt(p, merge293, at293)) + w.later(90 * time.Second) + note := nothingNote + if len(unchanged) > 0 { + note += "; unchanged, the build the mesh holds is theirs: " + strings.Join(sortedKeys(unchanged), ", ") + } + w.h.WalkMoved(w.ctl.put(Walk{ID: walk293, Repository: ctl474, Branch: "main", Commit: merge293, Created: w.now, + Updated: w.now, State: walkDone, Tiers: [][]string{}, Modules: map[string]*WalkModule{}, Note: note, + Delivery: &WalkDelivery{Awaits: "mesh-delivery", Merges: []WalkMerge{{Repository: ctl474, Commit: merge293}}, + Unchanged: unchanged}})) + w.settleAll() + return w +} + +// The live case: every module its plan named, the walk said matched what runs. Delivered at once, naming them, a +// success on the pull request, and nothing listed by stalled or asked of the operator, then or past the grace. +func TestAMergeWhoseModulesMatchWhatRunsIsDelivered(t *testing.T) { + w := replay293(t, map[string]string{"build-agent": matched, "route-proxy": matched}) + id := IDOf(ctl474, head293) + d := w.delivery(id) + if d.State != Delivered { + t.Fatalf("#293, its modules said unchanged by its walk, is %s, waiting for %q", d.State, w.h.waitsFor(d)) + } + why := d.Transitions[len(d.Transitions)-1].Why + for _, want := range []string{walk293, "build-agent, route-proxy", "matched what runs"} { + if !strings.Contains(why, want) { + t.Fatalf("delivered as %q, not saying %q", why, want) + } + } + if d.Stranded != "" { + t.Fatalf("it was read as stranded: %q", d.Stranded) + } + status := w.forge.statuses["novox/mesh-controller@"+head293+" mesh/delivery"] + if !strings.HasPrefix(status, "success ") { + t.Fatalf("mesh/delivery says %q", status) + } + w.later(strandedAfter + time.Minute) + w.settleAll() + if s := w.state(id); s != Delivered { + t.Fatalf("past the grace it is %s", s) + } + if lines := w.h.Stalled(); len(lines) != 0 { + t.Fatalf("stalled lists %+v", lines) + } +} + +// A walk that said only one of its modules unchanged left the other unbuilt: it is still stopped past the grace, +// naming only the module nothing delivered. +func TestAMergeWithAModuleNeitherBuiltNorUnchangedIsStillStopped(t *testing.T) { + w := replay293(t, map[string]string{"build-agent": matched}) + id := IDOf(ctl474, head293) + if s := w.state(id); s != Published { + t.Fatalf("within the grace it is %s", s) + } + w.later(strandedAfter + time.Minute) + w.settleAll() + d := w.delivery(id) + if d.State != Stopped { + t.Fatalf("route-proxy neither built nor unchanged, it is %s", d.State) + } + why := d.Transitions[len(d.Transitions)-1].Why + if !strings.Contains(why, "without building route-proxy (") { + t.Fatalf("its stop says %q", why) + } +} + +// A walk that says nothing unchanged — a controller from before, or one that built nothing at all — stops it as +// before (issue 474). +func TestAMergeWhoseWalkSaysNothingUnchangedIsStillStopped(t *testing.T) { + w := replay293(t, nil) + w.later(strandedAfter + time.Minute) + w.settleAll() + if d := w.delivery(IDOf(ctl474, head293)); d.State != Stopped || + !strings.Contains(d.Stranded, "without building build-agent, route-proxy") { + t.Fatalf("it is %s: %q", d.State, d.Stranded) + } +} + +// Batched behind a later merge of its branch (issue 474's shape), a merge whose module the walk said unchanged moves +// nothing at either commit: delivered, not covered by a later merge that delivered nothing for it. +func TestABatchedMergeWhoseModuleMatchesWhatRunsIsDelivered(t *testing.T) { + w := newWorldOver(t, newMemStore(), newFakeController(), newFakeForge(), at215.Add(-time.Minute)) + p215 := pr(ctl474, 215, head215, "fix/431-make-check-runs-what-the-gate-runs", "Makefile") + p217 := pr(ctl474, 217, head217, "fix/432-a-test-reads-what-it-carries", "cmd/mesh-controller/check_test.go") + w.h.PullUpdated(p215) + w.h.PullUpdated(p217) + w.h.Checked(verdictMoving(ctl474, 215, head215, "build-agent")) + w.h.Checked(verdictMoving(ctl474, 217, head217)) + w.later(time.Minute) + w.h.PullMerged(mergedAt(p215, merge215, at215)) + w.later(time.Minute) + w.h.PullMerged(mergedAt(p217, merge217, at215.Add(56*time.Second))) + w.later(90 * time.Second) + w.h.WalkMoved(w.ctl.put(Walk{ID: batchWalk, Repository: ctl474, Branch: "main", Commit: merge217, Created: w.now, + Updated: w.now, State: walkDone, Tiers: [][]string{}, Modules: map[string]*WalkModule{}, Note: nothingNote, + Delivery: &WalkDelivery{Awaits: "mesh-delivery", Merges: []WalkMerge{ + {Repository: ctl474, Commit: merge215, Carried: merge217}, {Repository: ctl474, Commit: merge217}}, + Unchanged: map[string]string{"build-agent": "its build source holds none of the 2 changed file(s)"}}})) + w.settleAll() + if d := w.delivery(IDOf(ctl474, head215)); d.State != Delivered { + t.Fatalf("#215, build-agent said unchanged by its batch's walk, is %s, waiting for %q", d.State, w.h.waitsFor(d)) + } +} + +func sortedKeys(m map[string]string) []string { + var out []string + for k := range m { + out = append(out, k) + } + sort.Strings(out) + return out +} diff --git a/modules/mesh-delivery/cmd/mesh-delivery/issue747_test.go b/modules/mesh-delivery/cmd/mesh-delivery/issue747_test.go new file mode 100644 index 00000000..4c159e07 --- /dev/null +++ b/modules/mesh-delivery/cmd/mesh-delivery/issue747_test.go @@ -0,0 +1,258 @@ +package main + +import ( + "strings" + "testing" + "time" +) + +// novox/hq issue 747: in the group fix/a-module-moves-only-when-its-program-changes, the controller's member #336 +// (merged as 5d22d7df) failed at 22:32 during an outage of the forge's tools. The walk of the controller's next merge, +// #339 at 60cc9ec3, which contains 5d22d7df, failed with it and was resumed to done by a rebuild. mesh-delivery still +// held #336 failed, so its group stopped every later walk of the catalogue that carried the catalogue member's merge +// 2edef5a0: the catalogue could deliver nothing. A failed or stopped delivery is covered once a later finished walk of +// its repository that contains its merge built its modules, and a group stops a walk only for a member whose change is +// still undelivered. +// +// The shape is the live state of 2026-10-14 23:07, as mesh-delivery.show gave it, read back after a restart: the +// controller holds none of the walks of the night any more, only the walk of the catalogue's next merge. + +const ( + g747 = "fix/a-module-moves-only-when-its-program-changes" + ctl747, cat747 = "novox/mesh-controller", "novox/mesh-catalog" + head336, merge336 = "b34055f580d4e2a4", "5d22d7df86c73452" + head339, merge339 = "e4acab32d6a20ade", "60cc9ec363dfa7c6" + head301, merge301 = "d94764f00168471f", "2edef5a00ae05605" + merge307 = "0f76b6e2aaaaaaaa" +) + +var ( + at747 = time.Date(2026, 10, 14, 20, 21, 6, 0, time.UTC) + id336, id339, id301 = IDOf(ctl747, head336), IDOf(ctl747, head339), IDOf(cat747, head301) + unchangedRouteProxy747 = map[string]string{"route-proxy": "its build source holds none of the changed files"} +) + +// night747 is the stuck state, with the forge saying, or not, that 60cc9ec3 contains 5d22d7df. +func night747(t *testing.T, contained bool) *world { return night747With(t, contained, nil) } + +// night747With is night747 with #339's walk having built only the modules given, saying the others unchanged. +func night747With(t *testing.T, contained bool, built []string) *world { + t.Helper() + return night747Over(t, func(f *fakeForge) { f.contains[merge336+" in "+merge339] = contained }, built) +} + +// night747Over is night747With with the forge set as given: one that is not told the pair cannot say. +func night747Over(t *testing.T, set func(*fakeForge), built []string) *world { + t.Helper() + at := at747 + failedAt := at.Add(11 * time.Minute) + store := newMemStore() + moved339, unchanged339 := []string{"build-agent", "mesh-controller"}, unchangedRouteProxy747 + if built != nil { + moved339, unchanged339 = built, map[string]string{} + for _, m := range []string{"build-agent", "mesh-controller", "route-proxy"} { + if !containsString(built, m) { + unchanged339[m] = "its build source holds none of the files #339 changed" + } + } + } + m336 := &Delivery{ID: id336, Repository: ctl747, Commit: head336, Number: 336, Base: "main", Branch: g747, Group: g747, + State: Failed, Since: failedAt, Created: at.Add(-time.Hour), MergedAs: merge336, MergedAt: at, + AnsweredBy: merge336, Check: &Verdict{Gate: "pass"}, + Plan: planMoving(ctl747, "build-agent", "mesh-controller", "route-proxy"), + Walk: &WalkSeen{ID: "plan-336", Commit: merge336, State: walkFailed, Branch: "main", MovedSaid: true, + Note: "mesh-controller failed to build in tier 0", Created: at.Add(7 * time.Minute), Tiers: 2, + Unchanged: unchangedRouteProxy747}, + Transitions: []Transition{{At: failedAt, From: Delivering, To: Failed, Event: EvFailed, + Why: "its walk failed, carrying its own commit"}}} + m339 := &Delivery{ID: id339, Repository: ctl747, Commit: head339, Number: 339, Base: "main", Branch: "fix/726", + State: Failed, Since: failedAt, Created: at.Add(-10 * time.Minute), MergedAs: merge339, MergedAt: at.Add(7 * time.Minute), + AnsweredBy: merge339, Check: &Verdict{Gate: "pass"}, Plan: planMoving(ctl747, "mesh-controller"), + // Its walk failed with #336's, and a rebuild resumed it to done: the record it keeps says so. + Walk: &WalkSeen{ID: "plan-339", Commit: merge339, State: walkDone, Branch: "main", MovedSaid: true, + Moved: moved339, Unchanged: unchanged339, + Created: at.Add(11 * time.Minute), Tiers: 2, Tier: 2}, + Transitions: []Transition{{At: failedAt, From: Delivering, To: Failed, Event: EvFailed, + Why: "its walk failed, carrying its own commit"}}} + m301 := &Delivery{ID: id301, Repository: cat747, Commit: head301, Number: 301, Base: "main", Branch: g747, Group: g747, + State: Stopped, Since: failedAt, Created: at.Add(-2 * time.Hour), MergedAs: merge301, MergedAt: at.Add(7 * time.Second), + Check: &Verdict{Gate: "pass"}, Plan: planMoving(cat747, "build-agent", "route-proxy"), + Transitions: []Transition{{At: failedAt, From: Ready, To: Stopped, Event: EvStop, By: byOwner, + Why: "its group " + g747 + " stopped: " + id336 + " failed"}}} + for _, d := range []*Delivery{m336, m339, m301} { + if err := store.PutDelivery(d); err != nil { + t.Fatal(err) + } + } + g := &Group{ID: g747, Members: []string{id301, id336}, Closed: true, Said: "failed", + Check: &GroupCheck{Asked: "check-1", At: at.Add(-time.Minute), + Verdict: "pass", Summary: "both compose", Commits: commitsOf([]*Delivery{m301, m336})}} + if err := store.PutGroup(g); err != nil { + t.Fatal(err) + } + ctlr := newFakeController() + // The catalogue's next merge: its walk carries 2edef5a0, merged before it and never walked, and waits for its word. + ctlr.put(Walk{ID: "plan-cat", Repository: cat747, Branch: "main", Commit: merge307, Created: at.Add(46 * time.Minute), + State: walkBuilding, Tiers: [][]string{{"gitea"}}, Modules: map[string]*WalkModule{"gitea": {}}, + Delivery: &WalkDelivery{Awaits: "mesh-delivery", Merges: []WalkMerge{ + {Repository: cat747, Commit: merge301, Carried: merge307}, {Repository: cat747, Commit: merge307}}}}) + ctlr.mergedAt[merge301], ctlr.mergedAt[merge307] = m301.MergedAt, at.Add(46*time.Minute) + forge := newFakeForge() + set(forge) + w := newWorldOver(t, store, ctlr, forge, at.Add(47*time.Minute)) + if err := w.h.Reconcile(); err != nil { + t.Fatal(err) + } + w.settleAll() + return w +} + +func groupState747(w *world) (string, string) { + g := w.h.groups[g747] + return GroupState(g, w.h.membersOf(g)) +} + +// **A failed member is covered by a later finished walk that contains it**: #339's walk, done at 60cc9ec3, which the +// forge says contains 5d22d7df, built build-agent and mesh-controller, and #336's own walk said route-proxy unchanged. +// #336 stays failed, reads covered by that walk on its pull request, and its group no longer reads failed. #339, whose +// own walk was resumed to done, is covered by it too. +func TestAFailedMemberIsCoveredByALaterDoneWalkThatContainsIt(t *testing.T) { + w := night747(t, true) + d := w.delivery(id336) + if state, desc := statusOf(d); d.State != Failed || state != "success" || !strings.Contains(desc, "covered after its failure") || + !strings.Contains(desc, "plan-339") { + t.Fatalf("#336's status reads %s %q", state, desc) + } + if s := statusOn(w, ctl747, head336); !strings.HasPrefix(s, "success") { + t.Fatalf("the forge was told %q of #336", s) + } + if state, desc := statusOf(w.delivery(id339)); state != "success" || !strings.Contains(desc, "plan-339") { + t.Fatalf("#339 reads %s %q; want it covered by its own walk, resumed to done", state, desc) + } + if state, why := groupState747(w); state == "failed" || state == "stopped" { + t.Fatalf("the group reads %s: %s", state, why) + } + // Told once, and kept: a restart tells nothing again. + notes := len(w.forge.notesOn(merge336)) + again := newWorldOver(t, w.store, w.ctl, w.forge, w.now.Add(time.Minute)) + again.settleAll() + if state, _ := statusOf(again.delivery(id336)); len(w.forge.notesOn(merge336)) != notes || state != "success" { + t.Fatalf("a restart told it again or forgot it: %d notes, was %d", len(w.forge.notesOn(merge336)), notes) + } +} + +// **The group then no longer stops the other repository's walk**: the catalogue member was stopped for #336 alone, +// so the walk of the catalogue's next merge, which carries its merge, is neither stopped nor held back, and the +// group waits for that walk to deliver the catalogue member's change. +func TestACoveredMemberNoLongerStopsTheOtherRepositorysWalk(t *testing.T) { + w := night747(t, true) + if len(w.ctl.withouts) > 0 || containsString(w.ctl.stops, "plan-cat") { + t.Fatalf("the catalogue's walk was stopped: without %v, stops %v", w.ctl.withouts, w.ctl.stops) + } + if stopped, _ := w.h.carriesStopped("plan-cat"); len(stopped) > 0 { + t.Fatalf("the catalogue's walk is held back for %v", stopped) + } + if state, why := groupState747(w); state != "delivering" || !strings.Contains(why, id301) { + t.Fatalf("the group reads %s: %s; want delivering, the catalogue member waiting for a later walk", state, why) + } + // Once a later walk of the catalogue that contains its merge built its modules, it is covered, and the group is + // delivered. + w.forge.contains[merge301+" in "+merge307] = true + r := w.ctl.walk("plan-cat") + sent := w.now + r.Modules = map[string]*WalkModule{} + for _, m := range []string{"build-agent", "route-proxy", "gitea"} { + r.Modules[m] = &WalkModule{State: "built", SentAt: &sent} + } + r.State = walkDone + w.h.WalkMoved(w.ctl.put(r)) + w.later(time.Minute) + w.settleAll() + if state, desc := statusOf(w.delivery(id301)); state != "success" || !strings.Contains(desc, "plan-cat") { + t.Fatalf("the catalogue member reads %s %q", state, desc) + } + if state, why := groupState747(w); state != "delivered" { + t.Fatalf("the group reads %s: %s", state, why) + } +} + +// **A failed member not contained still stops it**: the forge says 60cc9ec3 does not contain 5d22d7df, so #336's change +// never reached the machines; it stays failed and uncovered, its group failed, and the catalogue's walk that carries +// the stopped member's merge is asked to leave it out. +func TestAFailedMemberNotContainedStillStopsTheOtherRepositorysWalk(t *testing.T) { + w := night747(t, false) + d := w.delivery(id336) + if state, desc := statusOf(d); d.State != Failed || state != "failure" { + t.Fatalf("#336 is %s and reads %s %q", d.State, state, desc) + } + if state, _ := groupState747(w); state != "failed" { + t.Fatalf("the group reads %s", state) + } + if len(w.ctl.withouts) == 0 || !strings.HasPrefix(w.ctl.withouts[0], "plan-cat without "+cat747+"@"+merge301) { + t.Fatalf("the catalogue's walk was not asked to leave the stopped merge out: %v", w.ctl.withouts) + } +} + +// **A later walk that only said a module unchanged covers nothing of an earlier merge that moves it**: #339's walk +// built mesh-controller alone and said build-agent unchanged for #339's own files, which says nothing of #336's; only +// #336's own walk says that of its change (route-proxy). #336 stays failed and its group still stops the catalogue. +func TestALaterWalksUnchangedCoversNoModuleOfAnEarlierMerge(t *testing.T) { + w := night747With(t, true, []string{"mesh-controller"}) + if state, desc := statusOf(w.delivery(id336)); state != "failure" { + t.Fatalf("#336 reads %s %q, though no walk built build-agent for it", state, desc) + } + if state, _ := groupState747(w); state != "failed" { + t.Fatalf("the group reads %s", state) + } + if len(w.ctl.withouts) == 0 { + t.Fatal("the catalogue's walk was not asked to leave the stopped merge out") + } +} + +// **A forge that cannot say holds the walk only within the bound** (review of issue 747): while the forge errors on +// whether 60cc9ec3 contains 5d22d7df, the catalogue's walk is held back, loudly, neither stopped nor asked to leave the +// merge out, and #336 still reads failed. Past the bound the forge's silence holds nothing back any more: the walk +// is asked to leave the stopped merge out, as with no later walk at all. +func TestAForgeThatCannotSayHoldsTheWalkOnlyWithinTheBound(t *testing.T) { + for _, c := range []struct { + name string + set func(*fakeForge) + }{ + {"erroring", func(f *fakeForge) { f.containsDown = true }}, + {"not answering for the pair", func(*fakeForge) {}}, + } { + t.Run(c.name, func(t *testing.T) { + w := night747Over(t, c.set, nil) + if len(w.ctl.withouts) > 0 || len(w.ctl.stops) > 0 { + t.Fatalf("within the bound the walk was not held: without %v, stops %v", w.ctl.withouts, w.ctl.stops) + } + if state, desc := statusOf(w.delivery(id336)); state != "failure" { + t.Fatalf("#336 reads %s %q while the forge cannot say", state, desc) + } + said := "" + for _, l := range w.h.WalksWaiting() { + if l.Walk == "plan-cat" { + said += l.Says + } + } + w.later(3 * time.Minute) // past the grace an unheld walk is given, so `stalled` lists it + w.settleAll() + for _, l := range w.h.Stalled() { + if l.Walk == "plan-cat" { + said += " | " + l.Says + } + } + if strings.Count(said, "the forge is asked since") != 2 { + t.Fatalf("the hold is not said by `stalled walks` and `stalled`: %q", said) + } + if len(w.ctl.withouts) > 0 || len(w.ctl.stops) > 0 { + t.Fatalf("within the bound the walk was not held: without %v, stops %v", w.ctl.withouts, w.ctl.stops) + } + w.later(coverAskBound + time.Minute) + w.settleAll() + if len(w.ctl.withouts) == 0 || !strings.HasPrefix(w.ctl.withouts[0], "plan-cat without "+cat747+"@"+merge301) { + t.Fatalf("past the bound the walk was not asked to leave the stopped merge out: %v", w.ctl.withouts) + } + }) + } +} diff --git a/modules/mesh-delivery/cmd/mesh-delivery/issue753_test.go b/modules/mesh-delivery/cmd/mesh-delivery/issue753_test.go new file mode 100644 index 00000000..235c4507 --- /dev/null +++ b/modules/mesh-delivery/cmd/mesh-delivery/issue753_test.go @@ -0,0 +1,272 @@ +package main + +import ( + "strings" + "testing" + "time" +) + +// novox/hq issue 753: once issue 747's fix was live (2026-10-14 23:35), the catalogue's walk plan-1792012015539062661 at +// 0f76b6e2 (#307's merge; build-agent, gitea, route-proxy to all four machines) was resumed and said "published; its +// walk waits for mesh-delivery's word: no delivery that has not ended follows it or is answered by it". Every delivery +// it carried had ended: #301 stopped by its group for the controller's #336, delivered since, and waiting for "a later +// walk of its repository that contains its merge"; #304 stopped; #307 delivered. The only walk that contained #301's +// merge waited for #301's word: a deadlock, released by a hand `plans go`. A delivery whose stop lapsed gives its word +// to the walk that answers its merge, and a walk with no living delivery behind it says what it carries. +// +// The shape is the live state as `mesh-delivery.show` and `mesh-controller.plans` gave it, read back after a restart. + +const ( + merge304, head304, head307 = "4c0ffee0bbbbbbbb", "a304a304a304a304", "b307b307b307b307" + plan753 = "plan-1792012015539062661" +) + +var id304, id307 = IDOf(cat747, head304), IDOf(cat747, head307) + +// night753 is the stuck walk: #336 delivered (its change reached the machines), #301 stopped by the given act, #304 +// stopped by its group, #307 delivered, and the catalogue's walk at #307's merge naming all three merges, waiting. +func night753(t *testing.T, stop Transition) *world { return night753With(t, stop) } + +// night753With is night753 with more deliveries kept: one in #301's group is ordered after #336 and before #301. +func night753With(t *testing.T, stop Transition, more ...*Delivery) *world { + t.Helper() + return night753Over(t, stop, nil, nil, more...) +} + +// night753Over is night753With with more merges named by the walk and the controller set as given before the first +// tick. +func night753Over(t *testing.T, stop Transition, merges []WalkMerge, set func(*fakeController), more ...*Delivery) *world { + t.Helper() + at := at747 + store := newMemStore() + m336 := &Delivery{ID: id336, Repository: ctl747, Commit: head336, Number: 336, Base: "main", Branch: g747, Group: g747, + State: Delivered, Since: at.Add(70 * time.Minute), Created: at.Add(-time.Hour), MergedAs: merge336, MergedAt: at, + AnsweredBy: merge336, Check: &Verdict{Gate: "pass"}, Plan: planMoving(ctl747, "build-agent", "mesh-controller"), + Walk: &WalkSeen{ID: "plan-336", Commit: merge336, State: walkDone, Branch: "main", MovedSaid: true, + Moved: []string{"build-agent", "mesh-controller"}, Created: at.Add(7 * time.Minute), Tiers: 2, Tier: 2}} + m301 := &Delivery{ID: id301, Repository: cat747, Commit: head301, Number: 301, Base: "main", Branch: g747, Group: g747, + State: Stopped, Since: stop.At, Created: at.Add(-2 * time.Hour), MergedAs: merge301, MergedAt: at.Add(7 * time.Second), + Check: &Verdict{Gate: "pass"}, Plan: planMoving(cat747, "build-agent", "route-proxy"), Transitions: []Transition{stop}} + if stop.By == byOwner { + m301.StoppedFor = id336 + } + m304 := &Delivery{ID: id304, Repository: cat747, Commit: head304, Number: 304, Base: "main", Branch: "fix/forge-events", + State: Stopped, Since: at.Add(80 * time.Minute), Created: at.Add(-time.Hour), MergedAs: merge304, + MergedAt: at.Add(30 * time.Minute), Check: &Verdict{Gate: "pass"}, + Plan: planMoving(cat747, "build-agent", "gitea", "route-proxy"), + // Stopped with its walk, which the walk of #307 took over, and covered since by the walk of #306 (issue 747). + DeliveredBy: "plan-1792011830935225963 (novox/mesh-catalog at 0361c44a into main, done, contains its merge 4c0ffee0)", + CoverTold: true, + Walk: &WalkSeen{ID: "plan-1792009301770286799", Commit: merge304, State: walkSuperseded, Branch: "main", + TakenOverBy: plan753, Waits: true, WaitedFor: "mesh-delivery", Created: at.Add(31 * time.Minute)}, + Transitions: []Transition{{At: at.Add(80 * time.Minute), From: Published, To: Stopped, Event: EvStop, + Why: "its walk plan-1792009301770286799 was stopped by mesh-delivery"}}} + m307 := &Delivery{ID: id307, Repository: cat747, Commit: head307, Number: 307, Base: "main", Branch: "fix/ts-check", + State: Delivered, Since: at.Add(90 * time.Minute), Created: at.Add(-time.Hour), MergedAs: merge307, + MergedAt: at.Add(46 * time.Minute), Check: &Verdict{Gate: "pass"}, Plan: planMoving(cat747, "build-agent")} + for _, d := range append([]*Delivery{m336, m301, m304, m307}, more...) { + if err := store.PutDelivery(d); err != nil { + t.Fatal(err) + } + } + inGroup, order := []*Delivery{m301, m336}, []string{id336} + for _, d := range more { + if d.Group == g747 { + inGroup, order = append(inGroup, d), append(order, d.ID) + } + } + order = append(order, id301) + var ids []string + for _, d := range inGroup { + ids = append(ids, d.ID) + } + g := &Group{ID: g747, Members: ids, Closed: true, Said: "delivering", + Check: &GroupCheck{Asked: "check-1", At: at.Add(-time.Minute), + Verdict: "pass", Summary: "all compose", Commits: commitsOf(inGroup)}} + if err := store.PutGroup(g); err != nil { + t.Fatal(err) + } + ctlr := newFakeController() + modules := map[string]*WalkModule{"build-agent": {}, "gitea": {}, "route-proxy": {}} + ctlr.put(Walk{ID: plan753, Repository: cat747, Branch: "main", Commit: merge307, Created: at.Add(46 * time.Minute), + State: walkBuilding, Tiers: [][]string{{"build-agent", "gitea", "route-proxy"}}, Modules: modules, + Delivery: &WalkDelivery{Awaits: "mesh-delivery", Merges: []WalkMerge{ + {Repository: cat747, Commit: merge301, Carried: merge307}, {Repository: cat747, Commit: merge304, Carried: merge307}, + {Repository: cat747, Commit: merge307}}}}) + if len(merges) > 0 { + r := ctlr.walk(plan753) + r.Delivery.Merges = append(merges, r.Delivery.Merges...) + ctlr.put(r) + } + if set != nil { + set(ctlr) + } + ctlr.mergedAt[merge301], ctlr.mergedAt[merge304], ctlr.mergedAt[merge307] = m301.MergedAt, m304.MergedAt, m307.MergedAt + w := newWorldOver(t, store, ctlr, newFakeForge(), at.Add(100*time.Minute)) + if err := w.h.Reconcile(); err != nil { + t.Fatal(err) + } + w.settleAll() + return w +} + +// groupStop753 is #301's stop as its group made it: for #336, whose change was delivered since. +func groupStop753() Transition { + return Transition{At: at747.Add(11 * time.Minute), From: Ready, To: Stopped, Event: EvStop, By: byOwner, + Why: "its group " + g747 + " stopped: " + id336 + " failed"} +} + +func saidOf(lines []StalledLine, walk string) (string, State) { + said, state := "", State("") + for _, l := range lines { + if l.Walk == walk { + said, state = said+l.Says, l.State + } + } + return said, state +} + +// **A stopped delivery gives its word to the later walk it waits for**: #301, stopped for #336, delivered since, waits for +// a later walk of the catalogue that contains its merge; plan-1792012015539062661 names that merge and waits for this +// owner's word. #301 gives it, once, saying why, and `stalled walks` never calls the walk unheld. +func TestAStoppedDeliveryGivesItsWordToTheWalkItWaitsFor(t *testing.T) { + w := night753(t, groupStop753()) + if n := strings.Count(strings.Join(w.ctl.delivers, " "), plan753); n != 1 { + t.Fatalf("the walk was let go %d times (%v); want once", n, w.ctl.delivers) + } + d := w.delivery(id301) + if d.State != Stopped || !strings.Contains(d.LetGoWhy, lapsedWord(plan753)) || !strings.Contains(d.LetGoWhy, "issue 753") { + t.Fatalf("#301 is %s, its word given for %q", d.State, d.LetGoWhy) + } + if said, state := saidOf(w.h.WalksWaiting(), plan753); state == Unheld { + t.Fatalf("the walk reads unheld: %q", said) + } + if len(w.ctl.withouts) > 0 || len(w.ctl.stops) > 0 { + t.Fatalf("the walk was stopped: without %v, stops %v", w.ctl.withouts, w.ctl.stops) + } + // Once that walk built build-agent and route-proxy and ended done, #301 is covered by it (issue 747). + w.forge.contains[merge301+" in "+merge307] = true + r := w.ctl.walk(plan753) + sent := w.now + for _, m := range []string{"build-agent", "gitea", "route-proxy"} { + r.Modules[m] = &WalkModule{State: "built", SentAt: &sent} + } + r.State = walkDone + w.h.WalkMoved(w.ctl.put(r)) + w.later(time.Minute) + w.settleAll() + if state, desc := statusOf(w.delivery(id301)); state != "success" || !strings.Contains(desc, plan753) { + t.Fatalf("#301 reads %s %q", state, desc) + } +} + +// **A walk still holding a live delivery not at its turn waits for it**: the stopped delivery's word is one turn among +// the walk's, never a way past a live delivery it carries. +func TestALapsedStopDoesNotLetGoPastALiveDelivery(t *testing.T) { + at := at747 + held := &Delivery{ID: IDOf(cat747, "c308c308c308c308"), Repository: cat747, Commit: "c308c308c308c308", Number: 308, + Base: "main", Branch: "fix/held", State: Held, HeldWhy: "a person's", Since: at, Created: at.Add(-time.Hour), + MergedAs: "c308aaaaaaaaaaaa", MergedAt: at.Add(40 * time.Minute), Check: &Verdict{Gate: "pass"}, + Plan: planMoving(cat747, "gitea"), Walk: &WalkSeen{ID: plan753, Commit: merge307, State: walkBuilding, Branch: "main", + Waits: true, Created: at.Add(46 * time.Minute)}} + w := night753With(t, groupStop753(), held) + if containsString(w.ctl.delivers, plan753) { + t.Fatalf("the walk was let go past a held delivery it carries: %v", w.ctl.delivers) + } + if said, state := saidOf(w.h.WalksWaiting(), plan753); state != Held { + t.Fatalf("`stalled walks` reads %s %q; want the held delivery", state, said) + } +} + +// **A stop by a person gives no word**: #301 stopped by hand holds its change back; the walk is asked to leave it out, +// never let go for it. +func TestAPersonsStopGivesNoWord(t *testing.T) { + w := night753(t, Transition{At: at747.Add(11 * time.Minute), From: Ready, To: Stopped, Event: EvStop, By: "jochen", + Why: "stopped by hand"}) + if containsString(w.ctl.delivers, plan753) { + t.Fatalf("the walk was let go for a person's stop: %v", w.ctl.delivers) + } + if len(w.ctl.withouts) == 0 && len(w.ctl.stops) == 0 { + t.Fatal("the walk carrying a person's stop was neither left without it nor stopped") + } +} + +// **A walk no living delivery holds says what it carries** (the issue's third expectation): with nothing to give it +// its word, `stalled walks` and, past the grace, `stalled` name every ended delivery it carries and its state, never +// only that nothing holds it. +func TestAWalkWithNoLivingDeliverySaysWhatItCarries(t *testing.T) { + w := night753(t, groupStop753()) + // The walk read again as waiting, naming only #307's merge, which ended delivered: no delivery whose stop lapsed + // gives it its word, and no live one holds it. + r := w.ctl.walk(plan753) + r.Delivery.Go = nil + r.Delivery.Merges = []WalkMerge{{Repository: cat747, Commit: merge307}} + w.h.WalkMoved(w.ctl.put(r)) + said, state := saidOf(w.h.WalksWaiting(), plan753) + if state != Unheld || !strings.Contains(said, id307+" (delivered)") { + t.Fatalf("`stalled walks` reads %s %q; want unheld, naming %s (delivered)", state, said, id307) + } + w.settleAll() + w.later(matchGrace + time.Minute) + w.settleAll() + said, _ = saidOf(w.h.Stalled(), plan753) + if !strings.Contains(said, "every delivery it carries ended: ") || !strings.Contains(said, id307+" (delivered)") { + t.Fatalf("`stalled` reads %q", said) + } +} + +// m299 is a member of #301's group ordered before it, stopped for #336 as #301 was, its merge on another walk. +func m299() *Delivery { + at := at747 + return &Delivery{ID: IDOf(cat747, "e299e299e299e299"), Repository: cat747, Commit: "e299e299e299e299", Number: 299, + Base: "main", Branch: g747, Group: g747, State: Stopped, StoppedFor: id336, Since: at.Add(11 * time.Minute), + Created: at.Add(-3 * time.Hour), MergedAs: "e299aaaaaaaaaaaa", MergedAt: at.Add(5 * time.Second), + Check: &Verdict{Gate: "pass"}, Plan: planMoving(cat747, "gitea"), + Transitions: []Transition{{At: at.Add(11 * time.Minute), From: Ready, To: Stopped, Event: EvStop, By: byOwner, + Why: "its group " + g747 + " stopped: " + id336 + " failed"}}} +} + +// **A word the controller refuses is never silent** (review of issue 753): the controller keeps refusing it, and past +// the grace `stalled` lists the walk, with the refusal. +func TestARefusedWordIsListedByStalled(t *testing.T) { + w := night753Over(t, groupStop753(), nil, func(c *fakeController) { c.refuseDeliver = true }) + if containsString(w.ctl.delivers, plan753) { + t.Fatalf("the walk was let go: %v", w.ctl.delivers) + } + w.later(matchGrace + time.Minute) + w.settleAll() + said, state := saidOf(w.h.Stalled(), plan753) + if state != Published || !strings.Contains(said, "the controller refused its word: the controller refuses the word") { + t.Fatalf("`stalled` reads %s %q", state, said) + } + // Once the controller takes it, the walk is let go. + w.ctl.refuseDeliver = false + w.settleAll() + if !containsString(w.ctl.delivers, plan753) { + t.Fatalf("the walk was not let go once the controller took the word: %v", w.ctl.delivers) + } +} + +// **A walk held back for another stopped change is never silent** (review of issue 753): #304 was stopped by a person, +// and the controller refuses to leave its merge out or to stop the walk, so the word #301 owes is never sent; past the +// grace `stalled` lists the walk, saying why. +func TestAWalkHeldForAPersonsStopIsListedByStalled(t *testing.T) { + at := at747 + m305 := &Delivery{ID: IDOf(cat747, "f305f305f305f305"), Repository: cat747, Commit: "f305f305f305f305", Number: 305, + Base: "main", Branch: "fix/other", State: Stopped, Since: at.Add(80 * time.Minute), Created: at.Add(-time.Hour), + MergedAs: "f305aaaaaaaaaaaa", MergedAt: at.Add(35 * time.Minute), Check: &Verdict{Gate: "pass"}, + Plan: planMoving(cat747, "gitea"), Transitions: []Transition{{At: at.Add(80 * time.Minute), From: Ready, + To: Stopped, Event: EvStop, By: "jochen", Why: "stopped by hand"}}} + w := night753Over(t, groupStop753(), []WalkMerge{{Repository: cat747, Commit: m305.MergedAs, Carried: merge307}}, + func(c *fakeController) { c.refuseWithout, c.refuseStop = true, true }, m305) + if containsString(w.ctl.delivers, plan753) { + t.Fatalf("the walk carrying a person's stop was let go: %v", w.ctl.delivers) + } + w.later(matchGrace + time.Minute) + w.settleAll() + said, _ := saidOf(w.h.Stalled(), plan753) + if !strings.Contains(said, "owe it (novox/hq issue 753)") || !strings.Contains(said, "f305aaaaaaaaaaaa") || + !strings.Contains(said, "whose delivery was stopped") { + t.Fatalf("`stalled` reads %q", said) + } +} diff --git a/modules/mesh-delivery/cmd/mesh-delivery/issue783_test.go b/modules/mesh-delivery/cmd/mesh-delivery/issue783_test.go new file mode 100644 index 00000000..c0dd6062 --- /dev/null +++ b/modules/mesh-delivery/cmd/mesh-delivery/issue783_test.go @@ -0,0 +1,151 @@ +package main + +import ( + "strings" + "testing" + "time" +) + +// novox/hq issue 783: mesh-catalog commit b5417c28 was pushed while #305 (from branch adr-359-mesh-mail-733) was +// open, so this owner recorded its delivery as #305's. #305 then closed unmerged (its base branch had been +// deleted), and #315, opened from the same branch and head, merged it as 0914a33f. The delivery stayed stopped +// ("the pull request closed unmerged"), though it recorded merged_as 0914a33f: a rule did not hold — a delivery +// keyed by commit follows the pull request that merges that commit, and a closed duplicate does not end it — and +// the walk that carried the merge waited 64 minutes for this owner's word until a person's `plans go` released it. +// +// A delivery stopped only because its pull request closed unmerged, with no merge recorded, does not end its +// commit's delivery for another pull request with the same head: it is started over, under the same id, for the +// pull request that announces or merges it next. + +const ( + head783, merge783 = "d305d305d305d305", "d315d315d315d315" + head783b, merge783b = "d306d306d306d306", "d316d316d316d316" + head783c = "d307d307d307d307" + head783d, merge783d = "d308d308d308d308", "d318d318d318d318" + catalog783 = "novox/mesh-catalog" + branch783 = "adr-359-mesh-mail-733" +) + +// TestADeliveryStoppedByAClosedDuplicateFollowsThePullRequestThatMergesItsCommit is the issue's own shape: #305 +// announced, closed unmerged (stopped); #315 announced from the same head, checked and readied; #315 merges it, +// and the walk that answers that merge is let go — the delivery follows #315, never stuck on #305's stop. +func TestADeliveryStoppedByAClosedDuplicateFollowsThePullRequestThatMergesItsCommit(t *testing.T) { + w := newWorld(t) + id := IDOf(catalog783, head783) + + w.h.PullUpdated(pr(catalog783, 305, head783, branch783, "src/a.go")) + w.h.PullClosed(pr(catalog783, 305, head783, branch783)) + if w.state(id) != Stopped { + t.Fatalf("#305 closed unmerged left it %s; want stopped", w.state(id)) + } + + w.h.PullUpdated(pr(catalog783, 315, head783, branch783, "src/a.go")) + if d := w.delivery(id); d.State == Stopped || d.Number != 315 { + t.Fatalf("#315, the same head, left it %s, Number %d; want started over for #315, not stopped", d.State, d.Number) + } + if d := w.delivery(id); !strings.Contains(d.Transitions[len(d.Transitions)-1].Why, "#305 with the same head closed unmerged") { + t.Fatalf("the start-over's transition names what it replaces: %q", d.Transitions[len(d.Transitions)-1].Why) + } + + w.h.Checked(verdict(catalog783, 315, head783, "pass")) + if d := w.delivery(id); d.State != Ready { + t.Fatalf("a passing verdict for #315 left it %s", d.State) + } + + walk := w.ctl.put(aWalk("plan-783", catalog783, merge783, true)) + w.h.WalkMoved(walk) + w.h.PullMerged(merged(catalog783, 315, head783, merge783)) + if d := w.delivery(id); d.State != Published || d.Number != 315 { + t.Fatalf("#315's merge left it %s, Number %d; want published under #315", d.State, d.Number) + } + + w.settleAll() + if len(w.ctl.delivers) != 1 || w.ctl.delivers[0] != "plan-783" { + t.Fatalf("the walk that answers #315's merge was not let go: %v", w.ctl.delivers) + } + if d := w.delivery(id); d.State != Delivering { + t.Fatalf("let go, it is %s; want delivering", d.State) + } +} + +// TestAMergeWithoutAnAnnouncementStartsOverAStoppedDuplicateToo is decision 4: the merging pull request is never +// announced on its own (heard only as `pull.merged`) — the start-over happens in PullMerged itself, not only in +// PullUpdated, so a merge heard without the open announcement is not lost either. +func TestAMergeWithoutAnAnnouncementStartsOverAStoppedDuplicateToo(t *testing.T) { + w := newWorld(t) + id := IDOf(catalog783, head783b) + + w.h.PullUpdated(pr(catalog783, 306, head783b, branch783+"-b", "src/b.go")) + w.h.PullClosed(pr(catalog783, 306, head783b, branch783+"-b")) + if w.state(id) != Stopped { + t.Fatalf("#306 closed unmerged left it %s; want stopped", w.state(id)) + } + + walk := w.ctl.put(aWalk("plan-783b", catalog783, merge783b, true)) + w.h.WalkMoved(walk) + m := merged(catalog783, 316, head783b, merge783b) + m.HeadChecks = map[string]string{"mesh/merge-gate": "success", "mesh/repo-check": "success"} + w.h.PullMerged(m) + if d := w.delivery(id); d.State == Stopped || d.Number != 316 { + t.Fatalf("#316's merge, heard with no announcement of its own, left it %s, Number %d; want started over for #316", + d.State, d.Number) + } + + w.settleAll() + if len(w.ctl.delivers) != 1 || w.ctl.delivers[0] != "plan-783b" { + t.Fatalf("the walk that answers #316's merge was not let go: %v", w.ctl.delivers) + } +} + +// TestAStoppedDeliveryAnnouncedAgainByItsOwnPullRequestStaysStopped guards decision 3's "any other stopped +// delivery ... is untouched": the same pull request announced again after its own close is the ordinary +// 'announced again' path, never a start-over, and stays stopped. +func TestAStoppedDeliveryAnnouncedAgainByItsOwnPullRequestStaysStopped(t *testing.T) { + w := newWorld(t) + id := IDOf(catalog783, head783c) + + w.h.PullUpdated(pr(catalog783, 307, head783c, branch783+"-c", "src/c.go")) + w.h.PullClosed(pr(catalog783, 307, head783c, branch783+"-c")) + if w.state(id) != Stopped { + t.Fatalf("#307 closed unmerged left it %s; want stopped", w.state(id)) + } + + w.h.PullUpdated(pr(catalog783, 307, head783c, branch783+"-c", "src/c.go")) + if d := w.delivery(id); d.State != Stopped || d.Number != 307 { + t.Fatalf("#307 announced again by itself left it %s, Number %d; want it to stay stopped", d.State, d.Number) + } +} + +// TestAMergeHeardLongAfterStartingOverAStoppedDuplicateIsHistoryLikeAnyOtherReplayedMerge is review round 1's +// decision: a delivery started over from a merge heard without its open announcement (as in decision 4, above) +// takes FromForge exactly as a first-heard merge does, so one replayed from the bus's history long after it +// happened still falls under the history rule (novox/hq issue 313) and is never held for a person. +func TestAMergeHeardLongAfterStartingOverAStoppedDuplicateIsHistoryLikeAnyOtherReplayedMerge(t *testing.T) { + w := newWorld(t) + id := IDOf(catalog783, head783d) + + w.h.PullUpdated(pr(catalog783, 308, head783d, branch783+"-d", "src/d.go")) + w.h.PullClosed(pr(catalog783, 308, head783d, branch783+"-d")) + if w.state(id) != Stopped { + t.Fatalf("#308 closed unmerged left it %s; want stopped", w.state(id)) + } + + m := merged(catalog783, 318, head783d, merge783d) + m.MergedAt = w.now.Add(-historyLate - time.Minute).UTC().Format(time.RFC3339) + w.h.PullMerged(m) + + d := w.delivery(id) + if d.State == Stopped || d.Number != 318 { + t.Fatalf("#318's merge, heard long after it happened with no announcement of its own, left it %s, Number %d; "+ + "want started over for #318", d.State, d.Number) + } + if !d.FromForge { + t.Fatal("a merge heard without its own announcement is made from the forge's word: FromForge is false") + } + if d.History == "" { + t.Fatalf("a merge heard %s after it happened is history, never held for a person; History is empty", historyLate+time.Minute) + } + if d.State == Held { + t.Fatalf("history is never held for a person; state is %s, HeldWhy %q", d.State, d.HeldWhy) + } +} diff --git a/modules/mesh-delivery/cmd/mesh-delivery/issue902_test.go b/modules/mesh-delivery/cmd/mesh-delivery/issue902_test.go new file mode 100644 index 00000000..e602cc2b --- /dev/null +++ b/modules/mesh-delivery/cmd/mesh-delivery/issue902_test.go @@ -0,0 +1,66 @@ +package main + +import ( + "strings" + "testing" +) + +// novox/hq issue 902: a group that is delivering passed its composed check for its members' heads, so +// mesh/delivery-group says success there; pending would keep its later members from ever merging. +func TestReplay902(t *testing.T) { + for state, want := range map[string]string{ + "ready": "success", "delivering": "success", "delivered": "success", + "checking": "pending", "rejected": "failure", "failed": "failure", "stopped": "error", + } { + if got := groupStatusState(state); got != want { + t.Errorf("a group %s says %s on mesh/delivery-group, want %s", state, got, want) + } + } +} + +// group902 is a two-member group whose first member in its order merged. +func group902(t *testing.T) (*world, string, string, string, string) { + w := newWorld(t) + const ctlHead, catHead = "c1c1c1c1c1c1c1c1", "d2d2d2d2d2d2d2d2" + ctl, cat := IDOf("novox/mesh-controller", ctlHead), IDOf("novox/mesh-catalog", catHead) + w.h.PullUpdated(pr("novox/mesh-controller", 1, ctlHead, "feat/x", "internal/a.go")) + w.h.PullUpdated(pr("novox/mesh-catalog", 2, catHead, "feat/x", "modules/app/module.json")) + w.h.Checked(verdict("novox/mesh-controller", 1, ctlHead, "pass")) + w.h.Checked(verdict("novox/mesh-catalog", 2, catHead, "pass")) + w.settleAll() + w.h.Checked(CheckedEvent{Group: "feat/x", ID: "check-1", Verdict: "pass", Summary: "both compose"}) + w.settleAll() + w.h.PullMerged(merged("novox/mesh-catalog", 2, catHead, "m2m2m2m2m2m2")) + w.settleAll() + return w, ctl, cat, ctlHead, catHead +} + +func TestReplay902TheNextMemberOfADeliveringGroupReadsSuccess(t *testing.T) { + w, _, _, ctlHead, _ := group902(t) + g := w.h.groups["feat/x"] + st, _ := GroupState(g, w.h.membersOf(g)) + s := w.forge.statuses["novox/mesh-controller@"+ctlHead+" mesh/delivery-group"] + t.Logf("state=%s status=%q", st, s) + if st != "delivering" || !strings.HasPrefix(s, "success") { + t.Fatalf("want delivering/success, got %s %q", st, s) + } +} + +func TestReplay902AClosedGroupComposedAgainAndFailingSaysNoSuccess(t *testing.T) { + w, ctl, _, _, _ := group902(t) + const newHead = "e3e3e3e3e3e3e3e3" + _ = ctl + w.h.PullUpdated(pr("novox/mesh-controller", 1, newHead, "feat/x", "internal/a.go")) + w.h.Checked(verdict("novox/mesh-controller", 1, newHead, "pass")) + w.settleAll() + t.Logf("checks asked: %v", w.ctl.checks) + w.h.Checked(CheckedEvent{Group: "feat/x", ID: "check-2", Verdict: "fail", Summary: "no longer compose"}) + w.settleAll() + g := w.h.groups["feat/x"] + st, why := GroupState(g, w.h.membersOf(g)) + s := w.forge.statuses["novox/mesh-controller@"+newHead+" mesh/delivery-group"] + t.Logf("closed=%v check=%+v state=%s (%s) status=%q", g.Closed, g.Check, st, why, s) + if strings.HasPrefix(s, "success") { + t.Fatalf("a closed group whose composed check failed for its heads says success: %q", s) + } +} diff --git a/modules/mesh-delivery/cmd/mesh-delivery/main.go b/modules/mesh-delivery/cmd/mesh-delivery/main.go new file mode 100644 index 00000000..b61d5158 --- /dev/null +++ b/modules/mesh-delivery/cmd/mesh-delivery/main.go @@ -0,0 +1,422 @@ +// mesh-delivery: the holder of the mesh-delivery seat (novox/hq ADR 0239, to-be 47). A Go bundle the node's +// runtime launches. It owns the delivery — one commit in one repository, from its pull request's head to +// every machine — and the delivery group above it: their states in one compiled table, kept in its own state +// on the bus, every transition said as an event, noted on the commit and shown on the pull request through +// the forge's holder. It never sends to a machine: it asks the controller, through its seat's verbs. stdout +// is the MCP channel; what this module says, it says on stderr. +package main + +import ( + "encoding/json" + "fmt" + "os" + "strconv" + "strings" + "sync" + "time" + + stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/asker" + "git.novox.be/novox/mesh-sdk/go/asks" +) + +func logf(format string, a ...any) { fmt.Fprintf(os.Stderr, format+"\n", a...) } + +// listening is whether the events this owner follows reach it, in words. +type listening struct { + mu sync.Mutex + now string +} + +func (l *listening) set(s string) { l.mu.Lock(); l.now = s; l.mu.Unlock() } +func (l *listening) get() string { l.mu.Lock(); defer l.mu.Unlock(); return l.now } + +// The events this owner follows: the forge's pull requests, and the controller's verdicts and walks. +// The forge's status events say that a check answered (events.go). +var followed = []string{"gitea.pull.updated", "gitea.pull.merged", "gitea.pull.closed", + "gitea.forge.status", "gitea.forge.pull-request.edited", "gitea.forge.pull-request.closed", ControllerSeat + ".checked", ControllerSeat + ".plan-moved"} + +func main() { + operator := &asker.Client{Module: Module, State: stdio.State("asked")} + h := &Holder{ + Store: kvStore{}, + Controller: seatController{ask: stdio.Ask}, + Forge: toolForge{ask: stdio.Ask}, + Emit: func(event string, body any) error { return stdio.Emit(event, body) }, + Logf: logf, + Operator: operator, + Pulls: kvPulls{}, + } + h.init() + l := &listening{now: "not yet: starting"} + go run(h, l, operator) + if err := stdio.Serve("", tools(h, l)); err != nil { + logf("%v", err) + os.Exit(1) + } +} + +// run reads back the state, then takes the events, then reads once what it may have missed while it was away — the +// controller's walks, the forge's open pull requests, the operator's asks — and acts on events from then on +// (novox/hq ADR 0406 §1, issue 975): no tick, no forge poll, no reconcile loop. Each step is retried, each failure +// said, never given up on quietly. +func run(h *Holder, l *listening, operator *asker.Client) { + time.Sleep(500 * time.Millisecond) // Serve first: the state is reached through it + for wait := 2 * time.Second; ; wait = min(wait*2, time.Minute) { + err := h.Load() + if err == nil { + break + } + logf("[mesh-delivery] cannot read back its deliveries yet (%v); asking again in %s", err, wait) + time.Sleep(wait) + } + go h.Work() + for _, pattern := range followed { + pattern := pattern + for wait := 2 * time.Second; ; wait = min(wait*2, time.Minute) { + err := stdio.Subscribe(pattern, func(e stdio.Envelope) error { return handle(h, e) }) + if err == nil { + break + } + l.set("not yet: " + err.Error()) + logf("[mesh-delivery] not hearing %s yet (%v); asking again in %s", pattern, err, wait) + time.Sleep(wait) + } + } + // The operator's answers: the router's `decided` event, in place of reading every pending ask each tick. + for wait := 2 * time.Second; ; wait = min(wait*2, time.Minute) { + err := operator.Listen( + func(_ asks.Option, w asks.Warrant) error { h.AskDecided(w.Ask, "answered", &w); h.Poke(); return nil }, + func(handled asker.Handled, w asks.Warrant, err error) { + if err != nil { + logf("[mesh-delivery] the answer to the ask %s: %s: %v", w.Ask, handled, err) + } + if handled == asker.Refused || handled == asker.Recorded { + h.AskDecided(w.Ask, string(handled), &w) + h.Poke() + } + }) + if err == nil { + break + } + l.set("not yet: " + err.Error()) + logf("[mesh-delivery] not hearing the operator's answers yet (%v); asking again in %s", err, wait) + time.Sleep(wait) + } + l.set("listening") + logf("[mesh-delivery] listening: %s", strings.Join(followed, ", ")) + // Read once, now that events arrive: what happened while this owner was away. The forge's open pull requests (ready + // marks, heads nobody checks) are read by the first look; the asks the router closed meanwhile are read here. + h.wantForge() + h.hearAsks() + h.Poke() + // retries: the controller's walks, until they have been read once; plan-moved events carry every move after + for wait := 2 * time.Second; ; wait = min(wait*2, time.Minute) { + err := h.Reconcile() + if err == nil { + break + } + logf("[mesh-delivery] the controller's walks cannot be read yet (%v); asking again in %s", err, wait) + time.Sleep(wait) + } + h.Poke() +} + +// handle takes one event, then asks for a look at every step the event makes possible (Work, React). An event that cannot be read is +// said and acknowledged: read again it would fail again, and the controller's walks are read back by comparison anyway. +func handle(h *Holder, e stdio.Envelope) error { + taken := take(h, e) + h.Poke() + return taken +} + +func take(h *Holder, e stdio.Envelope) error { + switch e.Key { + case "gitea.pull.updated", "gitea.pull.merged", "gitea.pull.closed": + var p PullEvent + if err := json.Unmarshal(e.Body, &p); err != nil { + h.Refuse(fmt.Errorf("%s is not readable: %v", e.Key, err)) + return nil + } + switch e.Key { + case "gitea.pull.updated": + if h.PullUpdated(p) { + h.wantForge() // a head not seen before: the forge shows whether anyone checks it + } + h.KeepPull(p) + case "gitea.pull.merged": + h.PullMerged(p) + default: + h.PullClosed(p) + } + case "gitea.forge.pull-request.edited": + // The description edited after a head was announced (issue 1079): the review record in it is read again. + var p PullEvent + if err := json.Unmarshal(e.Body, &p); err != nil { + h.Refuse(fmt.Errorf("%s is not readable: %v", e.Key, err)) + return nil + } + h.PullEdited(p) + case "gitea.forge.pull-request.closed": + // The forge's own close event (issue 1107): the poll's gitea.pull.closed skips a repository whose update time did not + // move, and a close does not move it. A merged pull request is the webhook's merged event, never this one. + var p PullEvent + if err := json.Unmarshal(e.Body, &p); err != nil { + h.Refuse(fmt.Errorf("%s is not readable: %v", e.Key, err)) + return nil + } + if !p.Merged { + h.PullClosed(p) + } + case "gitea.forge.status": + var s ForgeStatusEvent + if err := json.Unmarshal(e.Body, &s); err != nil { + h.Refuse(fmt.Errorf("a commit status is not readable: %v", err)) + return nil + } + h.CommitStatus(s) + case ControllerSeat + ".checked": + var c CheckedEvent + if err := json.Unmarshal(e.Body, &c); err != nil { + h.Refuse(fmt.Errorf("a verdict is not readable: %v", err)) + return nil + } + h.Checked(c) + h.KeepVerdict(c) + case ControllerSeat + ".plan-moved": + var w Walk + if err := json.Unmarshal(e.Body, &w); err != nil { + h.Refuse(fmt.Errorf("a walk is not readable: %v", err)) + return nil + } + h.WalkMoved(w) + } + return nil +} + +func str(description string) map[string]any { + return map[string]any{"type": "string", "description": description} +} + +func strArg(a map[string]any, k string) string { s, _ := a[k].(string); return strings.TrimSpace(s) } + +// intArg is a whole number given as a number or as text; absent is zero. +func intArg(a map[string]any, k string) (int, error) { + switch v := a[k].(type) { + case nil: + return 0, nil + case float64: + if v == float64(int(v)) { + return int(v), nil + } + case int: + return v, nil + case string: + v = strings.TrimPrefix(strings.TrimSpace(v), "#") + if v == "" { + return 0, nil + } + if n, err := strconv.Atoi(v); err == nil { + return n, nil + } + } + return 0, fmt.Errorf("%q is not a whole number: %v", k, a[k]) +} + +func boolArg(a map[string]any, k string) bool { + switch v := a[k].(type) { + case bool: + return v + case string: + return v == "true" + } + return false +} + +// aPerson is who an act through the seat is by: the seat does not name its caller. +const aPerson = "a person, through the mesh-delivery seat" + +func need(a map[string]any, keys ...string) error { + for _, k := range keys { + if strArg(a, k) == "" { + return fmt.Errorf("needs %q", k) + } + } + return nil +} + +func tools(h *Holder, l *listening) []stdio.Tool { + seat := "mesh-delivery." + list := []stdio.Tool{ + {Name: seat + "deliveries", + Description: "Every delivery not final, those that ended in the last day, and any stop `stalled` still lists, one line each: its id " + + "(owner/repository@commit), its state, its group, what it waits for and since when. Narrowed by state, " + + "repository or group.", + Input: map[string]any{"type": "object", "properties": map[string]any{"state": str("one state"), + "repository": str("owner/repository"), "group": str("a group's id (its branch name)"), + "all": map[string]any{"type": "string", "enum": []string{"true", "false"}}}}, + Run: func(a map[string]any) (any, error) { + // Headed by the week's delivery times (novox/hq ADR 0282 decision 6). + return map[string]any{"times": h.TimesLine(), + "deliveries": h.Deliveries(strArg(a, "state"), strArg(a, "repository"), strArg(a, "group"), boolArg(a, "all"))}, nil + }}, + {Name: seat + "times", + Description: "Delivery time (novox/hq ADR 0282): from a merge to every machine of its walk running the build. " + + "Each class's budget (a leaf module three minutes, a core module ten), the last days' median and worst, " + + "how many within and over, the newest delivery of each class, every delivery over its budget with its " + + "longest phase, and the delivered ones whose time is not known and why. Given a delivery's id, its time " + + "phase by phase: window, queued, word, build, send-first, judgement, send-rest, between-tiers, apply — " + + "each measured, unknown (never a zero) or none.", + Input: map[string]any{"type": "object", "properties": map[string]any{ + "days": str("how many days back (default 7)"), "id": str("one delivery's id: its phases")}}, + Run: func(a map[string]any) (any, error) { + days, err := intArg(a, "days") + if err != nil { + return nil, err + } + return h.Times(days, strArg(a, "id")) + }}, + {Name: seat + "show", + Description: "One delivery or group whole: its delivery plan, every transition with when and why, the " + + "machine steps of its walk and its group.", + Input: map[string]any{"type": "object", "properties": map[string]any{"id": str("a delivery's id, or a group's")}, + "required": []string{"id"}}, + Run: func(a map[string]any) (any, error) { return h.Show(strArg(a, "id")) }}, + {Name: seat + "groups", + Description: "Every delivery group: its members, its composed check and its state, derived from its members. " + + "A group has no order: it is merged in one pass and delivered by one plan, whose tiers come from the module " + + "graph (novox/hq ADR 0418). composed_gate says its members' gate is the composed check's (ADR 0403).", + Input: map[string]any{"type": "object", "properties": map[string]any{ + "all": map[string]any{"type": "string", "enum": []string{"true", "false"}}}}, + Run: func(a map[string]any) (any, error) { return h.Groups(boolArg(a, "all")), nil }}, + {Name: seat + "what-if", + Description: "The delivery plan a change would have, asked of the controller's planner and kept nowhere.", + Input: map[string]any{"type": "object", "properties": map[string]any{"repository": str("owner/repository"), + "paths": str("the files it changes, comma-separated"), "base": str("the branch it merges into (default main)")}, + "required": []string{"repository", "paths"}}, + Run: func(a map[string]any) (any, error) { + if err := need(a, "repository", "paths"); err != nil { + return nil, err + } + return h.WhatIf(strArg(a, "repository"), strArg(a, "base"), strings.Split(strArg(a, "paths"), ",")) + }}, + {Name: seat + "checks", + Description: "What the mesh's checks said of a pull request's head or of one commit: each of the commit's " + + "statuses (mesh/merge-gate, mesh/repo-check, mesh/delivery, …) with its state, description and when it " + + "was set; the merge check's full verdict as the controller said it — each layer's summary, the machine " + + "that ran it, when, its build id and its report; and whether the branch's protection would let it " + + "merge, every required status being success.", + Input: map[string]any{"type": "object", "properties": map[string]any{"repository": str("owner/repository"), + "number": str("a pull request's number: its head is read"), + "commit": str("a commit's sha, or the start of one, instead of a pull request")}, + "required": []string{"repository"}}, + Run: func(a map[string]any) (any, error) { + if err := need(a, "repository"); err != nil { + return nil, err + } + number, err := intArg(a, "number") + if err != nil { + return nil, err + } + return h.Checks(strArg(a, "repository"), number, strArg(a, "commit")) + }}, + {Name: seat + "table", + Description: "The state table every delivery runs by: each transition with its guard, each state's bound " + + "and what healer H2 may do once it has passed; and the machine steps' table.", + Run: func(map[string]any) (any, error) { return TableText(), nil }}, + {Name: seat + "stalled", + Description: "Every delivery held past its state's bound, with the transition the table lets healer H2 take. " + + "With walks: true, instead each walk waiting for this owner's word and what holds it — a held delivery, " + + "its reason and the verb that releases it, or the deliveries not at their turn (novox/hq issue 529).", + Input: map[string]any{"type": "object", "properties": map[string]any{ + "walks": map[string]any{"type": "boolean", "description": "the waiting walks and what holds each, instead"}}}, + Run: func(a map[string]any) (any, error) { + if walks, _ := a["walks"].(bool); walks { + return h.WalksWaiting(), nil + } + return h.Stalled(), nil + }}, + {Name: seat + "recheck", + Description: "Check a rejected or ready delivery again: its merge check statuses put back to pending on the forge, " + + "so nothing merges on the old verdict; proposed again, and its check asked. With why.", + Input: map[string]any{"type": "object", "properties": map[string]any{"id": str("the delivery's id"), + "why": str("why, kept with the transition")}, "required": []string{"id", "why"}}, + Run: func(a map[string]any) (any, error) { + if err := need(a, "id", "why"); err != nil { + return nil, err + } + return h.Recheck(strArg(a, "id"), strArg(a, "why"), aPerson) + }}, + {Name: seat + "release", + Description: "A person's word that a held delivery goes on: it starts delivering. A delivery held for a failed backup asks the controller to take the backups again first. With why.", + Input: map[string]any{"type": "object", "properties": map[string]any{"id": str("the delivery's id"), + "why": str("why, kept with the transition")}, "required": []string{"id", "why"}}, + Run: func(a map[string]any) (any, error) { + if err := need(a, "id", "why"); err != nil { + return nil, err + } + return h.Release(strArg(a, "id"), strArg(a, "why"), aPerson) + }}, + {Name: seat + "stop", + Description: "Stop a delivery that is not final, with why: its walk is ended through the controller, and " + + "in a delivering group every member after it is stopped too, naming it. When its walk carries other " + + "deliveries, it first answers what ending the walk would take down, and acts only on a way: `again` " + + "ends the walk and walks the others again without this merge, `walk` stops the whole walk (novox/hq ADR 0299).", + Input: map[string]any{"type": "object", "properties": map[string]any{"id": str("the delivery's id"), + "why": str("why, kept with the transition"), + "way": str("on a walk shared with other deliveries: again (the others walked again without it) or walk " + + "(the whole walk stopped)")}, "required": []string{"id", "why"}}, + Run: func(a map[string]any) (any, error) { + if err := need(a, "id", "why"); err != nil { + return nil, err + } + return h.StopWay(strArg(a, "id"), strArg(a, "why"), aPerson, strArg(a, "way")) + }}, + {Name: seat + "close", + Description: "Healer H2's verb: for a delivery held past its bound, read its walk again and take the " + + "transition the table names for that state. Refused for any other.", + Input: map[string]any{"type": "object", "properties": map[string]any{"id": str("the delivery's id"), + "why": str("the condition it answers")}, "required": []string{"id", "why"}}, + Run: func(a map[string]any) (any, error) { + if err := need(a, "id", "why"); err != nil { + return nil, err + } + return h.Close(strArg(a, "id"), strArg(a, "why")) + }}, + {Name: seat + "retire-history", + Description: "Retire held deliveries that are history, on a person's word, with why: each made from the " + + "forge's word for a merge on the trunk heard 15 minutes or more after it happened — replayed when this " + + "owner first held the seat — with no walk of it open. Never released, delivered or put back: retired, " + + "final. One by id, refused when the rule does not take it; or every held one (of one repository when " + + "named), answering how many it retired and naming those it left held. dry \"true\" answers what it " + + "would do and changes nothing.", + Input: map[string]any{"type": "object", "properties": map[string]any{ + "why": str("why, kept with every transition"), "id": str("one delivery's id; every held one when empty"), + "repository": str("owner/repository: only its held deliveries"), + "dry": map[string]any{"type": "string", "enum": []string{"true", "false"}}}, + "required": []string{"why"}}, + Run: func(a map[string]any) (any, error) { + if err := need(a, "why"); err != nil { + return nil, err + } + return h.RetireHistory(strArg(a, "id"), strArg(a, "repository"), strArg(a, "why"), aPerson, + boolArg(a, "dry")) + }}, + {Name: "delivery_status", + Description: "Whether this owner works: how many deliveries in each state, the groups, what is owed to " + + "the forge and the bus and not yet done, how many owed effects were tried since it started (done, failed " + + "and tried again, given up) and when the last was, walks with no delivery yet, whether its events arrive, when " + + "the controller's walks were last read or why they could not be, and the last refusals.", + Run: func(map[string]any) (any, error) { return h.Status(l.get()), nil }}, + } + // A verb that changes a delivery asks for a look at what it makes possible (react.go). + for i := range list { + run := list[i].Run + list[i].Run = func(a map[string]any) (any, error) { + out, err := run(a) + h.Poke() + return out, err + } + } + return list +} diff --git a/modules/mesh-delivery/cmd/mesh-delivery/manifest_test.go b/modules/mesh-delivery/cmd/mesh-delivery/manifest_test.go new file mode 100644 index 00000000..0c866f56 --- /dev/null +++ b/modules/mesh-delivery/cmd/mesh-delivery/manifest_test.go @@ -0,0 +1,177 @@ +package main + +import ( + "encoding/json" + "os" + "path/filepath" + "reflect" + "sort" + "strings" + "testing" + "time" +) + +// The manifest says what the code does: the seat it claims and the verbs it serves, the events it follows +// and says, the tools it calls, its state — and names nothing of one installation. + +type manifest struct { + Module string `json:"module"` + Claims []struct { + Name string `json:"name"` + Scope string `json:"scope"` + Serves []string `json:"serves"` + } `json:"claims"` + Seats []any `json:"seats"` + Uses []string `json:"uses"` + Consumes []string `json:"consumes"` + Emits []string `json:"emits"` + Invokes []string `json:"invokes"` + State []string `json:"state"` + Tools []string `json:"tools"` +} + +func readManifest(t *testing.T) (manifest, string) { + t.Helper() + raw, err := os.ReadFile(filepath.Join("..", "..", "module.json")) + if err != nil { + t.Fatal(err) + } + var m manifest + if err := json.Unmarshal(raw, &m); err != nil { + t.Fatal(err) + } + return m, string(raw) +} + +func TestItClaimsTheDeliverySeatAndFollowsWhatItHandles(t *testing.T) { + m, raw := readManifest(t) + if m.Module != "mesh-delivery" || len(m.Claims) != 1 || m.Claims[0].Name != "mesh-delivery" || m.Claims[0].Scope != "mesh" { + t.Fatalf("claims %+v", m.Claims) + } + // The seat is the mesh's own (mesh-*): a module claims it and never declares it. + if len(m.Seats) != 0 { + t.Fatal("the manifest declares a seat; mesh-delivery is the mesh's own") + } + if !reflect.DeepEqual(m.Consumes, followed) { + t.Fatalf("consumes %v, follows %v", m.Consumes, followed) + } + if !reflect.DeepEqual(m.Emits, []string{"transition", "group", "merged"}) { + t.Fatalf("emits %v", m.Emits) + } + if !reflect.DeepEqual(m.State, []string{"deliveries", "groups", "asked", "pulls"}) { + t.Fatalf("state %v", m.State) + } + // It calls exactly what its ports ask: the controller's verbs and the forge's tools. + want := []string{} + for _, v := range []string{"delivery-plan", "delivery-check", "deliver", "delivery-backups-again", "delivery-stop", "delivery-walks", "forget-checks"} { + want = append(want, "seat:"+ControllerSeat+"."+v) + } + want = append(want, "gitea.gitea_note_append", "gitea.gitea_delivery_view", "gitea.gitea_commit_status", + "gitea.gitea_commit_statuses", "gitea.gitea_contains", "gitea.gitea_open_pulls", + // The merge on a ready mark (novox/hq issue 722): the merge tool, a comment, a label taken off. + "gitea.gitea_merge_pull_request", "gitea.gitea_add_comment") + // A merge into main asks the operator for the Approve (novox/hq ADR 0398 §3, issue 948), as the knowledge base asks. + want = append(want, "seat:operator-channel.notify") + if !reflect.DeepEqual(m.Uses, []string{"operator-channel"}) { + t.Fatalf("uses %v", m.Uses) + } + if !reflect.DeepEqual(m.Invokes, want) { + t.Fatalf("invokes %v, wanted %v", m.Invokes, want) + } + for _, never := range []string{"/home/", "jochen", "g14", "shanks", "novox", "zurag", "anchor"} { + if strings.Contains(strings.ToLower(raw), never) { + t.Errorf("module.json names %q", never) + } + } +} + +func TestTheToolsAgreeWithTheManifest(t *testing.T) { + m, _ := readManifest(t) + var own, verbs []string + for _, tool := range tools(&Holder{}, &listening{}) { + if strings.TrimSpace(tool.Description) == "" { + t.Errorf("%s has no description", tool.Name) + } + if seat, verb, ok := strings.Cut(tool.Name, "."); ok { + if seat != "mesh-delivery" { + t.Errorf("%s is a verb of a seat this does not hold", tool.Name) + } + verbs = append(verbs, verb) + continue + } + own = append(own, tool.Name) + } + sort.Strings(own) + if !reflect.DeepEqual(own, m.Tools) { + t.Fatalf("serves %v, lists %v", own, m.Tools) + } + if !reflect.DeepEqual(verbs, m.Claims[0].Serves) { + t.Fatalf("verbs %v, claim %v", verbs, m.Claims[0].Serves) + } +} + +// The forge's and the controller's answers are read whatever wraps them. +func TestAnAnswerIsReadWhateverWrapsIt(t *testing.T) { + for _, c := range []struct { + raw string + ok bool + answer string + }{ + {`{"output":"x","ok":true,"answer":{"held":true}}`, true, `{"held":true}`}, + {`{"output":"refused: no plan","ok":false}`, false, ``}, + {`"{\"output\":\"x\",\"ok\":true,\"answer\":[1]}"`, true, `[1]`}, + {`{"content":[{"type":"text","text":"{\"output\":\"x\",\"ok\":true,\"answer\":2}"}]}`, true, `2`}, + {`{"content":[{"type":"text","text":"boom"}],"isError":true}`, false, ``}, + } { + answer, _, ok := answerOf(json.RawMessage(c.raw)) + if ok != c.ok || (c.ok && string(answer) != c.answer) { + t.Errorf("%s → %s %v", c.raw, answer, ok) + } + } +} + +// **No ready mark is left in the module** (novox/hq ADR 0421 §4, issue 1022; how it is checked): no label, comment, verb +// or trusted setting of the mark stands in any of its code or its manifest. Tests are left out: they name the words to +// prove them gone. +func TestNoReadyMarkIsLeftInTheModule(t *testing.T) { + forbidden := []string{"mesh/ready", "ready at", "ready-authors", "mark-ready", "ReadyLabel", "pull-request.labeled", + "pull-request.unlabeled", "pull-request.commented"} + files, err := filepath.Glob("*.go") + if err != nil { + t.Fatal(err) + } + files = append(files, filepath.Join("..", "..", "module.json")) + for _, f := range files { + if strings.HasSuffix(f, "_test.go") { + continue + } + raw, err := os.ReadFile(f) + if err != nil { + t.Fatal(err) + } + for _, word := range forbidden { + if strings.Contains(string(raw), word) { + t.Errorf("%s still holds %q: the ready mark is gone (ADR 0421 §4)", f, word) + } + } + } +} + +// A row written while the ready mark stood still reads: its mark's fields are ignored, its ask and merge kept (ADR 0421 +// §4 consequences, issue 1022). +func TestARowWrittenWithTheReadyMarkStillReads(t *testing.T) { + store := newMemStore() + row := `{"id":"novox/app@111111111111","repository":"novox/app","commit":"` + readyHead + `","number":7,"base":"main", +"state":"ready","since":"2026-10-17T10:00:00Z","created":"2026-10-17T10:00:00Z","let_go":"0001-01-01T00:00:00Z", +"ready":{"head":"` + readyHead + `","by":"lead@shanks","marked_at":"2026-10-17T10:01:00Z","seen":"2026-10-17T10:02:00Z", +"last_try":"0001-01-01T00:00:00Z","asked_at":"2026-10-17T10:03:00Z","ask_id":"merge-7-111111111111-1","heads":{"novox/app@111111111111":"` + readyHead + `"}}}` + store.deliveries["novox/app@111111111111"] = []byte(row) + w := newWorldOver(t, store, newFakeController(), newFakeForge(), time.Date(2026, 10, 17, 11, 0, 0, 0, time.UTC)) + d := w.delivery("novox/app@111111111111") + if d.Ready == nil || d.Ready.AskID != "merge-7-111111111111-1" || d.Ready.Head != readyHead { + t.Fatalf("the old row is not read: %+v", d.Ready) + } + if !MemberOf(d).Ready { + t.Fatalf("an ask waiting is a ready member") + } +} diff --git a/modules/mesh-delivery/cmd/mesh-delivery/model.go b/modules/mesh-delivery/cmd/mesh-delivery/model.go new file mode 100644 index 00000000..863d832a --- /dev/null +++ b/modules/mesh-delivery/cmd/mesh-delivery/model.go @@ -0,0 +1,1030 @@ +package main + +import ( + "encoding/json" + "fmt" + "maps" + "regexp" + "sort" + "strings" + "time" +) + +// What mesh-delivery owns (novox/hq ADR 0239): the delivery — one commit in one repository — and the +// delivery group above it. Each is kept whole, one key of the module's own state per delivery and per +// group, written only by this holder. + +// Delivery is one commit in one repository, from its pull request's head to every machine. +type Delivery struct { + // ID is owner/repository@. + ID string `json:"id"` + Repository string `json:"repository"` + Commit string `json:"commit"` + // The pull request, when there is one: its number, title, base, head branch and page. + Number int `json:"number,omitempty"` + Title string `json:"title,omitempty"` + Base string `json:"base,omitempty"` + Branch string `json:"branch,omitempty"` + HTMLURL string `json:"html_url,omitempty"` + CloneURL string `json:"clone_url,omitempty"` + // The diffset, as the forge said it: what the planner maps onto modules. + Paths []string `json:"paths,omitempty"` + PathsTruncated bool `json:"paths_truncated,omitempty"` + Removed []string `json:"removed,omitempty"` + ModuleDirs []string `json:"module_dirs,omitempty"` + ModuleDirsSaid bool `json:"module_dirs_said,omitempty"` + // After are the repositories its pull request's description names in `after: ` lines. They are not read + // (novox/hq ADR 0419 §2): kept only to say once on the pull request that the line is not read. AfterSaid is whether + // that was said, for this pull request. + After []string `json:"after,omitempty"` + AfterSaid bool `json:"after_said,omitempty"` + // Issues are the issues its title and body name ("issue 948"), shown to the operator who is asked to approve its merge. + Issues []int `json:"issues,omitempty"` + // Group is the delivery group it is in: its head branch name. Empty for a lone delivery. + Group string `json:"group,omitempty"` + + State State `json:"state"` + Since time.Time `json:"since"` + Created time.Time `json:"created"` + + // Plan is its delivery plan, as the controller's planner computed it (the build plan, the deploy plan, + // the steps that are not an ordinary send). + Plan *DeliveryPlan `json:"plan,omitempty"` + // Check is its own check's verdict. + Check *Verdict `json:"check,omitempty"` + // CheckAsk is this owner's own ask of its check, when the forge's announcement did not bring one: a head + // adopted while it already waited, or one whose announcement the controller missed. + CheckAsk *CheckAsk `json:"check_ask,omitempty"` + // TrunkNow is its trunk's head as this owner last heard it move while it was open: the commit the last merge + // into its trunk landed as (novox/hq ADR 0323 §7). A verdict that names another trunk commit is out of date. + // Empty until a merge into its trunk is heard after it was made. + TrunkNow string `json:"trunk_now,omitempty"` + // BaseSHA is the base branch's head the forge last announced its head with (ADR 0323): sent with a check this + // owner asks, for a verdict no build judges to name its trunk commit. + BaseSHA string `json:"base_sha,omitempty"` + // AskedAgain is when its head was announced again while its verdict was out of date — the merge tool's ask of + // its check against the trunk's head (ADR 0323 §5) — and no verdict has come since: its one-hour bound runs from + // then. Nil while nothing is asked, and cleared by a verdict or another move of its trunk. + AskedAgain *time.Time `json:"asked_again,omitempty"` + + // The facts the table's guards read. NewerHead is the head that superseded it; ClosedUnmerged the pull + // request closed; MergedAs the commit it landed on the trunk as; HeldWhy what holds it for a person. + NewerHead string `json:"newer_head,omitempty"` + ClosedUnmerged bool `json:"closed_unmerged,omitempty"` + // Others are the other open pull requests announced at this head while this one was live (issue 1114), as they were + // announced: the record is theirs to take over when this one's pull request closes unmerged. + Others []PullEvent `json:"others,omitempty"` + MergedAs string `json:"merged_as,omitempty"` + MergedAt time.Time `json:"merged_at"` + HeldWhy string `json:"held_why,omitempty"` + // BackupsAgainAfter is the revision of its walk when a person released it for a failed backup: a record at that + // revision or before still holds the failure the controller was asked to clear, and does not hold it again + // (novox/hq ADR 0404, issue 1068). + BackupsAgainAfter int64 `json:"backups_again_after,omitempty"` + // FromForge is a delivery made from the forge's word at its merge: its head was never heard by this owner. + // History is why it is history, once it is (novox/hq issue 313). + FromForge bool `json:"from_forge,omitempty"` + History string `json:"history,omitempty"` + // CoveredBy says which later merges into the same trunk delivered every module it moves, once they did + // (novox/hq issue 319); Lifted why what held it for a person no longer holds (issue 316). + CoveredBy string `json:"covered_by,omitempty"` + Lifted string `json:"lifted,omitempty"` + // Stranded says why a merge whose walk ended done before its word, without moving every module it moves, is + // stopped: nothing later delivered them either (novox/hq issue 474). Set once the cover test says no. + Stranded string `json:"stranded,omitempty"` + // CoverTold is whether the forge was told that such a stop was covered since: its status, its view and its note + // (novox/hq issue 480). Kept, so a restart tells it once, and a stop covered before this was built is told too. + CoverTold bool `json:"cover_told,omitempty"` + // DeliveredBy says, for a delivery that failed or was stopped, the later finished walk of its repository whose + // commit contains its merge and which built every module it moves — each it did not build, its own walk said + // unchanged: its change reached the machines after all (novox/hq issue 747). Its state stays final, as a covered + // stop's does (issue 480), and it reads covered: its group counts it as delivered. + DeliveredBy string `json:"delivered_by,omitempty"` + // StoppedFor is, for a member its group stopped, the member whose end stopped it (novox/hq issue 747): once that + // member's change reached the machines, the stop has lapsed, and holds no walk of its repository back. + StoppedFor string `json:"stopped_for,omitempty"` + // HeldBack is why the walk it waits on is not let go: it carries a merge whose delivery was stopped, and is + // asked to leave it out (novox/hq ADR 0299). Said by `waits`, and by `stalled` past its bound. + HeldBack string `json:"held_back,omitempty"` + // ContainedIn is the forge's answer, by a later merge commit, to whether that commit contains this one's + // merge (gitea_contains): asked once per pair, and kept (ADR 0270). + ContainedIn map[string]bool `json:"contained_in,omitempty"` + // AnsweredBy is the commit of its repository the walk it follows carries for it: its own merge, or a later merge + // of its branch that contains it (novox/hq ADR 0276). Unwalked says it is held because no walk answered it. + AnsweredBy string `json:"answered_by,omitempty"` + // Carrier says, for a merge a later one carries, which: its delivery and walk, in words. + Carrier string `json:"carrier,omitempty"` + Unwalked bool `json:"unwalked,omitempty"` + // Walk is the walk that delivers it, as last read: its own trunk commit's, or one that carries a later merge + // of its branch; Steps the machines it reached. + Walk *WalkSeen `json:"walk,omitempty"` + Steps []Step `json:"steps,omitempty"` + // Time is its delivery time, class and phases (novox/hq ADR 0282), once its walk's end is known. + Time *DeliveryTime `json:"delivery_time,omitempty"` + // LetGo is when this owner asked the controller to start its walk, and why. + LetGo time.Time `json:"let_go"` + LetGoWhy string `json:"let_go_why,omitempty"` + + // Ready is what this owner did with the operator's Approve of its merge into main, and with the merge (novox/hq ADR + // 0421, ready.go). The json key is the one the ready mark's record had, so rows written before still read. + Ready *ReadyRun `json:"ready,omitempty"` + // ReviewedBy and ReviewedAt are the review record its pull request's description holds: the line + // `Reviewed-by: at ` (novox/hq ADR 0421 §5, issue 1022), as written. Read at every head; never a + // gate: the ask says first when the line is missing or names another head. + ReviewedBy string `json:"reviewed_by,omitempty"` + ReviewedAt string `json:"reviewed_at,omitempty"` + // FreshWanted is whether a passing verdict that arrived out of date is to have its check asked against the + // trunk's head, before the operator is asked (novox/hq ADR 0421 §3, issue 1022). Cleared when it is asked. + FreshWanted bool `json:"fresh_wanted,omitempty"` + + Transitions []Transition `json:"transitions,omitempty"` + // Owed are the forge's and the bus's effects of its transitions not yet done: said only after the + // delivery is kept, retried until done. + Owed []Effect `json:"owed,omitempty"` +} + +// Transition is one move of the table, kept. +type Transition struct { + At time.Time `json:"at"` + From State `json:"from"` + To State `json:"to"` + Event Event `json:"event"` + Why string `json:"why"` + By string `json:"by,omitempty"` +} + +// Verdict is a check's verdict: the gate's and the repository's own. ID is the build seat's ask that ran +// it, On the machine it ran on, At when this owner heard it; Report is the check's own account, as bounded +// as the controller says it. All of it is what `checks` answers, whole, beside the forge's short statuses. +type Verdict struct { + ID string `json:"id,omitempty"` + On string `json:"on,omitempty"` + At time.Time `json:"at"` + Gate string `json:"gate"` + Summary string `json:"summary"` + Modules []string `json:"modules,omitempty"` + Dependents []string `json:"dependents,omitempty"` + Repo string `json:"repo,omitempty"` + RepoSaid string `json:"repo_summary,omitempty"` + Report string `json:"report,omitempty"` + // Head and Trunk are what it judged (novox/hq ADR 0323 §3): the head merged into the trunk commit, twelve + // characters each. Trunk is empty for a verdict of a build seat that judged the head alone. Beside are the + // commits of the other repositories the check read, by directory; Took how long it ran. + Head string `json:"head,omitempty"` + Trunk string `json:"trunk,omitempty"` + Beside map[string]string `json:"beside,omitempty"` + Took string `json:"took,omitempty"` +} + +// Judged is what a verdict judged, in words: "head 1a2b3c4d5e6f merged into trunk 7a8b9c0d1e2f" (ADR 0323 §3); +// empty for a verdict that names no trunk commit. +func (v *Verdict) Judged() string { + if v == nil || v.Trunk == "" { + return "" + } + return fmt.Sprintf("head %s merged into trunk %s", shortOf(v.Head, 12), shortOf(v.Trunk, 12)) +} + +// outOfDate is why a delivery's verdict no longer counts: it names a trunk commit that is not its trunk's head +// as this owner heard it (novox/hq ADR 0323 §7). Empty when it counts — or names no trunk commit, as a verdict +// of a build seat from before the rule does, which counts as it did. +func (d *Delivery) outOfDate() string { + if d.Check == nil || d.Check.Trunk == "" || d.TrunkNow == "" || sameCommit(d.Check.Trunk, d.TrunkNow) { + return "" + } + return outOfDateWords(d.Check.Trunk, d.TrunkNow) +} + +// outOfDateWords is a verdict out of date, as the forge's holder says it on the pull request (ADR 0323 §4). +func outOfDateWords(judged, now string) string { + return fmt.Sprintf("out of date: judged against trunk %s, the trunk is now %s; judged again on a recheck or when "+ + "it is merged", shortOf(judged, 12), shortOf(now, 12)) +} + +// CheckAsk is a head's own check, asked of the controller by this owner (or re-asked by healer H2). +type CheckAsk struct { + // ID is the ask's id as the controller answered it. + ID string `json:"id"` + At time.Time `json:"at"` + By string `json:"by"` + Why string `json:"why"` + // Reasked is how many times H2 asked it again: once at most, then the operator's. + Reasked int `json:"reasked,omitempty"` +} + +// GateComposed is the gate layer of a group member's own verdict when the group's composed check judges it instead +// (novox/hq ADR 0403 §1): no gate was run for the member alone. +const GateComposed = "composed" + +// Passes is whether the verdict lets the delivery be ready: the gate passed or warned, and the +// repository's own check did not fail or error. +func (v *Verdict) Passes() bool { + if v == nil { + return false + } + // A group member's own gate is the group's composed check's (novox/hq ADR 0403 §1): the controller says the layer + // "composed", and the member counts as passed here once its repository's check did not fail, as the composed + // check, asked after every member is ready, then judges them together. + gate := v.Gate == "pass" || v.Gate == "warning" || v.Gate == GateComposed + repo := v.Repo == "" || v.Repo == "pass" || v.Repo == "warning" + return gate && repo +} + +// DeliveryPlan is a delivery plan as the controller's planner says it (mesh-controller link.ChangePlan). +type DeliveryPlan struct { + Repository string `json:"repository"` + Base string `json:"base"` + Head string `json:"head"` + Moved []string `json:"moved,omitempty"` + Dependents []string `json:"dependents,omitempty"` + New []string `json:"new,omitempty"` + Unread []string `json:"unread,omitempty"` + Tiers [][]string `json:"tiers,omitempty"` + Machines []MachinePlan `json:"machines,omitempty"` + Steps []string `json:"steps,omitempty"` + Summary string `json:"summary"` + // Class is the class of the slowest-budgeted module it moves, as the planner says it: core, leaf, or none when + // it moves no module (novox/hq ADR 0313 §2). Empty in a plan from a controller that did not say it. + Class string `json:"class,omitempty"` + // Trunk is the trunk commit the head was merged into when its verdict judged it (novox/hq ADR 0323 §3). + Trunk string `json:"trunk,omitempty"` +} + +// receiving are the machines the plan sends a module to, in the plan's order; none when the plan names no +// machines, as a plan kept before they were said may not. +func (p *DeliveryPlan) receiving(module string) []string { + if p == nil { + return nil + } + var out []string + for _, m := range p.Machines { + if containsString(m.Receives, module) { + out = append(out, m.Machine) + } + } + return out +} + +// MachinePlan is one machine's part of a plan. +type MachinePlan struct { + Machine string `json:"machine"` + Receives []string `json:"receives,omitempty"` + Waits []string `json:"waits,omitempty"` +} + +// waitsForItsOwnWalk is whether it is published again after the walk that carried a later commit for it failed. +func (d *Delivery) waitsForItsOwnWalk() bool { + return d.State == Published && d.Walk != nil && d.Walk.State == walkFailed && d.carriedLater() +} + +// carriedLater is whether the walk it follows carries a later merge of its branch for it, not its own. +func (d *Delivery) carriedLater() bool { + return d.AnsweredBy != "" && d.MergedAs != "" && !sameCommit(d.AnsweredBy, d.MergedAs) +} + +// MovesNothing is whether the delivery's plan moves no module the mesh holds: nothing for a walk to do. A +// module it adds is not moved: the controller builds and registers it and sends it nowhere, and opens no walk +// for it (novox/hq issues 300 and 319). A delivery with no plan known is not said to move nothing. +func (d *Delivery) MovesNothing() bool { + return d.Plan != nil && len(d.Plan.Moved) == 0 +} + +// Step is one machine's part of a delivering walk. +type Step struct { + Machine string `json:"machine"` + Module string `json:"module"` + Build string `json:"build,omitempty"` + State StepState `json:"state"` + Since time.Time `json:"since"` + Why string `json:"why,omitempty"` +} + +// Effect is one thing owed outside the state: an event said, a line appended to a commit's note, the view +// on the pull request refreshed, a status set. +type Effect struct { + Kind string `json:"kind"` + Event string `json:"event,omitempty"` // an emit's module event, when not transition or group (issue 1025) + Commit string `json:"commit,omitempty"` + Line string `json:"line,omitempty"` + Context string `json:"context,omitempty"` + State string `json:"state,omitempty"` + Tries int `json:"tries,omitempty"` + Last string `json:"last,omitempty"` + Since time.Time `json:"since"` + // Next is when a failed effect is tried again: later after every failure, up to retryCap. + Next *time.Time `json:"next,omitempty"` +} + +// The kinds of effect. +const ( + EffectEmit = "emit" + EffectNote = "note" + EffectView = "view" + EffectStatus = "status" +) + +// FromForgesWord is whether the delivery was made from the forge's word at its merge: said by FromForge, or — +// for one made before FromForge was kept — by its first transition. +func (d *Delivery) FromForgesWord() bool { + if d.FromForge { + return true + } + return len(d.Transitions) > 0 && d.Transitions[0].Event == EvAnnounced && + strings.Contains(d.Transitions[0].Why, "so it is made from the forge's word") +} + +// shownOnItsPullRequest is whether its view and status are kept on its pull request: not for history, whose +// pull request the forge may no longer hold — or holds under its number another one. +// coveredStop is whether a delivery that ended stopped or failed was covered since, and reads covered. A stop +// because its walk ended done before its word without building its modules (novox/hq issue 474) is covered when a +// later merge into its trunk that contains it was delivered, carrying every module it moves to every machine running +// it (ADR 0270's cover test, kept as CoveredBy; issue 480). Any stop or failure is covered when a later finished walk +// of its repository whose commit contains its merge built its modules (DeliveredBy, novox/hq issue 747). Its state +// stays final, as a covered superseded member stays superseded (issue 442), and it reads covered, by that merge or +// walk. A person's stop with a cover found only while it was held keeps its error. +func (d *Delivery) coveredStop() bool { + switch d.State { + case Stopped: + return d.Stranded != "" && d.CoveredBy != "" || d.DeliveredBy != "" + case Failed: + return d.DeliveredBy != "" + } + return false +} + +// coveredStopSaid is how a covered stop or failure reads: on the forge's status, in its view and note, and in `show`. +func (d *Delivery) coveredStopSaid() string { + ended := "its stop" + if d.State == Failed { + ended = "its failure" + } + if d.Stranded != "" && d.CoveredBy != "" { + return "covered after " + ended + ": a later merge that contains it was delivered: " + d.CoveredBy + } + return "covered after " + ended + ": a later walk that contains it was delivered: " + d.DeliveredBy +} + +// memberReached is whether a group counts a member as delivered: its change reached the machines, or had nothing to +// reach them with — delivered, history or covered, a superseded member a later delivered walk carried (issue 442), +// or a stop or failure covered since (issues 480 and 747). +func memberReached(d *Delivery) bool { + return d.State.Reached() || d.State == Superseded && d.CoveredBy != "" || d.coveredStop() +} + +// stoppedFor is the member whose end stopped this one in its group: kept since novox/hq issue 747, and read from the +// stop's reason for a stop made before ("its group stopped: …"). +func (d *Delivery) stoppedFor() string { + if d.StoppedFor != "" || d.State != Stopped || d.Group == "" { + return d.StoppedFor + } + prefix := "its group " + d.Group + " stopped: " + for i := len(d.Transitions) - 1; i >= 0; i-- { + t := d.Transitions[i] + if t.Event != EvStop { + continue + } + if t.By != byOwner || !strings.HasPrefix(t.Why, prefix) { + return "" + } + member, _, _ := strings.Cut(strings.TrimPrefix(t.Why, prefix), " ") + return strings.TrimRight(member, ",;:") + } + return "" +} + +// stopLapsed is whether a member its group stopped for another member is stopped for nothing any more: that member's +// change reached the machines since — delivered, or its failure covered by a later walk (novox/hq issue 747). Its own +// change is still to deliver, by a later walk of its repository that contains it; its stop holds none back. find +// gives a delivery by its id. +func stopLapsed(d *Delivery, find func(string) *Delivery) bool { + if d.State != Stopped || d.coveredStop() { + return false + } + by := d.stoppedFor() + if by == "" || by == d.ID { + return false + } + o := find(by) + return o != nil && memberReached(o) +} + +func (d *Delivery) shownOnItsPullRequest() bool { + return d.Number > 0 && d.History == "" && d.State != Retired +} + +// Owner and Repo split the repository. +func (d *Delivery) Owner() string { o, _, _ := strings.Cut(d.Repository, "/"); return o } +func (d *Delivery) Repo() string { _, r, _ := strings.Cut(d.Repository, "/"); return r } + +// IDOf is a delivery's id: the repository and twelve characters of its commit. +func IDOf(repository, commit string) string { + return strings.ToLower(repository) + "@" + shortOf(commit, 12) +} + +func shortOf(commit string, n int) string { + if len(commit) > n { + return commit[:n] + } + return commit +} + +// sameCommit is whether two commits are one, however either is abbreviated. +func sameCommit(a, b string) bool { + if a == "" || b == "" { + return false + } + if len(a) > len(b) { + a, b = b, a + } + return len(a) >= 7 && strings.HasPrefix(strings.ToLower(b), strings.ToLower(a)) +} + +// reviewedLine is the review record of a pull request's description (novox/hq ADR 0421 §5, issue 1022): +// `Reviewed-by: at `, on a line of its own, who as who@node. +var reviewedLine = regexp.MustCompile(`(?im)^\s*Reviewed-by:\s*(\S+)\s+at\s+([0-9a-f]{40}|[0-9a-f]{64})\b`) + +// ReviewIn reads the review record of a pull request's description: who reviewed and the full sha named. The last such +// line counts; empty when there is none. +func ReviewIn(body string) (who, sha string) { + for _, m := range reviewedLine.FindAllStringSubmatch(body, -1) { + who, sha = m[1], strings.ToLower(m[2]) + } + return who, sha +} + +// reviewWords is what the delivery's review record says of its head, for the ask: reviewed by whom, or, first, that it +// is not reviewed at this head (never a gate, ADR 0421 §5). ok is whether it is reviewed at this head. +func reviewWords(d *Delivery) (words string, ok bool) { + switch { + case d.ReviewedBy == "": + return "not reviewed at this head: the description holds no Reviewed-by line", false + case !strings.EqualFold(d.ReviewedAt, d.Commit): + return fmt.Sprintf("not reviewed at this head: the Reviewed-by line names %s, the head is %s", + shortOf(d.ReviewedAt, 12), shortOf(d.Commit, 12)), false + } + // The router refuses an @ in an ask's words: who@node is said "who on node". + return "reviewed by " + strings.Replace(d.ReviewedBy, "@", " on ", 1) + " at " + shortOf(d.Commit, 12), true +} + +// afterLines are the `after: ` lines of a pull request's description. +var afterLines = regexp.MustCompile(`(?im)^\s*after:\s*([A-Za-z0-9_.\-/]+)\s*$`) + +// AfterIn reads them. +func AfterIn(body string) []string { + var out []string + for _, m := range afterLines.FindAllStringSubmatch(body, -1) { + out = append(out, strings.TrimSuffix(m[1], ".git")) + } + sort.Strings(out) + return out +} + +// The walk: the controller's plan of one trunk commit, as its `plan-moved` and `delivery-walks` say it +// (mesh-controller inventory.Plan). Read, never written here. + +// The states of a walk as the controller keeps them. +const ( + walkBuilding = "building" + walkRolling = "rolling" + walkDone = "done" + walkFailed = "failed" + walkSuperseded = "superseded" +) + +// Walk is a walk whole, as the controller says it. +type Walk struct { + ID string `json:"id"` + Repository string `json:"repository"` + Branch string `json:"branch,omitempty"` + Commit string `json:"commit"` + Created time.Time `json:"created"` + Updated time.Time `json:"updated"` + State string `json:"state"` + Tier int `json:"tier"` + Tiers [][]string `json:"tiers"` + Modules map[string]*WalkModule `json:"modules"` + Note string `json:"note,omitempty"` + Revision int64 `json:"revision"` + Release *WalkRelease `json:"release,omitempty"` + Delivery *WalkDelivery `json:"delivery,omitempty"` + // Backups is what the backups the controller asks before it sends anything came to (novox/hq ADR 0404, issue + // 1068); absent when the plan moves no backed-up item. + Backups *WalkBackups `json:"backups,omitempty"` + // Phases is the walk's time from its merge, phase by phase, as the controller works it out (novox/hq ADR + // 0282): what a delivery's time is kept from. Absent from a controller that does not say it. + Phases *WalkPhases `json:"phases,omitempty"` +} + +// WalkBackups is the backups of a walk before its send, as the controller's plan record says them: state pending, +// done or failed (failed as soon as any entry failed), each backup asked, and the machines with no backup holder. +type WalkBackups struct { + State string `json:"state"` + Asked []WalkBackupAsked `json:"asked,omitempty"` + NoHolder []string `json:"no_holder,omitempty"` +} + +// WalkBackupAsked is one backup asked of a machine for a module: its state, and the backup's own words. +type WalkBackupAsked struct { + Machine string `json:"machine"` + Module string `json:"module"` + State string `json:"state"` + AskedAt *time.Time `json:"asked_at,omitempty"` + Words string `json:"words,omitempty"` +} + +// WalkBackupFailure is one backup that failed or stayed silent past its bound, as a delivery keeps it. +type WalkBackupFailure struct { + Machine string `json:"machine"` + Module string `json:"module"` + Words string `json:"words"` +} + +// WalkModule is one module of a walk. +type WalkModule struct { + State string `json:"state,omitempty"` + SentAt *time.Time `json:"sent_at,omitempty"` + First []string `json:"first,omitempty"` + FirstAt *time.Time `json:"first_at,omitempty"` + Commit string `json:"commit,omitempty"` + Why string `json:"why,omitempty"` + Build string `json:"build,omitempty"` + Gate *WalkGate `json:"gate,omitempty"` + GatedBy string `json:"gated_by,omitempty"` + // Made is what the walk's build of it did (novox/hq ADR 0350 §4): "reused from ", naming the pull + // request's head, the trunk commit and the digests, or "built: ". Empty from a controller before it. + Made string `json:"made,omitempty"` +} + +// WalkGate is a module's judging on its first machine. +type WalkGate struct { + Machines []string `json:"machines"` + From string `json:"from,omitempty"` + To string `json:"to,omitempty"` + Since *time.Time `json:"since,omitempty"` + Passes int `json:"passes,omitempty"` + Verdict string `json:"verdict,omitempty"` + Why string `json:"why,omitempty"` + JudgedAt *time.Time `json:"judged_at,omitempty"` + Rollback string `json:"rollback,omitempty"` + Carried []WalkCarry `json:"carried,omitempty"` +} + +// WalkCarry is one build a gated send carried. +type WalkCarry struct { + Module string `json:"module"` + Node string `json:"node"` + From string `json:"from,omitempty"` + To string `json:"to"` + Build string `json:"build,omitempty"` +} + +// WalkRelease is a backlog walk's way through the machines. +type WalkRelease struct { + Order []string `json:"order"` + Next int `json:"next"` + Gate *WalkGate `json:"gate,omitempty"` + Done []string `json:"done,omitempty"` + By string `json:"by,omitempty"` +} + +// WalkDelivery is what a walk waits for, and which merges it answers (novox/hq ADR 0276). +type WalkDelivery struct { + Awaits string `json:"awaits"` + Go *time.Time `json:"go,omitempty"` + By string `json:"by,omitempty"` + Why string `json:"why,omitempty"` + Stopped string `json:"stopped,omitempty"` + StoppedWhy string `json:"stopped_why,omitempty"` + // Merges are every merge the walk answers: each merge commit, its repository, and the commit of that + // repository the walk carries — the merge itself, or a later merge of its branch that contains it. Absent + // from a controller that does not say it: the walk then answers the merge of its own commit, and a merge + // the forge says its commit contains. + Merges []WalkMerge `json:"merges,omitempty"` + // TakenOverBy is, on a walk folded before it started, the walk that took it over and answers every merge + // it answered. + TakenOverBy string `json:"taken_over_by,omitempty"` + // WalkedAgainBy is, on a walk stopped without some of its merges, the batch that walks its other merges again, + // and LeftOut the merges it left out (novox/hq ADR 0299): a delivery of a merge left out ends stopped with it, + // one of a merge walked again follows that batch's walk. + WalkedAgainBy string `json:"walked_again_by,omitempty"` + LeftOut []WalkLeftOut `json:"left_out,omitempty"` + // Batch is set while the record is a batch, not yet a walk: its state `assembling` or `queued`. A batch, once + // cut, is the walk under the same id: its next record is `building`. A merge this owner reads in a batch is + // not held as unwalked while the batch is heard of. + Batch *WalkBatch `json:"batch,omitempty"` + // Unchanged are the modules the walk did not build because their build source holds none of the files its + // merges changed, each with why (novox/hq issue 611, ADR 0267 §5): the build the mesh holds is theirs. A delivery + // plan that read them whole names them; this says they matched what runs, not that the walk left them unbuilt. + // Absent from a controller that does not say it. + Unchanged map[string]string `json:"unchanged,omitempty"` +} + +// WalkMerge is one merge a walk answers. +type WalkMerge struct { + Repository string `json:"repository"` + Commit string `json:"commit"` + // Carried is the commit of the repository the walk carries for it: Commit itself, or a later merge of its + // branch that contains it. Empty means Commit. + Carried string `json:"carried,omitempty"` +} + +// WalkLeftOut is a merge a stop left out (novox/hq ADR 0299): a merge stopped, or a later merge of its +// repository's branch that contains it, Contains naming the stopped merge. +type WalkLeftOut struct { + Repository string `json:"repository"` + Commit string `json:"commit"` + Contains string `json:"contains,omitempty"` + // Shares are the modules it moves that a stopped merge moves too, when that is why it was left out: Contains + // then names that stopped merge, which it does not contain. + Shares []string `json:"shares,omitempty"` +} + +// leftOut is the merge of a repository a stop left out, when it did. +func (w Walk) leftOut(repository, merge string) (WalkLeftOut, bool) { + if w.Delivery == nil || merge == "" { + return WalkLeftOut{}, false + } + for _, o := range w.Delivery.LeftOut { + if strings.EqualFold(o.Repository, repository) && sameCommit(o.Commit, merge) { + return o, true + } + } + return WalkLeftOut{}, false +} + +// WalkBatch is a batch's window: when it closes unless another merge arrives, when it closes at the latest, +// and, once closed, the walk it waits behind. +type WalkBatch struct { + ClosesAt *time.Time `json:"closes_at,omitempty"` + AtMost *time.Time `json:"at_most,omitempty"` + Behind string `json:"behind,omitempty"` +} + +// The states a batch's record is in before it is a walk (novox/hq ADR 0276). +const ( + walkAssembling = "assembling" + walkQueued = "queued" +) + +// IsBatch is whether the record is a batch still to be cut into a walk. +func (w Walk) IsBatch() bool { return w.State == walkAssembling || w.State == walkQueued } + +// supersededNote reads the walk that took a superseded walk over from its note, for a controller that does not +// say TakenOverBy: "superseded at tier N by plan-… (…)". +var supersededNote = regexp.MustCompile(`^superseded at tier \d+ by (plan-[0-9A-Za-z_-]+)`) + +// takenOverBy is the walk that took this one over, said or read from its note; empty for none. +func (w Walk) takenOverBy() string { + if w.State != walkSuperseded { + return "" + } + if w.Delivery != nil && w.Delivery.TakenOverBy != "" { + return w.Delivery.TakenOverBy + } + if m := supersededNote.FindStringSubmatch(w.Note); m != nil { + return m[1] + } + return "" +} + +// carriedFor is the commit the walk carries for a merge of a repository, and whether it answers that merge by +// what it says: the merge named among its merges, or its own commit. +func (w Walk) carriedFor(repository, merge string) (string, bool) { + if w.Delivery != nil { + for _, m := range w.Delivery.Merges { + if strings.EqualFold(m.Repository, repository) && sameCommit(m.Commit, merge) { + if m.Carried != "" { + return m.Carried, true + } + return m.Commit, true + } + } + } + if strings.EqualFold(w.Repository, repository) && sameCommit(w.Commit, merge) { + return w.Commit, true + } + return "", false +} + +// WalkSeen is what a delivery keeps of its walk. +type WalkSeen struct { + ID string `json:"id"` + Commit string `json:"commit"` + State string `json:"state"` + Note string `json:"note,omitempty"` + Revision int64 `json:"revision"` + Waits bool `json:"waits,omitempty"` + WaitedFor string `json:"waited_for,omitempty"` + GoBy string `json:"go_by,omitempty"` + StoppedBy string `json:"stopped_by,omitempty"` + // StoppedWhy is why it was stopped; WalkedAgainBy, on a walk stopped without some merges, the batch that walks + // this delivery's merge again; Contains, when this delivery's merge was left out for containing a stopped + // merge, that merge (novox/hq ADR 0299). + StoppedWhy string `json:"stopped_why,omitempty"` + WalkedAgainBy string `json:"walked_again_by,omitempty"` + Contains string `json:"contains,omitempty"` + // Shares are, when this delivery's merge was left out for moving modules a stopped merge moves, those modules: + // built at the branch, which holds the stopped merge, they would deliver it (ADR 0299). + Shares []string `json:"shares,omitempty"` + Tier int `json:"tier"` + Tiers int `json:"tiers"` + Updated time.Time `json:"updated"` + // Branch is the branch the walk's commit was merged into; empty in a record kept before it was read. + Branch string `json:"branch,omitempty"` + // Created is when the controller made the walk: a delivery follows the newest walk that answers it. + Created time.Time `json:"created"` + // TakenOverBy is the walk that took a superseded one over (novox/hq ADR 0276): its deliveries follow it. + TakenOverBy string `json:"taken_over_by,omitempty"` + // Moved are the modules the walk sent, as its record names them; MovedSaid says they were read, which a + // record kept before novox/hq issue 474 did not. What a walk that ended before its word delivered is read + // from them. + Moved []string `json:"moved,omitempty"` + MovedSaid bool `json:"moved_said,omitempty"` + // Unchanged are the modules the walk said matched what runs, with why (novox/hq issue 611): not built, and + // nothing for it to deliver. + Unchanged map[string]string `json:"unchanged,omitempty"` + // Built is, per module, what the walk's build of it did (novox/hq ADR 0350 §4): "reused from " or + // "built: " — what `show` says beside the build phase. + Built map[string]string `json:"built,omitempty"` + // BackupsFailed are the backups before the send that failed, each with the backup's words (ADR 0404). + BackupsFailed []WalkBackupFailure `json:"backups_failed,omitempty"` +} + +// backupWords says the failed backups: the machine, the module and the backup's own words. +func (w *WalkSeen) backupWords() string { + var parts []string + for _, f := range w.BackupsFailed { + parts = append(parts, fmt.Sprintf("%s on %s: %s", f.Module, f.Machine, f.Words)) + } + return "a backup before the send failed, and nothing was sent: " + strings.Join(parts, "; ") +} + +// Seen is a walk as a delivery keeps it. +func (w Walk) Seen() *WalkSeen { + s := &WalkSeen{ID: w.ID, Commit: w.Commit, State: w.State, Note: w.Note, Revision: w.Revision, Tier: w.Tier, + Tiers: len(w.Tiers), Updated: w.Updated} + if d := w.Delivery; d != nil { + s.WaitedFor = d.Awaits + s.Waits = d.Awaits != "" && d.Go == nil + s.GoBy = d.By + s.StoppedBy, s.StoppedWhy = d.Stopped, d.StoppedWhy + if len(d.Unchanged) > 0 { + s.Unchanged = maps.Clone(d.Unchanged) + } + } + s.Branch = w.Branch + s.Created = w.Created + s.TakenOverBy = w.takenOverBy() + // Only a module sent, or passed on its first machine, was built for the machines: one the record names with + // why it was not sent, or a walk that never started, built nothing (review of novox/hq issue 474). + for name, m := range w.Modules { + if m != nil && (m.SentAt != nil || m.Gate != nil && m.Gate.Verdict == "passed") { + s.Moved = append(s.Moved, name) + } + if m != nil && m.Made != "" { + if s.Built == nil { + s.Built = map[string]string{} + } + s.Built[name] = m.Made + } + } + sort.Strings(s.Moved) + s.MovedSaid = true + if w.Backups != nil { + for _, a := range w.Backups.Asked { + if a.State == "failed" { + s.BackupsFailed = append(s.BackupsFailed, WalkBackupFailure{Machine: a.Machine, Module: a.Module, Words: a.Words}) + } + } + } + return s +} + +// endedUnstarted is whether a published delivery's walk ended done without ever being let go: a batch's walk the +// controller judged to move nothing it holds is done when it is cut (novox/hq issue 474). No start can come for it, +// so its record settles the delivery. +func (d *Delivery) endedUnstarted() bool { + return d.State == Published && d.Walk != nil && d.Walk.State == walkDone && !d.Walk.Started() +} + +// walkStopped is whether the walk it follows was stopped through this owner, and walkedAgain whether its merge is +// walked again by another walk since (novox/hq ADR 0299). +func (d *Delivery) walkStopped() bool { + return d.Walk != nil && d.Walk.State == walkFailed && d.Walk.StoppedBy != "" +} + +func (d *Delivery) walkedAgain() bool { return d.walkStopped() && d.Walk.WalkedAgainBy != "" } + +// stoppedWords is why a delivery ends with the walk it follows stopped: the stop, and the stopped merge its own +// contains when it was left out for that. +func (d *Delivery) stoppedWords() string { + s := fmt.Sprintf("its walk %s was stopped by %s", d.Walk.ID, d.Walk.StoppedBy) + if d.Walk.StoppedWhy != "" { + s += ": " + d.Walk.StoppedWhy + } + switch { + case len(d.Walk.Shares) > 0: + s += fmt.Sprintf("; its merge moves %s, as the stopped merge %s does: built at the branch, which holds that "+ + "merge, it would deliver it, so it was left out with it (novox/hq ADR 0299)", strings.Join(d.Walk.Shares, ", "), + shortOf(d.Walk.Contains, 12)) + case d.Walk.Contains != "": + s += fmt.Sprintf("; its merge contains the stopped merge %s, so it cannot be walked without delivering it, "+ + "and was left out with it (novox/hq ADR 0299)", shortOf(d.Walk.Contains, 12)) + } + return s +} + +// walkMovedItsModules is whether the walk it follows answered every module its plan moves, and whether that is +// known: each built, or said unchanged — its build source holds none of the files the walk's merges changed, so the +// build the mesh holds is its build (novox/hq issue 611) — from the modules the walk's record names, or, for a +// record kept before they were, from a walk of no tier, which moved nothing. Unknown when its plan is not known. +func (d *Delivery) walkMovedItsModules() (moved, known bool) { + if d.Walk == nil || d.Plan == nil { + return false, false + } + if !d.Walk.MovedSaid && d.Walk.Tiers > 0 { + return false, false + } + return len(d.notMovedBy()) == 0, true +} + +// notMovedBy is the modules its plan moves that the walk it follows neither built nor said unchanged: what a walk +// that ended without building them left undelivered. +func (d *Delivery) notMovedBy() []string { + var out []string + if d.Walk == nil || d.Plan == nil { + return nil + } + for _, m := range d.Plan.Moved { + if _, unchanged := d.Walk.Unchanged[m]; !containsString(d.Walk.Moved, m) && !unchanged { + out = append(out, m) + } + } + return out +} + +// unchangedOfIts is the modules its plan moves that the walk it follows said matched what runs, sorted (novox/hq +// issue 611). +func (d *Delivery) unchangedOfIts() []string { + var out []string + if d.Walk == nil || d.Plan == nil { + return nil + } + for _, m := range d.Plan.Moved { + if _, unchanged := d.Walk.Unchanged[m]; unchanged && !containsString(d.Walk.Moved, m) { + out = append(out, m) + } + } + sort.Strings(out) + return out +} + +// builtNoneOfIts is whether the walk it follows built none of the modules its plan moves and said each unchanged: +// its merge moves nothing, whichever commit the walk carried (ADR 0270 §1). +func (d *Delivery) builtNoneOfIts() bool { + return d.Plan != nil && len(d.Plan.Moved) > 0 && len(d.unchangedOfIts()) == len(d.Plan.Moved) +} + +// Started is whether the walk is past its wait: it waited for nobody, or was let go — whatever came of it +// since. A walk superseded or stopped while it waited never started. +func (s *WalkSeen) Started() bool { return s != nil && !s.Waits && (s.WaitedFor == "" || s.GoBy != "") } + +// Waited is whether it waited for a word at all. +func (s *WalkSeen) Waited() bool { return s != nil && s.WaitedFor != "" } + +// LetGoByAPerson is whether a person's `plans go` started it. +func (s *WalkSeen) LetGoByAPerson() bool { return s != nil && strings.HasPrefix(s.GoBy, "a person") } + +// Open is whether the controller still works it. +func (w Walk) Open() bool { return w.State == walkBuilding || w.State == walkRolling } + +// The steps that say a module reached every machine running it: sent to all at once, or to the rest after its +// first machines passed. +const ( + stepEveryMachine = "every machine running it" + stepTheRest = "the rest running it" +) + +// StepsOf are the machine steps a walk's record says, for one delivery: every module it moved, on its first +// machines from its gate, and the rest once sent. +func StepsOf(w Walk) []Step { + var out []Step + names := make([]string, 0, len(w.Modules)) + for n := range w.Modules { + names = append(names, n) + } + sort.Strings(names) + for _, name := range names { + m := w.Modules[name] + if m == nil || m.FirstAt == nil { + if m != nil && m.SentAt != nil && m.FirstAt == nil && m.Why == "" { + out = append(out, Step{Machine: stepEveryMachine, Module: name, Build: m.Build, + State: StepSent, Since: *m.SentAt}) + } + continue + } + gate := m.Gate + state, why, since := StepSent, "", *m.FirstAt + if gate != nil { + switch { + case gate.Rollback == "rolled-back": + state, why = StepRolledBack, gate.Why + case gate.Verdict == "failed": + state, why = StepFailed, gate.Why + case gate.Verdict == "passed": + state, why = StepPassed, gate.Why + case gate.Since != nil: + state = StepJudging + why = fmt.Sprintf("%d healthy judging(s) so far", gate.Passes) + } + if gate.JudgedAt != nil { + since = *gate.JudgedAt + } + } + for _, machine := range m.First { + out = append(out, Step{Machine: machine, Module: name, Build: m.Build, State: state, Since: since, Why: why}) + } + if m.SentAt != nil && state == StepPassed { + out = append(out, Step{Machine: stepTheRest, Module: name, Build: m.Build, State: StepSent, + Since: *m.SentAt}) + } + } + return out +} + +// Group is a delivery group: two or more deliveries sharing a head branch name, one level only. +type Group struct { + ID string `json:"id"` + Members []string `json:"members"` + Created time.Time `json:"created"` + Updated time.Time `json:"updated"` + // Check is its composed check: asked of the controller for these member commits, and its verdict. + Check *GroupCheck `json:"check,omitempty"` + // SaidGate is, by member id, why the member's mesh/merge-gate was last said pending: written again only when it + // changes (novox/hq ADR 0403 §1, §6, issue 978). Cleared once a composed verdict stands at the heads, which the + // forge module then sets as every member's gate. + SaidGate map[string]string `json:"said_gate,omitempty"` + // Closed is set once a member reached the trunk: a group delivering takes no new member. + Closed bool `json:"closed,omitempty"` + // Said is the derived state last said, so a change of it is said once. + Said string `json:"said,omitempty"` + // Hold is why the group's merge or its Approve's ask is held back by a member's required status that is not a + // success (issue 1275); nil when nothing is. + Hold *GroupHold `json:"hold,omitempty"` + // SaidStatus is the mesh/delivery-group status last written for it: a change of the status a state takes is + // written too, though the state did not change (novox/hq issue 902). + SaidStatus string `json:"said_status,omitempty"` + Owed []Effect `json:"owed,omitempty"` +} + +// GroupHold is a group held back from its Approve or its merge because a member's required status is not a success. +type GroupHold struct { + // Kind is "unasked" (the Approve is not asked) or "blocked" (the approved merge is not started: nothing is merged). + Kind string `json:"kind"` + Member string `json:"member"` + Says string `json:"says"` + Since time.Time `json:"since"` +} + +// GroupCheck is a group's composed check. +type GroupCheck struct { + Asked string `json:"asked"` + At time.Time `json:"at"` + Commits map[string]string `json:"commits"` + Verdict string `json:"verdict,omitempty"` + Summary string `json:"summary,omitempty"` + // Trunks are the trunk commit each member's head was merged into, by member id (novox/hq ADR 0323 §6). + Trunks map[string]string `json:"trunks,omitempty"` + // OutOfDate is why its verdict no longer counts: a merge that is not one of the group's members moved a + // member's trunk, at OutOfDateAt. It counts as outstanding (ADR 0239 §18), and runs again once a member is + // judged again after it. + OutOfDate string `json:"out_of_date,omitempty"` + OutOfDateAt time.Time `json:"out_of_date_at"` +} + +// Commits is each member's commit, by id. +func commitsOf(ds []*Delivery) map[string]string { + out := map[string]string{} + for _, d := range ds { + out[d.ID] = d.Commit + } + return out +} + +// sameMembers is whether two member→commit maps are one. +func sameMembers(a, b map[string]string) bool { + if len(a) != len(b) { + return false + } + for k, v := range a { + if b[k] != v { + return false + } + } + return true +} + +func mustJSON(v any) json.RawMessage { + raw, _ := json.Marshal(v) + return raw +} diff --git a/modules/mesh-delivery/cmd/mesh-delivery/operator_test.go b/modules/mesh-delivery/cmd/mesh-delivery/operator_test.go new file mode 100644 index 00000000..6c312b21 --- /dev/null +++ b/modules/mesh-delivery/cmd/mesh-delivery/operator_test.go @@ -0,0 +1,104 @@ +package main + +import ( + "sync" + "time" + + "git.novox.be/novox/mesh-sdk/go/asks" +) + +// fakeOperator is the operator channel: it keeps the asks made, and answers as the test says. In auto mode it +// approves every ask the moment it is read (the tests of other behaviour, which are not about the Approve). +type fakeOperator struct { + mu sync.Mutex + now func() time.Time + auto bool + asked []asks.Ask + cancelled []string + state map[string]string + warrant map[string]*asks.Warrant + refuse error +} + +func newFakeOperator(now func() time.Time) *fakeOperator { + return &fakeOperator{now: now, state: map[string]string{}, warrant: map[string]*asks.Warrant{}} +} + +func (o *fakeOperator) Ask(a asks.Ask) error { + o.mu.Lock() + defer o.mu.Unlock() + if o.refuse != nil { + return o.refuse + } + if err := a.Check(o.now()); err != nil { + return err + } + o.asked = append(o.asked, a) + o.state[a.ID] = "open" + return nil +} + +func (o *fakeOperator) Cancel(id string) error { + o.mu.Lock() + defer o.mu.Unlock() + o.cancelled = append(o.cancelled, id) + o.state[id] = "cancelled" + return nil +} + +func (o *fakeOperator) Where(id string) (string, *asks.Warrant, error) { + o.mu.Lock() + defer o.mu.Unlock() + if o.auto && o.state[id] == "open" { + o.answerLocked(id, OptionApprove) + } + return o.state[id], o.warrant[id], nil +} + +func (o *fakeOperator) ask(id string) asks.Ask { + for _, a := range o.asked { + if a.ID == id { + return a + } + } + panic("no ask " + id) +} + +func (o *fakeOperator) answerLocked(id, option string) { + a := o.ask(id) + opt, _ := a.Option(option) + o.state[id] = "answered" + o.warrant[id] = &asks.Warrant{Ask: id, Asker: Module, Outcome: asks.OutcomeChosen, Option: option, Label: opt.Label, + Level: opt.Level, By: &asks.Person{Who: asks.Operator, Kind: "telegram", Verified: "user id verified"}, + At: o.now(), AskDigest: a.Digest()} +} + +// answer is the operator's choice, proven by the router. +func (o *fakeOperator) answer(id, option string) { + o.mu.Lock() + defer o.mu.Unlock() + o.answerLocked(id, option) +} + +// end is an ask that closed without a choice. +func (o *fakeOperator) end(id string) { + o.mu.Lock() + defer o.mu.Unlock() + o.state[id] = "expired" + o.warrant[id] = &asks.Warrant{Ask: id, Asker: Module, Outcome: asks.OutcomeExpired, At: o.now()} +} + +// open is the ids of the asks waiting for an answer, in the order asked. +func (o *fakeOperator) open() []string { + o.mu.Lock() + defer o.mu.Unlock() + var out []string + for _, a := range o.asked { + if o.state[a.ID] == "open" { + out = append(out, a.ID) + } + } + return out +} + +func (o *fakeOperator) count() int { o.mu.Lock(); defer o.mu.Unlock(); return len(o.asked) } diff --git a/modules/mesh-delivery/cmd/mesh-delivery/ports.go b/modules/mesh-delivery/cmd/mesh-delivery/ports.go new file mode 100644 index 00000000..c893fe0f --- /dev/null +++ b/modules/mesh-delivery/cmd/mesh-delivery/ports.go @@ -0,0 +1,487 @@ +package main + +import ( + "encoding/json" + "errors" + "fmt" + "sort" + "strconv" + "strings" + + stdio "git.novox.be/novox/mesh-sdk/go" +) + +// What mesh-delivery asks of others, and where it keeps what it owns. **It never sends to a machine, writes +// a declaration, a grant or a bus object, and never registers a build** (novox/hq ADR 0239 decision 6): it +// asks the controller, through the controller's seat, and the forge's holder, through its tools. + +// ControllerSeat is the seat whose verbs the controller serves. +const ControllerSeat = "mesh-controller" + +// Member is one delivery as the controller's planner takes it (mesh-controller orderMember). +type Member struct { + ID string `json:"id"` + Repository string `json:"repository"` + Base string `json:"base,omitempty"` + Head string `json:"head,omitempty"` + Number int `json:"number,omitempty"` + Paths []string `json:"paths,omitempty"` + PathsTruncated bool `json:"paths_truncated,omitempty"` + Removed []string `json:"removed,omitempty"` + ModuleDirs []string `json:"module_dirs,omitempty"` + ModuleDirsSaid bool `json:"module_dirs_said,omitempty"` + CloneURL string `json:"clone_url,omitempty"` + // Ready is whether the operator's Approve of its merge is asked or given: its check is published on the ready stream (ADR 0406 §3, 0421 §10). + Ready bool `json:"ready,omitempty"` + // BaseSHA is the base branch's head the forge last announced the head with (novox/hq ADR 0323): the trunk commit + // the controller's verdict names when no build judges the head. + BaseSHA string `json:"base_sha,omitempty"` +} + +// MemberOf is a delivery as a member. +func MemberOf(d *Delivery) Member { + return Member{Ready: d.Ready != nil && d.Ready.Merged == nil && (d.Ready.AskID != "" || d.Ready.Approved != nil), ID: d.ID, Repository: d.Repository, Base: d.Base, Head: d.Commit, Number: d.Number, Paths: d.Paths, + PathsTruncated: d.PathsTruncated, Removed: d.Removed, ModuleDirs: d.ModuleDirs, ModuleDirsSaid: d.ModuleDirsSaid, + CloneURL: d.CloneURL, BaseSHA: d.BaseSHA} +} + +// Controller is the controller's verbs this owner asks with. +type Controller interface { + // Plan is the delivery plan of a diffset, kept nowhere. + Plan(repository, base, head string, paths, moduleDirs, removed []string) (*DeliveryPlan, error) + // Check asks a group's composed check, or — with no group and one member — that head's own check again. + Check(group string, members []Member) (string, error) + // Deliver lets a waiting walk start. + Deliver(walk, why string) error + // BackupsAgain asks the controller to take a held walk's failed backups again, and to go on when they pass + // (novox/hq ADR 0404 clause 4). + BackupsAgain(walk, why string) error + // Stop ends a walk, said as stopped by whom. + Stop(walk, why, by string) error + // StopWithout ends a walk and walks its other merges again without the merges named, each + // `@`, and without each later merge of their repository, which contains one (novox/hq + // ADR 0299). + StopWithout(walk, why, by string, without []string) (StopAnswer, error) + // Walks are the walks the controller keeps — one, given its id — and whether this seat has a holder on record. + Walks(walk string) (bool, []Walk, error) + // ForgetChecks lets go of the builds a pull request's merge checks made for its merge (novox/hq ADR 0350 §5). + ForgetChecks(owner, repo string, number int) error +} + +// Forge is what the forge's holder does for a delivery. +type Forge interface { + // Note appends one line to a commit's note under refs/notes/mesh-plan; a line already there is not added. + Note(owner, repo, commit, line string) error + // View keeps the delivery's view current on its pull request: one comment, edited in place. + View(owner, repo string, number int, body string) error + // Status sets one status of a commit, linking to the view. + Status(owner, repo, commit, context, state, description, target string) error + // Statuses reads a commit's statuses — a pull request's head's, given its number — and whether the branch + // it merges into would let it merge. Reads only. + Statuses(owner, repo string, number int, commit, base string) (*ForgeChecks, error) + // Contains is whether a commit is contained in another — an ancestor of it, or it. Reads only. + Contains(owner, repo, commit, in string) (bool, error) + // OpenPulls is every open pull request, with what its base requires and what its head has (novox/hq issue + // 347). Reads only. + OpenPulls() ([]OpenPull, error) + // Merge asks the forge's merge tool to merge a pull request at the head named (gitea_merge_pull_request, novox/hq + // issue 722): the one path every merge takes. approved are the heads the operator's Approve covers, each + // `owner/repo@` (ADR 0421 §9); the tool refuses a pull request whose head differs. Its refusal is the + // error, in its words. + Merge(owner, repo string, number int, head string, approved []string) error + // Comment adds a comment to a pull request. + Comment(owner, repo string, number int, body string) error +} + +// ForgeChecks is what the forge says of a commit's checks (the gitea module's gitea_commit_statuses). +type ForgeChecks struct { + Commit string `json:"commit"` + Pull *ForgePull `json:"pull,omitempty"` + Combined string `json:"combined,omitempty"` + Statuses []ForgeStatus `json:"statuses"` + Merge ForgeMerge `json:"merge"` +} + +// ForgePull is a pull request as the forge says it. +type ForgePull struct { + Number int `json:"number"` + Title string `json:"title,omitempty"` + State string `json:"state,omitempty"` + Merged bool `json:"merged,omitempty"` + Base string `json:"base,omitempty"` + Head string `json:"head,omitempty"` + HeadSHA string `json:"head_sha,omitempty"` + MergeCommit string `json:"merge_commit_sha,omitempty"` + HTMLURL string `json:"html_url,omitempty"` +} + +// ForgeStatus is one status of a commit, the newest of its context. +type ForgeStatus struct { + Context string `json:"context"` + State string `json:"state"` + Description string `json:"description"` + TargetURL string `json:"target_url,omitempty"` + CreatedAt string `json:"created_at,omitempty"` + UpdatedAt string `json:"updated_at,omitempty"` + Creator string `json:"creator,omitempty"` +} + +// ForgeMerge is whether the branch's protection lets the commit merge: every required status a success. +// Mergeable is nil when the protection could not be read. +type ForgeMerge struct { + Branch string `json:"branch"` + Protected bool `json:"protected"` + Required []string `json:"required"` + Blocking []struct { + Context string `json:"context"` + State string `json:"state"` + } `json:"blocking"` + Mergeable *bool `json:"mergeable"` + Says string `json:"says"` +} + +// Store keeps deliveries and groups: one key each, in the module's own state (ADR 0201). +type Store interface { + PutDelivery(d *Delivery) error + DeleteDelivery(id string) error + Deliveries() ([]*Delivery, error) + PutGroup(g *Group) error + DeleteGroup(id string) error + Groups() ([]*Group, error) +} + +// --- over the runtime ----------------------------------------------------------------------------------- + +// asking is how a tool is asked: stdio.Ask, or a test's. +type asking func(key string, body any) (json.RawMessage, error) + +// seatController asks the controller's seat through the runtime. +type seatController struct{ ask asking } + +// answerOf reads a verb's answer whatever wraps it: the controller's {output, ok, answer}, a text the +// runtime handed over, or the protocol's content list. +func answerOf(raw json.RawMessage) (json.RawMessage, string, bool) { + var s string + if json.Unmarshal(raw, &s) == nil { + return answerOf(json.RawMessage(s)) + } + var m map[string]json.RawMessage + if json.Unmarshal(raw, &m) != nil { + return raw, string(raw), true + } + if content, has := m["content"]; has { + var items []struct { + Text string `json:"text"` + } + var e struct { + IsError bool `json:"isError"` + } + _ = json.Unmarshal(raw, &e) + if json.Unmarshal(content, &items) == nil && len(items) > 0 { + inner, out, ok := answerOf(json.RawMessage(items[0].Text)) + return inner, out, ok && !e.IsError + } + } + var v struct { + Output string `json:"output"` + OK *bool `json:"ok"` + Answer json.RawMessage `json:"answer"` + } + if json.Unmarshal(raw, &v) == nil && v.OK != nil { + return v.Answer, v.Output, *v.OK + } + return raw, string(raw), true +} + +func lastLine(s string) string { + lines := strings.Split(strings.TrimSpace(s), "\n") + return strings.TrimSpace(lines[len(lines)-1]) +} + +func (c seatController) verb(name string, args map[string]any) (json.RawMessage, error) { + raw, err := c.ask("seat:"+ControllerSeat+"."+name, args) + if err != nil { + return nil, err + } + answer, output, ok := answerOf(raw) + if !ok { + return nil, fmt.Errorf("the controller refused %s: %s", name, lastLine(output)) + } + return answer, nil +} + +func (c seatController) Plan(repository, base, head string, paths, moduleDirs, removed []string) (*DeliveryPlan, error) { + args := map[string]any{"repository": repository, "paths": strings.Join(paths, ",")} + if base != "" { + args["base"] = base + } + if head != "" { + args["head"] = head + } + if len(moduleDirs) > 0 { + args["module-dirs"] = strings.Join(moduleDirs, ",") + } + if len(removed) > 0 { + args["removed"] = strings.Join(removed, ",") + } + raw, err := c.verb("delivery-plan", args) + if err != nil { + return nil, err + } + var a struct { + Plan *DeliveryPlan `json:"plan"` + } + if err := json.Unmarshal(raw, &a); err != nil || a.Plan == nil { + return nil, errors.New("the controller's delivery-plan answered no plan") + } + return a.Plan, nil +} + +func (c seatController) Check(group string, members []Member) (string, error) { + args := map[string]any{"members": string(mustJSON(members))} + if group != "" { + args["group"] = group + } + raw, err := c.verb("delivery-check", args) + if err != nil { + return "", err + } + var a struct { + Asked string `json:"asked"` + Rechecked string `json:"rechecked"` + } + _ = json.Unmarshal(raw, &a) + if a.Asked != "" { + return a.Asked, nil + } + return a.Rechecked, nil +} + +func (c seatController) Deliver(walk, why string) error { + _, err := c.verb("deliver", map[string]any{"plan": walk, "why": why}) + return err +} + +func (c seatController) BackupsAgain(walk, why string) error { + _, err := c.verb("delivery-backups-again", map[string]any{"plan": walk, "why": why}) + return err +} + +func (c seatController) ForgetChecks(owner, repo string, number int) error { + _, err := c.verb("forget-checks", map[string]any{"owner": owner, "repo": repo, "number": strconv.Itoa(number)}) + return err +} + +func (c seatController) Stop(walk, why, by string) error { + _, err := c.verb("delivery-stop", map[string]any{"plan": walk, "why": why, "by": by}) + return err +} + +// StopAnswer is what the controller's `delivery-stop` with merges to leave out answers (novox/hq ADR 0299): the walk +// stopped, the batch that walks its other merges again with those merges, and every merge left out. +type StopAnswer struct { + Stopped string `json:"stopped"` + WalkedAgainBy string `json:"walked_again_by,omitempty"` + WalkedAgain []string `json:"walked_again,omitempty"` + LeftOut []WalkLeftOut `json:"left_out,omitempty"` + // Kept are the modules the stopped walk sent that nothing walks again, on the machines named; Withheld the + // modules the walk that walks the others withholds; StillCarriedBy the merges elsewhere that will deliver a + // stopped change when walked — a later merge of its repository, or one moving its modules (ADR 0299). + Kept []WalkKept `json:"kept,omitempty"` + Withheld []string `json:"withheld,omitempty"` + StillCarriedBy []string `json:"still_carried_by,omitempty"` +} + +// WalkKept is a module a stopped walk left running its build, and where. +type WalkKept struct { + Module string `json:"module"` + Machines []string `json:"machines"` +} + +func (c seatController) StopWithout(walk, why, by string, without []string) (StopAnswer, error) { + raw, err := c.verb("delivery-stop", map[string]any{"plan": walk, "why": why, "by": by, + "without": strings.Join(without, ",")}) + if err != nil { + return StopAnswer{}, err + } + var a StopAnswer + if json.Unmarshal(raw, &a) != nil || a.Stopped == "" { + return StopAnswer{}, fmt.Errorf("the controller's delivery-stop did not say what it left out: %s", + lastLine(string(raw))) + } + return a, nil +} + +func (c seatController) Walks(walk string) (bool, []Walk, error) { + args := map[string]any{} + if walk != "" { + args["plan"] = walk + } + raw, err := c.verb("delivery-walks", args) + if err != nil { + return false, nil, err + } + var a struct { + Held bool `json:"held"` + Walks []Walk `json:"walks"` + } + if err := json.Unmarshal(raw, &a); err != nil { + return false, nil, fmt.Errorf("the controller's delivery-walks is not readable: %w", err) + } + return a.Held, a.Walks, nil +} + +// toolForge asks the forge's holder through its tools. +type toolForge struct{ ask asking } + +func (f toolForge) tool(name string, args map[string]any) error { + raw, err := f.ask("gitea."+name, args) + if err != nil { + return err + } + if _, output, ok := answerOf(raw); !ok { + return fmt.Errorf("the forge refused %s: %s", name, lastLine(output)) + } + return nil +} + +func (f toolForge) Note(owner, repo, commit, line string) error { + return f.tool("gitea_note_append", map[string]any{"owner": owner, "repo": repo, "sha": commit, + "ref": "mesh-plan", "line": line}) +} + +func (f toolForge) View(owner, repo string, number int, body string) error { + return f.tool("gitea_delivery_view", map[string]any{"owner": owner, "repo": repo, "number": number, "body": body}) +} + +func (f toolForge) Status(owner, repo, commit, context, state, description, target string) error { + return f.tool("gitea_commit_status", map[string]any{"owner": owner, "repo": repo, "sha": commit, + "context": context, "state": state, "description": description, "target_url": target}) +} + +func (f toolForge) Statuses(owner, repo string, number int, commit, base string) (*ForgeChecks, error) { + args := map[string]any{"owner": owner, "repo": repo} + if number > 0 { + args["number"] = number + } + if commit != "" { + args["sha"] = commit + } + if base != "" { + args["base"] = base + } + raw, err := f.ask("gitea.gitea_commit_statuses", args) + if err != nil { + return nil, err + } + answer, output, ok := answerOf(raw) + if !ok { + return nil, fmt.Errorf("the forge refused gitea_commit_statuses: %s", lastLine(output)) + } + var c ForgeChecks + if err := json.Unmarshal(answer, &c); err != nil || c.Commit == "" { + return nil, fmt.Errorf("the forge's gitea_commit_statuses is not readable: %s", clip(string(answer), 200)) + } + return &c, nil +} + +func (f toolForge) Merge(owner, repo string, number int, head string, approved []string) error { + return f.tool("gitea_merge_pull_request", map[string]any{"owner": owner, "repo": repo, "number": number, "head": head, + "approved_heads": approved}) +} + +func (f toolForge) Comment(owner, repo string, number int, body string) error { + return f.tool("gitea_add_comment", map[string]any{"owner": owner, "repo": repo, "number": number, "body": body}) +} + +func (f toolForge) OpenPulls() ([]OpenPull, error) { + raw, err := f.ask("gitea.gitea_open_pulls", map[string]any{}) + if err != nil { + return nil, err + } + answer, output, ok := answerOf(raw) + if !ok { + return nil, fmt.Errorf("the forge refused gitea_open_pulls: %s", lastLine(output)) + } + var open struct { + Pulls []OpenPull `json:"pulls"` + } + if err := json.Unmarshal(answer, &open); err != nil || open.Pulls == nil { + return nil, fmt.Errorf("the forge's gitea_open_pulls is not readable: %s", clip(string(answer), 200)) + } + return open.Pulls, nil +} + +func (f toolForge) Contains(owner, repo, commit, in string) (bool, error) { + raw, err := f.ask("gitea.gitea_contains", map[string]any{"owner": owner, "repo": repo, "commit": commit, "in": in}) + if err != nil { + return false, err + } + answer, output, ok := answerOf(raw) + if !ok { + return false, fmt.Errorf("the forge refused gitea_contains: %s", lastLine(output)) + } + var a struct { + Contains *bool `json:"contains"` + } + if err := json.Unmarshal(answer, &a); err != nil || a.Contains == nil { + return false, fmt.Errorf("the forge's gitea_contains is not readable: %s", clip(string(answer), 200)) + } + return *a.Contains, nil +} + +// kvStore is the module's own state: `deliveries` and `groups`, one key each. +type kvStore struct{} + +// kvKey is an id as a bucket key: only the characters a key may carry. +func kvKey(id string) string { + var b strings.Builder + for _, r := range id { + switch { + case r >= 'a' && r <= 'z', r >= 'A' && r <= 'Z', r >= '0' && r <= '9', r == '-', r == '_', r == '=': + b.WriteRune(r) + default: + b.WriteString("_" + fmt.Sprintf("%02x", r) + "_") + } + } + return b.String() +} + +func (kvStore) PutDelivery(d *Delivery) error { + _, err := stdio.State("deliveries").Put(kvKey(d.ID), d) + return err +} +func (kvStore) DeleteDelivery(id string) error { return stdio.State("deliveries").Delete(kvKey(id)) } +func (kvStore) PutGroup(g *Group) error { + _, err := stdio.State("groups").Put(kvKey(g.ID), g) + return err +} +func (kvStore) DeleteGroup(id string) error { return stdio.State("groups").Delete(kvKey(id)) } + +func readAll[T any](state string) ([]*T, error) { + keys, err := stdio.State(state).Keys() + if err != nil { + return nil, err + } + sort.Strings(keys) + var out []*T + for _, k := range keys { + e, err := stdio.State(state).Get(k) + if err != nil { + return nil, err + } + if e == nil { + continue + } + var v T + if err := json.Unmarshal(e.Value, &v); err != nil { + logf("[mesh-delivery] %s %s cannot be read (%v); left as it is", state, k, err) + continue + } + out = append(out, &v) + } + return out, nil +} + +func (kvStore) Deliveries() ([]*Delivery, error) { return readAll[Delivery]("deliveries") } +func (kvStore) Groups() ([]*Group, error) { return readAll[Group]("groups") } diff --git a/modules/mesh-delivery/cmd/mesh-delivery/pullkey.go b/modules/mesh-delivery/cmd/mesh-delivery/pullkey.go new file mode 100644 index 00000000..81bdaa67 --- /dev/null +++ b/modules/mesh-delivery/cmd/mesh-delivery/pullkey.go @@ -0,0 +1,115 @@ +package main + +import ( + "encoding/json" + "errors" + "fmt" + "strings" + + stdio "git.novox.be/novox/mesh-sdk/go" +) + +// The pull request's key (novox/hq ADR 0406 §4, issue 975): mesh-delivery keeps one key per pull request on the bus, +// holding its head, the trunk commit its current check is asked against and its verdict, written by compare-and-set. +// It writes a new head when the forge says one (`pull.updated`, which carries the head and the trunk commit it was +// announced against), and a new trunk commit only when the check is asked again against one — the merge tool's ask is +// that same announcement. A merge of a group's member, and a trunk move that leaves an inactive pull request unasked, +// are announced as nothing (the gitea module announces `pull.updated` only for a head and for the merge tool's ask), so +// they write nothing. A verdict for that head and trunk commit writes the verdict. The build seat reads the key. + +// PullKey is what the bus keeps of one pull request. +type PullKey struct { + Head string `json:"head"` + Trunk string `json:"trunk,omitempty"` + Verdict string `json:"verdict,omitempty"` +} + +// PullKeys keeps the pull requests' keys: Change reads the key, applies the change to what it read and writes the result +// by compare-and-set when the change says to write; a write another writer won reads the key again and applies again. +type PullKeys interface { + Change(key string, apply func(PullKey) (PullKey, bool)) error +} + +// pullKeyName is a pull request's key: owner.repo.number, with any dot in a name made an underscore. +func pullKeyName(owner, repo string, number int) string { + clean := func(s string) string { return strings.NewReplacer(".", "_", "/", "_").Replace(s) } + return fmt.Sprintf("%s.%s.%d", clean(owner), clean(repo), number) +} + +// KeepPull writes the key of a pull request whose head was announced, when the head or the trunk commit it was +// announced against is not what the key holds; a new head or trunk clears the verdict. +func (h *Holder) KeepPull(p PullEvent) { + if h.Pulls == nil || p.Number == 0 || p.HeadSHA == "" { + return + } + err := h.Pulls.Change(pullKeyName(p.Owner, p.Repo, p.Number), func(k PullKey) (PullKey, bool) { + if sameCommit(k.Head, p.HeadSHA) && k.Head != "" && (p.BaseSHA == "" || sameCommit(k.Trunk, p.BaseSHA)) { + return k, false + } + return PullKey{Head: p.HeadSHA, Trunk: p.BaseSHA}, true + }) + if err != nil { + h.Logf("[mesh-delivery] the key of %s/%s#%d could not be written: %v", p.Owner, p.Repo, p.Number, err) + h.again("a pull request's key that could not be written") + } +} + +// KeepVerdict writes a verdict into the key of its pull request, when the key holds that head and trunk commit. +func (h *Holder) KeepVerdict(c CheckedEvent) { + if h.Pulls == nil || c.Group != "" || c.Number == 0 || c.Commit == "" || c.Trunk == "" { + return + } + verdict := c.Verdict + if verdict == "" { + return + } + err := h.Pulls.Change(pullKeyName(c.Owner, c.Repo, c.Number), func(k PullKey) (PullKey, bool) { + if !sameCommit(k.Head, c.Commit) || !sameCommit(k.Trunk, c.Trunk) || k.Verdict == verdict { + return k, false + } + k.Verdict = verdict + return k, true + }) + if err != nil { + h.Logf("[mesh-delivery] the verdict in the key of %s/%s#%d could not be written: %v", c.Owner, c.Repo, c.Number, err) + h.again("a pull request's key that could not be written") + } +} + +// kvPulls keeps the keys in the module's own state `pulls`. +type kvPulls struct{} + +// changeAttempts is how often a lost compare-and-set is read and applied again before it is said. +const changeAttempts = 10 + +func (kvPulls) Change(key string, apply func(PullKey) (PullKey, bool)) error { + state := stdio.State("pulls") + for range changeAttempts { + e, err := state.Get(key) + if err != nil { + return err + } + var k PullKey + if e != nil { + if err := json.Unmarshal(e.Value, &k); err != nil { + return err + } + } + next, write := apply(k) + if !write { + return nil + } + if e == nil { + _, err = state.Create(key, next) + } else { + _, err = state.Update(key, next, e.Revision) + } + if err == nil { + return nil + } + if !errors.Is(err, stdio.ErrChanged) { + return err + } + } + return fmt.Errorf("%s was written by another writer first %d times: %w", key, changeAttempts, stdio.ErrChanged) +} diff --git a/modules/mesh-delivery/cmd/mesh-delivery/pullkey_test.go b/modules/mesh-delivery/cmd/mesh-delivery/pullkey_test.go new file mode 100644 index 00000000..05d44623 --- /dev/null +++ b/modules/mesh-delivery/cmd/mesh-delivery/pullkey_test.go @@ -0,0 +1,174 @@ +package main + +import ( + "errors" + "sync" + "testing" +) + +// **The pull request's key** (novox/hq ADR 0406 §4, issue 975): one key per pull request on the bus, holding its head, +// the trunk commit its current check is asked against and its verdict, written by compare-and-set. A head the forge +// announces writes head and trunk; a verdict for that head and trunk writes the verdict; nothing else writes. + +type fakePulls struct { + mu sync.Mutex + keys map[string]PullKey + writes int + // lose makes the first n writes lose to another writer, which sets rival. + lose int + rival PullKey +} + +func newFakePulls() *fakePulls { return &fakePulls{keys: map[string]PullKey{}} } + +func (f *fakePulls) Change(key string, apply func(PullKey) (PullKey, bool)) error { + f.mu.Lock() + defer f.mu.Unlock() + for tries := 0; tries < 5; tries++ { + next, write := apply(f.keys[key]) + if !write { + return nil + } + if f.lose > 0 { // another writer was first: read again and apply again + f.lose-- + f.keys[key] = f.rival + continue + } + f.keys[key] = next + f.writes++ + return nil + } + return errors.New("lost the race too often") +} + +func (f *fakePulls) get(key string) PullKey { + f.mu.Lock() + defer f.mu.Unlock() + return f.keys[key] +} + +const appKey = "novox.app.7" + +func announcedAt(head, trunk string) PullEvent { + p := pr("novox/app", 7, head, "feat/x", "src/a.go") + p.BaseSHA = trunk + return p +} + +func TestAnAnnouncedHeadWritesTheKeyAndTheSameAnnouncementAgainWritesNothing(t *testing.T) { + w := newWorld(t) + pulls := newFakePulls() + w.h.Pulls = pulls + p := announcedAt(readyHead, "t1t1t1t1t1t1") + w.h.PullUpdated(p) + w.h.KeepPull(p) + if got := pulls.get(appKey); got.Head != readyHead || got.Trunk != "t1t1t1t1t1t1" || got.Verdict != "" { + t.Fatalf("the key holds %+v", got) + } + w.h.KeepPull(p) + if pulls.writes != 1 { + t.Fatalf("the same announcement wrote %d times", pulls.writes) + } +} + +func TestAVerdictForTheKeysHeadAndTrunkIsWrittenAndOneForAnotherIsNot(t *testing.T) { + w := newWorld(t) + pulls := newFakePulls() + w.h.Pulls = pulls + w.h.KeepPull(announcedAt(readyHead, "t1t1t1t1t1t1")) + old := judgedAt("novox/app", 7, pushed, "pass", "t1t1t1t1t1t1") + w.h.KeepVerdict(old) + if pulls.get(appKey).Verdict != "" { + t.Fatal("a verdict for another head was written") + } + other := judgedAt("novox/app", 7, readyHead, "pass", "t9t9t9t9t9t9") + w.h.KeepVerdict(other) + if pulls.get(appKey).Verdict != "" { + t.Fatal("a verdict for another trunk commit was written") + } + w.h.KeepVerdict(judgedAt("novox/app", 7, readyHead, "pass", "t1t1t1t1t1t1")) + if got := pulls.get(appKey); got.Verdict != "pass" { + t.Fatalf("the key holds %+v", got) + } +} + +func TestANewTrunkCommitClearsTheVerdictAndANewHeadReplacesTheKey(t *testing.T) { + w := newWorld(t) + pulls := newFakePulls() + w.h.Pulls = pulls + w.h.KeepPull(announcedAt(readyHead, "t1t1t1t1t1t1")) + w.h.KeepVerdict(judgedAt("novox/app", 7, readyHead, "pass", "t1t1t1t1t1t1")) + w.h.KeepPull(announcedAt(readyHead, "t2t2t2t2t2t2")) + if got := pulls.get(appKey); got.Trunk != "t2t2t2t2t2t2" || got.Verdict != "" { + t.Fatalf("the check asked again against a new trunk left %+v", got) + } + w.h.KeepPull(announcedAt(pushed, "t2t2t2t2t2t2")) + if got := pulls.get(appKey); got.Head != pushed { + t.Fatalf("a new head left %+v", got) + } +} + +func TestALostRaceReadsTheKeyAgainAndAppliesTheChangeToWhatItFound(t *testing.T) { + w := newWorld(t) + pulls := newFakePulls() + pulls.lose, pulls.rival = 1, PullKey{Head: readyHead, Trunk: "t1t1t1t1t1t1", Verdict: "pass"} + w.h.Pulls = pulls + // The rival writer already holds this very head and trunk with a verdict: the change finds nothing to write. + w.h.KeepPull(announcedAt(readyHead, "t1t1t1t1t1t1")) + if got := pulls.get(appKey); got.Verdict != "pass" || pulls.writes != 0 { + t.Fatalf("after the lost race the key holds %+v after %d writes", got, pulls.writes) + } +} + +// A check asked for a head whose Approve is asked is on the ready stream (ADR 0406 §3, ADR 0421 §10); one for a head +// with no Approve is asked as before. +func TestAPullRequestWithItsApproveAskedHasItsCheckAskedOnTheReadyStream(t *testing.T) { + w := newWorld(t) + w.op.auto = false + w.h.PullUpdated(pr("novox/app", 7, readyHead, "feat/x", "src/a.go")) + w.h.Checked(judgedAt("novox/app", 7, readyHead, "pass", "t1t1t1t1t1t1")) + w.settleAll() + w.h.PullMerged(merged("novox/app", 9, "9999999999999999", "t2t2t2t2t2t2")) + w.settleAll() + if len(w.ctl.checkReady) == 0 || !w.ctl.checkReady[0] { + t.Fatalf("the check of a pull request whose Approve is asked was asked with ready %v", w.ctl.checkReady) + } + // One with no Approve asked is asked as before. + w2 := newWorld(t) + w2.h.PullUpdated(pr("novox/app", 8, pushed, "feat/y", "src/a.go")) + w2.settleAll() + if len(w2.ctl.checkReady) == 0 || w2.ctl.checkReady[0] { + t.Fatalf("the check of a pull request with no Approve asked was asked with ready %v", w2.ctl.checkReady) + } +} + +func TestATrunkLessVerdictWritesNothingToTheKey(t *testing.T) { + w := newWorld(t) + pulls := newFakePulls() + w.h.Pulls = pulls + w.h.KeepPull(announcedAt(readyHead, "t1t1t1t1t1t1")) + w.h.KeepVerdict(judgedAt("novox/app", 7, readyHead, "pass", "")) + if got := pulls.get(appKey); got.Verdict != "" { + t.Fatalf("a verdict naming no trunk commit was written: %+v", got) + } +} + +// The forge's catch-up after a restart can announce a merge again past the bus's de-duplication window (issue 975): +// heard again, it changes nothing, and moves no trunk back. +func TestTheSameMergeHeardAgainAfterALaterMergeMovesNoTrunkBack(t *testing.T) { + w := newWorld(t) + a, b := "aaaaaaaaaaaaaaaa", "bbbbbbbbbbbbbbbb" + w.h.PullUpdated(pr("novox/app", 7, readyHead, "feat/x", "src/a.go")) + w.h.PullUpdated(pr("novox/app", 8, a, "feat/y", "src/a.go")) + w.h.PullUpdated(pr("novox/app", 9, b, "feat/z", "src/a.go")) + w.h.Checked(judgedAt("novox/app", 7, readyHead, "pass", "m8m8m8m8m8m8")) + w.h.PullMerged(merged("novox/app", 8, a, "m8m8m8m8m8m8")) + w.h.PullMerged(merged("novox/app", 9, b, "m9m9m9m9m9m9")) + id := IDOf("novox/app", readyHead) + before := w.delivery(id) + w.h.PullMerged(merged("novox/app", 8, a, "m8m8m8m8m8m8")) // announced again, past the window + after := w.delivery(id) + if after.TrunkNow != before.TrunkNow || after.TrunkNow != "m9m9m9m9m9m9" { + t.Fatalf("the trunk moved from %q to %q on a merge heard again; wanted it to stay m9m9m9m9m9m9", before.TrunkNow, after.TrunkNow) + } +} diff --git a/modules/mesh-delivery/cmd/mesh-delivery/react.go b/modules/mesh-delivery/cmd/mesh-delivery/react.go new file mode 100644 index 00000000..a1edeff8 --- /dev/null +++ b/modules/mesh-delivery/cmd/mesh-delivery/react.go @@ -0,0 +1,119 @@ +package main + +import ( + "sync" + "time" +) + +// How this owner keeps time (novox/hq ADR 0406 §1, issue 975). It has no tick, no forge poll and no reconcile loop: +// a look (React) is taken after each event that can change what a step needs, and a step that waits on the clock — a +// grace, a bound, a retry after a failure — says when it must be looked at again (lookAt). The one timer below is +// armed for the earliest of those and fires one more look; it is never a tick, since a look that finds nothing +// waiting arms nothing. + +// askAgainAfter is how long after a failed ask of the controller, the forge or the operator it is asked again: the +// failure is the reason, and it is said in the log when it happens. +const askAgainAfter = 30 * time.Second + +// lookWanted is the earliest time a look must be taken again, and the bounded step that wants it. +type lookWanted struct { + mu sync.Mutex + at time.Time + step string + working bool + timer *time.Timer +} + +// lookAt says a bounded step, waiting on the clock, must be looked at again at a time. Safe to call with or without +// the holder's lock. +func (h *Holder) lookAt(at time.Time, step string) { + h.wanted.mu.Lock() + defer h.wanted.mu.Unlock() + if h.wanted.at.IsZero() || at.Before(h.wanted.at) { + h.wanted.at, h.wanted.step = at, step + } +} + +// again says an ask that failed is asked again after askAgainAfter: a bounded step retried by the look's timer. +func (h *Holder) again(step string) { h.lookAt(h.Now().Add(askAgainAfter), step) } + +// Poke asks for a look; any number of pokes before it starts are one look. +func (h *Holder) Poke() { + select { + case h.poke <- struct{}{}: + default: + } +} + +// Work takes a look each time it is poked, for ever: the one goroutine that acts on events. +func (h *Holder) Work() { + h.wanted.mu.Lock() + h.wanted.working = true + h.wanted.mu.Unlock() + for range h.poke { + h.React() + } +} + +// React takes every step the facts allow, after an event that can change what they are: the forge's open pull requests +// read when a check may have answered, the steps that need another's word (look), what is owed (Flush), and then +// the time the next look is wanted at, if any step waits on the clock. Never concurrent with itself. +func (h *Holder) React() { + h.reactMu.Lock() + defer h.reactMu.Unlock() + h.wanted.mu.Lock() + h.wanted.at, h.wanted.step = time.Time{}, "" + h.wanted.mu.Unlock() + if h.uncheckedDue() { + h.wantForge() + } + if h.takeForgeWanted() { + if err := h.SeeOpenPulls(); err != nil { + h.wantForge() + h.again("the forge's open pull requests, read when a check may have changed") + } + } + h.look() + if h.takeReread() { + // The one read of the controller's walks besides the one at the start: a merge no walk opened for, at its bound. + if err := h.Reconcile(); err != nil { + h.Logf("[mesh-delivery] the controller's walks cannot be read: %v", err) + h.mu.Lock() + h.rereadWalks = true + h.mu.Unlock() + h.again("the controller's walks, read once at the bound of a merge no walk opened for") + } else { + h.look() + } + } + h.Flush() + h.arm() +} + +// arm sets the timer for the earliest time a step wants a look at, or stops it when none does. +func (h *Holder) arm() { + h.wanted.mu.Lock() + defer h.wanted.mu.Unlock() + if !h.wanted.working { + return + } + if h.wanted.timer != nil { + h.wanted.timer.Stop() + h.wanted.timer = nil + } + if h.wanted.at.IsZero() { + return + } + wait := max(h.wanted.at.Sub(h.Now()), 100*time.Millisecond) + // retries: the time-bound step the last look left waiting (a grace, a bound or an ask that failed), at its time + h.wanted.timer = time.AfterFunc(wait, h.Poke) +} + +// takeReread answers whether the controller's walks are to be read once more, and takes the want. +func (h *Holder) takeReread() bool { + h.mu.Lock() + defer h.mu.Unlock() + reread := h.rereadWalks + h.rereadWalks = false + return reread +} diff --git a/modules/mesh-delivery/cmd/mesh-delivery/ready.go b/modules/mesh-delivery/cmd/mesh-delivery/ready.go new file mode 100644 index 00000000..4a8a6fcd --- /dev/null +++ b/modules/mesh-delivery/cmd/mesh-delivery/ready.go @@ -0,0 +1,528 @@ +package main + +import ( + "fmt" + "sort" + "strings" + "time" + + "git.novox.be/novox/mesh-sdk/go/asks" +) + +// The Approve is asked as soon as the checks pass (novox/hq ADR 0421, issue 1022). +// +// Before, a reviewer marked a pull request ready (a label and a comment) and only then was the operator's Approve asked +// (issue 722, ADR 0364). The review now finishes before the push (playbook 07) and its record, a line in the pull +// request's description, is given when the pull request is opened, so the mark repeated what the push already meant and +// added a step and a wait. This owner asks the operator's Approve itself, within the event that makes it possible: the +// verdict that passes, the commit status that sets the checks. Then: +// +// - the Approve is the only act: nothing is merged before it, and nothing else is needed after it (no label, comment +// or second Approve); it is asked once for a head, and once for a group's heads together; +// - it merges through the forge's merge tool (gitea_merge_pull_request), the one path every merge takes, at the head +// the Approve names, handing the tool the heads the Approve covers (ADR 0421 §9); +// - only when its delivery is ready here: its check passed against its trunk's head. A verdict out of date is asked +// again by this owner at once (askFresh), and an Approve already asked stays through a trunk move (ADR 0421 §3): the +// merge is asked again on the passing verdict, with no second Approve; +// - a group member only once the group's composed check passed for its members' heads as they are; +// - a push ends the Approve and says so once on the pull request (ADR 0421 §6); the new head is asked on its own; +// - a check that failed merges nothing: it is said once on the pull request, and as a stalled line, which the +// controller raises as the condition `delivery..stalled`. So is a merge the tool refused for an hour. +// +// The review record is read and put in the ask; it is never a gate (ADR 0421 §5). What this owner does is kept on the +// delivery (Ready), said in its log, and the merge's transitions name the Approve as their warrant. + +// readyRefusedFor is how long the merge tool may refuse an approved head before it is said as stalled: none (issue 1115) — +// a refusal is said at once, and the merge is asked again at each event that can change it, with no timer. +const readyRefusedFor time.Duration = 0 + +// groupRefusedFor is how long the merge tool may refuse a closed group's member before it is said as stalled: none (issue 1115). +const groupRefusedFor time.Duration = 0 + +// groupClosed is whether a group is closed: a member reached the trunk. Under the lock. +func (h *Holder) groupClosed(id string) bool { + g := h.groups[id] + return g != nil && g.Closed +} + +// ReadyRun is what this owner did with a delivery's Approve and merge: kept on the delivery and shown whole by `show`. +type ReadyRun struct { + // Head is the head the run is for. + Head string `json:"head"` + // Seen is when this owner first made the run: the checks passed. + Seen time.Time `json:"seen"` + // Waits is what the merge waits on, when it waits: a verdict, the group's composed check or the operator's Approve. + Waits string `json:"waits,omitempty"` + // Tries is how many times the merge was asked; LastTry when, and Answer what the merge tool last answered. + Tries int `json:"tries,omitempty"` + LastTry time.Time `json:"last_try"` + Answer string `json:"answer,omitempty"` + // merging is whether a merge ask is on its way, outside the lock: not asked twice meanwhile. Not kept. + merging bool + // OwnAskedAt is when its own check was asked after the merge tool refused it for a pending status of its own (issue + // 1049): once for the head this run is for. + OwnAskedAt time.Time `json:"own_asked_at,omitempty"` + // Urgent is set once a refusal of this head was one that is urgent at once: its severity never steps down to a + // warning after (issue 1275, AC6). + Urgent bool `json:"urgent,omitempty"` + // RefusedSince is when the merge tool began refusing it without a break. + RefusedSince *time.Time `json:"refused_since,omitempty"` + // Merged is when the merge tool merged it. + Merged *time.Time `json:"merged,omitempty"` + // Failed is why it is not merged and will not be on this head: its check failed. Said once on the pull request. + Failed string `json:"failed,omitempty"` + // OtherBase is the branch its pull request merges into when that is not main: never asked, never merged here, said + // once on the pull request (novox/hq ADR 0398 §4, issue 948). + OtherBase string `json:"other_base,omitempty"` + + // The operator's Approve of the merge into main (novox/hq ADR 0398 §3, issue 948, approval.go). AskID is the ask + // waiting for an answer, Ask the ask as made (a warrant is held to it), Heads the heads it covers by delivery id (a + // group's members together), AskedAt when and Asks how many were made for this head. + AskID string `json:"ask_id,omitempty"` + Ask *asks.Ask `json:"ask,omitempty"` + Heads map[string]string `json:"heads,omitempty"` + AskedAt time.Time `json:"asked_at"` + Asks int `json:"asks,omitempty"` + // Approved and Declined are when the operator chose, Warrant what the router's warrant says of who and how; Refused + // why an answer was not taken (it did not hold to the ask). + Approved *time.Time `json:"approved,omitempty"` + Declined *time.Time `json:"declined,omitempty"` + Warrant string `json:"warrant,omitempty"` + Refused string `json:"refused,omitempty"` +} + +// MergeBranch is the one branch this owner merges into (novox/hq ADR 0398 §4): a merge into any other is its author's. +const MergeBranch = "main" + +// mergedBy is who a merge this owner made on an Approve is said to be by, in its transitions. +func (r *ReadyRun) mergedBy() string { + if r == nil || r.Merged == nil { + return "" + } + by := byOwner + ", on the operator's Approve at " + shortOf(r.Head, 12) + if r.Warrant != "" { + by += "; the merge into main approved: " + r.Warrant + } + return by +} + +// pullKey is a pull request as a key: owner/repo#number. +func pullKey(p OpenPull) string { + return strings.ToLower(p.Owner+"/"+p.Repo) + fmt.Sprintf("#%d", p.Number) +} + +// runOf is the delivery's run, made when it has none. +func (h *Holder) runOf(d *Delivery, now time.Time) *ReadyRun { + if d.Ready == nil { + d.Ready = &ReadyRun{Head: d.Commit, Seen: now} + h.keep(d) + } + return d.Ready +} + +// readyAsks works out, under the lock, what each delivery whose checks passed needs: the operator asked, a merge asked, +// a failure said, an out of date verdict's check asked again. The asking happens outside the lock. +func (h *Holder) readyAsks(now time.Time) []func() { + var asks []func() + h.endStaleAsks() + ids := make([]string, 0, len(h.deliveries)) + for id := range h.deliveries { + ids = append(ids, id) + } + sort.Strings(ids) + for _, id := range ids { + d := h.deliveries[id] + if d.Number == 0 || d.State.Final() || d.MergedAs != "" { + continue // not of a pull request, or past its merge + } + if d.Ready == nil && d.State == Ready { + h.runOf(d, now) + h.Poke() // its check, if none is asked, is worth asking now (askCheck) + h.Logf("[mesh-delivery] %s: its checks passed; the operator's Approve of its merge into main is asked once it may merge", d.ID) + } + if a := h.sayAfterLine(d); a != nil { + asks = append(asks, a) + } + if a := h.askFresh(d, now); a != nil { + asks = append(asks, a) + } + r := d.Ready + if r == nil || r.Merged != nil || r.Declined != nil { + continue + } + if d.State == Rejected { + if r.Failed == "" { + r.Failed = "its check failed" + if d.Check != nil { + r.Failed += ": " + oneLine(d.Check.Summary) + if j := d.Check.Judged(); j != "" { + r.Failed += " (" + j + ")" + } + } + r.Waits = "" + h.Logf("[mesh-delivery] %s passed its checks and is not merged: %s", d.ID, r.Failed) + h.keep(d) + asks = append(asks, h.sayOn(d, fmt.Sprintf("mesh-delivery: **not merged.** This pull request's checks passed at "+ + "%s, and now %s. A new head, or a recheck that passes, asks the operator again.", d.Commit, r.Failed))) + } + continue + } + if d.Base != MergeBranch { + if r.OtherBase != d.Base { + r.OtherBase = d.Base + h.Logf("[mesh-delivery] %s passed its checks and is not asked: it merges into %s, not %s", d.ID, d.Base, MergeBranch) + h.keep(d) + asks = append(asks, h.sayOn(d, fmt.Sprintf("mesh-delivery: **not merged.** This pull request's checks passed at "+ + "%s, and it merges into %s. Only a merge into %s is this owner's; a merge into another branch is its "+ + "author's (novox/hq ADR 0398 §4).", d.Commit, d.Base, MergeBranch))) + } + continue + } + r.OtherBase = "" + r.Failed = "" + waits := h.readyWaits(d) + if waits != r.Waits { + r.Waits = waits + if waits != "" { + h.Logf("[mesh-delivery] %s waits: %s", d.ID, waits) + } + h.keep(d) + } + var approvalAsk func() + if waits == "" { + waits, approvalAsk = h.approval(d, now) + } + if waits != r.Waits { + r.Waits = waits + h.keep(d) + } + if approvalAsk != nil { + asks = append(asks, approvalAsk) + } + if waits != "" || r.merging { + continue + } + if !r.LastTry.IsZero() { + continue // refused: asked again at the next event that decides whether it may merge (mergeAgainNow), with no timer + } + if g := h.groups[d.Group]; d.Group != "" && g != nil { + // A group is merged in one pass, in its order, and stops at the first refusal (issue 1275). + if a := h.groupPass(g, d, now); a != nil { + asks = append(asks, a) + } + continue + } + r.Tries++ + r.LastTry, r.merging = now, true + h.keep(d) + asks = append(asks, h.askMerge(d.ID, r.Heads)) + } + asks = append(h.cancels, asks...) + h.cancels = nil + return asks +} + +// sayAfterLine tells a pull request, once, that its description holds an `after: ` line, which is not read +// (novox/hq ADR 0419 §2): the order of a delivery comes from the module graph alone, and no refusal asks for the line. +// Once per pull request, not per head. Under the lock; the comment is said outside it. +func (h *Holder) sayAfterLine(d *Delivery) func() { + if len(d.After) == 0 || d.AfterSaid { + return nil + } + d.AfterSaid = true + for _, o := range h.deliveries { + if o != d && o.Number == d.Number && o.Repository == d.Repository && o.AfterSaid { + h.keep(d) + return nil // said on this pull request for an earlier head + } + } + h.keep(d) + h.Logf("[mesh-delivery] %s: its description holds an after: line (%s), which is not read", d.ID, strings.Join(d.After, ", ")) + return h.sayOn(d, "mesh-delivery: this description holds an `after:` line ("+strings.Join(d.After, ", ")+"). It is not read: "+ + "the order of a delivery comes from the module graph alone, and nothing asks for the line (novox/hq ADR 0419).") +} + +// askFresh is the ask of a head's check against its trunk's head, when its passing verdict is out of date and either +// arrived so, or has an Approve asked or given for it (novox/hq ADR 0421 §3, issue 1022; issue 1047 for a group's +// member): the Approve stays through a trunk move, and the merge is asked again on the verdict this brings, with no +// second Approve. Nil when nothing needs asking. Under the lock; the ask runs outside it. +func (h *Holder) askFresh(d *Delivery, now time.Time) func() { + if d.State != Proposed || d.outOfDate() == "" || d.AskedAgain != nil || !d.Check.Passes() || d.Base != MergeBranch || + d.Number == 0 || d.TrunkNow == "" { + return nil + } + // A group's member too (issue 1047, design 47: a move of the trunk asks every check again): the group's one Approve is + // asked after its composed check, so a member ready before that has no Approve to carry; its check is asked at once + // all the same, and the composed check follows on the new verdict. The rest of ADR 0395 §1 stays with issue 888. + if !d.FreshWanted && h.covering(d) == nil && h.groups[d.Group] == nil { + return nil + } + if tried, was := h.askTried[d.ID]; was && now.Sub(tried) < time.Minute { + h.lookAt(tried.Add(time.Minute), "a check against the trunk's head that could not be asked") + return nil + } + h.askTried[d.ID] = now + h.lookAt(now.Add(time.Minute), "a check against the trunk's head that could not be asked") + id, m := d.ID, MemberOf(d) + m.BaseSHA = d.TrunkNow + return func() { + asked, err := h.Controller.Check("", []Member{m}) + h.mu.Lock() + defer h.mu.Unlock() + if err != nil { + h.Logf("[mesh-delivery] %s's check against the trunk's head cannot be asked yet: %v", id, err) + return + } + delete(h.askTried, id) + d := h.deliveries[id] + if d == nil || d.State != Proposed || d.outOfDate() == "" { + return + } + t := h.Now() + d.AskedAgain, d.FreshWanted = &t, false + h.Logf("[mesh-delivery] %s: its verdict is out of date; its check against the trunk's head %s asked as %s", id, + shortOf(d.TrunkNow, 12), asked) + h.keep(d) + } +} + +// mergeAgainNow lets an approved delivery's merge be asked at once: something that decides whether it may merge happened (a +// verdict, an Approve, a group's composed check), so the wait after a refusal does not hold it. Under the lock. +func (h *Holder) mergeAgainNow(d *Delivery) { + if d != nil && d.Ready != nil && d.Ready.Merged == nil { + d.Ready.LastTry = time.Time{} + } +} + +// readyWaits is what a delivery's merge waits on; empty when the Approve may be asked and the merge asked. A ready +// delivery is asked. So is a proposed one whose verdict went out of date, when an Approve was asked for it already (it +// stays through a trunk move, ADR 0421 §3) and its check against the trunk's head is not on its way; one without an +// Approve waits for that check first. +func (h *Holder) readyWaits(d *Delivery) string { + switch { + case d.State == Ready: + case d.State == Proposed && d.outOfDate() != "" && d.AskedAgain != nil: + return "its check against the trunk's head is asked and has not answered" + case d.State == Proposed && d.outOfDate() != "" && d.Check.Passes() && h.covering(d) != nil: + case d.State == Proposed && d.outOfDate() != "": + return "its verdict is out of date: its check against the trunk's head is asked before the operator is" + case d.State == Proposed && d.AskedAgain != nil: + return "its check against the trunk's head is asked and has not answered" + default: + return "its check has not passed yet (" + string(d.State) + ")" + } + if d.Group == "" { + return "" + } + return h.groupTurnToMerge(d) +} + +// groupTurnToMerge is why a group member may not merge yet; empty when it may: the group's composed check passed for +// its members' heads as they are and is not out of date. No member waits for another's merge or delivery (novox/hq +// ADR 0418 §4): once the composed check passed and the Approve covers the heads, every member is merged in one pass. +func (h *Holder) groupTurnToMerge(d *Delivery) string { + g := h.groups[d.Group] + if g == nil { + return "" + } + members := h.membersOf(g) + if state, why := GroupState(g, members); state != "ready" && state != "delivering" { + return fmt.Sprintf("its group %s is %s: %s", g.ID, state, why) + } + if c := g.Check; c == nil || (c.Verdict != "pass" && c.Verdict != "warning") || c.OutOfDate != "" || + !sameMembers(c.Commits, commitsOf(members)) { + return "its group " + g.ID + "'s composed check has not passed for its members' heads" + } + return "" +} + +// askMerge asks the forge's merge tool to merge a head the operator approved, and keeps its answer. The tool is handed +// the heads the Approve covers, each `owner/repo@` (novox/hq ADR 0421 §9), and refuses a pull request whose +// head differs from the one approved. +func (h *Holder) askMerge(id string, heads map[string]string) func() { + return func() { h.mergeOne(id, heads) } +} + +// mergeOne asks the merge tool for one approved head and keeps its answer; whether it merged. +func (h *Holder) mergeOne(id string, heads map[string]string) bool { + h.mu.Lock() + d := h.deliveries[id] + if d == nil || d.Ready == nil { + h.mu.Unlock() + return false + } + owner, repo, number, head := d.Owner(), d.Repo(), d.Number, d.Commit + if sha := heads[id]; sha != "" { + head = sha + } + h.mu.Unlock() + err := h.Forge.Merge(owner, repo, number, head, approvedHeads(heads)) + h.mu.Lock() + defer h.mu.Unlock() + d = h.deliveries[id] + if d == nil || d.Ready == nil { + return false + } + r, now := d.Ready, h.Now() + r.merging = false + if err == nil { + // Issue 1025: the announcement is owed, kept with the delivery, so a death right after the merge + // still says it once. + d.Owed = append(d.Owed, Effect{Kind: EffectEmit, Event: "merged", Line: string(mustJSON(mergedEvent(d, head))), Since: now}) + r.Merged, r.Answer, r.RefusedSince = &now, "merged", nil + h.Logf("[mesh-delivery] %s merged on the operator's Approve at %s, its check passed against its trunk's head", id, + shortOf(head, 12)) + h.keep(d) + return true + } + answer := oneLine(err.Error()) + if r.RefusedSince == nil { + r.RefusedSince = &now + } + if !outOfDateRefusal(answer) { + r.Urgent = true // never a warning again for this head + } + if answer != r.Answer { + h.Logf("[mesh-delivery] %s is approved, and the merge tool did not merge it (asked again at the next verdict, Approve or composed check): %s", id, + answer) + } + r.Answer = answer + h.keep(d) + return false +} + +// approvedHeads are the heads an Approve covers as the merge tool is handed them: `owner/repo@`, sorted. +func approvedHeads(heads map[string]string) []string { + var out []string + for id, sha := range heads { + repository, _, _ := strings.Cut(id, "@") + out = append(out, repository+"@"+sha) + } + sort.Strings(out) + return out +} + +// sayOn comments on a delivery's pull request. +func (h *Holder) sayOn(d *Delivery, body string) func() { + owner, repo, number, id := d.Owner(), d.Repo(), d.Number, d.ID + return func() { + if err := h.Forge.Comment(owner, repo, number, body); err != nil { + h.Logf("[mesh-delivery] %s: could not say on its pull request: %v", id, err) + } + } +} + +// readyLines are the approved heads that will not merge: a check that failed at once, a merge refused past its bound; +// as stalled lines, which the controller raises as conditions. Under the lock. +func (h *Holder) readyLines(now time.Time) []StalledLine { + var out []StalledLine + for _, d := range h.deliveries { + r := d.Ready + if r == nil || r.Merged != nil || d.MergedAs != "" || d.State.Final() { + continue + } + switch { + case r.Failed != "" && d.State == Rejected: + out = append(out, StalledLine{ID: d.ID, Number: d.Number, State: d.State, For: now.Sub(r.Seen).Round(time.Second).String(), + Bound: "0s", H2: "none: a new head, or a recheck that passes — the operator's", + Says: fmt.Sprintf("%s#%d passed its checks at %s and is not merged: %s", d.Repository, d.Number, + shortOf(r.Head, 12), r.Failed)}) + case r.Refused != "": + out = append(out, StalledLine{ID: d.ID, Number: d.Number, State: d.State, For: now.Sub(r.Seen).Round(time.Second).String(), + Bound: "0s", H2: "none: a new head, which asks the operator again — the operator's", + Says: fmt.Sprintf("%s#%d at %s is not merged: the operator's answer %s", d.Repository, + d.Number, shortOf(r.Head, 12), r.Refused)}) + case r.RefusedSince != nil && d.Group != "" && h.groupClosed(d.Group) && now.Sub(*r.RefusedSince) >= groupRefusedFor: + // A closed group's member (novox/hq ADR 0418 §3, issue 1082): its sibling merged, and its own merge is refused + // again after each composed check. Said after the merge window, with the group and the tool's last answer. + out = append(out, StalledLine{ID: d.ID, Number: d.Number, State: d.State, + For: now.Sub(*r.RefusedSince).Round(time.Second).String(), Bound: groupRefusedFor.String(), + H2: "none: the group's composed check is asked again after each refusal, and the merge each minute — the operator's", + Severity: refusalSeverity(r, now), + Says: fmt.Sprintf("group %s is half merged: %s#%d is approved at %s and the merge tool has refused it since %s: %s. "+ + "Needed: the refusal's cause is met (the merge is asked again at each verdict, status or Approve), or the operator decides.", + d.Group, d.Repository, d.Number, shortOf(r.Head, 12), r.RefusedSince.UTC().Format(time.RFC3339), r.Answer)}) + case r.RefusedSince != nil && now.Sub(*r.RefusedSince) >= readyRefusedFor: + out = append(out, StalledLine{ID: d.ID, Number: d.Number, State: d.State, + For: now.Sub(*r.RefusedSince).Round(time.Second).String(), Bound: readyRefusedFor.String(), + H2: "none: the merge is asked again on a verdict, an Approve, a group's composed check or a merge check's " + + "status turning success — the operator's", + Says: fmt.Sprintf("%s%s#%d is approved at %s, and the merge tool has refused it since %s: %s%s", heldInGroup(d), + d.Repository, d.Number, shortOf(r.Head, 12), r.RefusedSince.UTC().Format(time.RFC3339), r.Answer, neededOf(d)), + Severity: severityOf(d, now)}) + } + } + return out +} + +// mergedEvent is the module event `merged` (issue 1025): the pull request merged, its head and group, and the ask, +// option and heads the operator's Approve was bound to (the warrant the controller reads back from the router). A merge +// without an Approve names no ask and no option: no warrant is invented. Under the lock, before anything clears them. +func mergedEvent(d *Delivery, head string) map[string]any { + r := d.Ready + heads := map[string]string{} + for k, v := range r.Heads { + heads[k] = v + } + ev := map[string]any{"owner": d.Owner(), "repo": d.Repo(), "number": d.Number, "head_sha": head, "group": d.Group, + "ask_id": "", "option": "", "heads": heads} + if r.Approved != nil { + ev["ask_id"], ev["option"] = r.AskID, OptionApprove + } + return ev +} + +// heldInGroup is the words that put a refused member in its group's pass: none for a lone delivery. +func heldInGroup(d *Delivery) string { + if d.Group == "" { + return "" + } + return "group " + d.Group + " is held, and no member after this one is merged: " +} + +// neededOf is what a refused member needs, for a member of a group (issue 1275). +func neededOf(d *Delivery) string { + if d.Group == "" { + return "" + } + return ". Needed: the refusal's cause is met (the merge is asked again at each verdict, status or Approve), or the operator decides." +} + +// outOfDateFor is how long a refusal for an out of date verdict is a warning before it is urgent (issue 1275, AC6). +const outOfDateFor = 10 * time.Minute + +// outOfDateRefusal is whether the merge tool refused because the member's verdict is judged against an earlier trunk: the +// forge's "its verdict was judged against trunk , and main is now . The check of head merged into trunk was +// asked" (gitea merge.ts), whose fresh check is already running. +// +// Words that also name another cause (a status not a success, missing or failed) are not this: the other cause is urgent +// at once (ADR 0442, issue 1275 AC6). +func outOfDateRefusal(answer string) bool { + for _, other := range []string{"not a success", "(missing)", "(failure)", "(error)", "could not be asked"} { + if strings.Contains(answer, other) { + return false + } + } + // gitea merge.ts: "Its check is already asked and not yet answered" (on its way), or "The check of head merged into + // trunk was asked" (asked now). Its other refusals, among them "its check could not be asked", are not this. + return strings.Contains(answer, "its verdict ") && strings.Contains(answer, " is now ") && + (strings.Contains(answer, "Its check is already asked and not yet answered") || + strings.Contains(answer, "merged into trunk") && strings.Contains(answer, "was asked")) +} + +// refusalSeverity is "warning" while a member's refusal for an out of date verdict is younger than ten minutes, as its +// fresh check runs, and "urgent" for it after that and for every other refusal at once. +func refusalSeverity(r *ReadyRun, now time.Time) string { + if !r.Urgent && r.RefusedSince != nil && outOfDateRefusal(r.Answer) && now.Sub(*r.RefusedSince) < outOfDateFor { + return "warning" + } + return "urgent" +} + +// severityOf is the severity of a refused member of a group, empty for a lone delivery. +func severityOf(d *Delivery, now time.Time) string { + if d.Group == "" { + return "" + } + return refusalSeverity(d.Ready, now) +} diff --git a/modules/mesh-delivery/cmd/mesh-delivery/ready_test.go b/modules/mesh-delivery/cmd/mesh-delivery/ready_test.go new file mode 100644 index 00000000..0f09e359 --- /dev/null +++ b/modules/mesh-delivery/cmd/mesh-delivery/ready_test.go @@ -0,0 +1,239 @@ +package main + +import ( + "errors" + "sort" + "strings" + "testing" + "time" +) + +// **A pull request whose checks passed and whose merge the operator approved is merged by this owner** (novox/hq issues +// 722 and 1022): before, a reviewed pull request with green checks waited until a session called the merge tool, and +// again after every check the tool asked because the trunk had moved. + +const ( + readyHead = "1111111111111111111111111111111111111111" + pushed = "2222222222222222222222222222222222222222" +) + +// openPull is an open pull request into main as gitea_open_pulls reads it. +func openPull(repo string, number int, head string) OpenPull { + owner, name, _ := strings.Cut(repo, "/") + return OpenPull{Owner: owner, Repo: name, Number: number, Base: "main", HeadSHA: head} +} + +func judgedAt(repo string, number int, sha, gate, trunk string) CheckedEvent { + c := verdict(repo, number, sha, gate) + c.Trunk = trunk + return c +} + +func TestAnApprovedPullRequestIsMergedThroughTheMergeTool(t *testing.T) { + w := newWorld(t) + w.op.auto = false + id := IDOf("novox/app", readyHead) + w.h.PullUpdated(pr("novox/app", 7, readyHead, "feat/x", "src/a.go")) + w.h.Checked(judgedAt("novox/app", 7, readyHead, "pass", "t1t1t1t1t1t1")) + w.settleAll() + if len(w.forge.merges) != 0 { + t.Fatalf("a pull request nobody approved was merged: %v", w.forge.merges) + } + w.op.answer(w.op.open()[0], OptionApprove) + w.settleAll() + if len(w.forge.merges) != 1 || w.forge.merges[0] != "novox/app#7@"+readyHead { + t.Fatalf("an approved pull request was merged %v; wanted once, at the approved head", w.forge.merges) + } + d := w.delivery(id) + if d.Ready == nil || d.Ready.Merged == nil || d.Ready.Tries != 1 { + t.Fatalf("what was done with the Approve is not kept: %+v", d.Ready) + } + // Its merge's transition names the Approve as its warrant. + w.h.PullMerged(merged("novox/app", 7, readyHead, "m7m7m7m7m7m7")) + w.h.WalkMoved(w.ctl.put(aWalk("plan-7", "novox/app", "m7m7m7m7m7m7", true))) + w.settleAll() + d = w.delivery(id) + var last Transition + for _, tr := range d.Transitions { + if tr.Event == EvMerged { + last = tr + } + } + if !strings.Contains(last.By, "the operator's Approve at "+readyHead[:12]) { + t.Fatalf("the merge's transition does not name the Approve: %+v", last) + } + if len(w.forge.merges) != 1 { + t.Fatalf("merged again: %v", w.forge.merges) + } +} + +// 1144/AC3: a refused merge is not asked again by any timer; only an event (here, mergeAgainNow called directly, as a +// verdict, an Approve or a composed check does) asks it again. +func TestAMergeRefusedIsSaidAtOnceAndAskedAgainOnlyOnAnEvent(t *testing.T) { + w := newWorld(t) + w.h.PullUpdated(pr("novox/app", 7, readyHead, "feat/x", "src/a.go")) + w.h.Checked(judgedAt("novox/app", 7, readyHead, "pass", "t1t1t1t1t1t1")) + w.forge.refuse = errors.New("the forge refused gitea_merge_pull_request: mesh/repo-check is pending") + w.forge.open = []OpenPull{openPull("novox/app", 7, readyHead)} + w.h.SeeOpenPulls() + w.settleAll() + if lines := w.h.Stalled(); len(lines) != 1 || !strings.Contains(lines[0].Says, "refused it since") { + t.Fatalf("a merge refused is not said at once: %+v", lines) + } + w.later(time.Hour) + w.settleAll() + if len(w.forge.merges) != 1 { + t.Fatalf("asked again with no event: %v", w.forge.merges) + } + again := w.delivery(IDOf("novox/app", readyHead)) + w.h.mu.Lock() + w.h.mergeAgainNow(again) // what a verdict, an Approve or a composed check does + w.h.mu.Unlock() + w.settleAll() + if len(w.forge.merges) != 2 { + t.Fatalf("not asked again on an event: %v", w.forge.merges) + } +} + +// 1144/AC1: a refused merge, approved and ready, is asked again at once when the forge's forge.status says one of +// the merge checks (mesh/merge-gate, mesh/repo-check) is success for the commit the merge was refused at — the +// delivery's approved head — and merges once the tool accepts (novox/hq mesh-catalog#366: the verdict reached this +// owner before the forge wrote the statuses, the merge asked then was refused on a pending check, and nothing asked +// it again when the statuses turned success). +func TestAMergeRefusedIsAskedAgainWhenItsCheckTurnsSuccessOnTheForge(t *testing.T) { + w := newWorld(t) + w.h.PullUpdated(pr("novox/app", 7, readyHead, "feat/x", "src/a.go")) + w.h.Checked(judgedAt("novox/app", 7, readyHead, "pass", "t1t1t1t1t1t1")) + w.forge.refuse = errors.New("the forge refused gitea_merge_pull_request: mesh/repo-check (pending)") + w.forge.open = []OpenPull{openPull("novox/app", 7, readyHead)} + w.h.SeeOpenPulls() + w.settleAll() + if len(w.forge.merges) != 1 { + t.Fatalf("not asked once, refused: %v", w.forge.merges) + } + w.forge.refuse = nil + w.h.CommitStatus(ForgeStatusEvent{Owner: "novox", Repo: "app", SHA: readyHead, Context: "mesh/repo-check", State: "success"}) + w.settleAll() + if len(w.forge.merges) != 2 { + t.Fatalf("not asked again when its check turned success: %v", w.forge.merges) + } + if w.delivery(IDOf("novox/app", readyHead)).Ready.Merged == nil { + t.Fatalf("it did not merge once the tool accepted") + } +} + +// 1144/AC2: a status of another context, of another commit, or a state other than success changes nothing for a +// refused merge: it is not asked again. +func TestAMergeRefusedIsNotAskedAgainOnAStatusThatDoesNotMatch(t *testing.T) { + w := newWorld(t) + w.h.PullUpdated(pr("novox/app", 7, readyHead, "feat/x", "src/a.go")) + w.h.Checked(judgedAt("novox/app", 7, readyHead, "pass", "t1t1t1t1t1t1")) + w.forge.refuse = errors.New("the forge refused gitea_merge_pull_request: mesh/repo-check (pending)") + w.forge.open = []OpenPull{openPull("novox/app", 7, readyHead)} + w.h.SeeOpenPulls() + w.settleAll() + if len(w.forge.merges) != 1 { + t.Fatalf("not asked once, refused: %v", w.forge.merges) + } + w.forge.refuse = nil + for _, e := range []ForgeStatusEvent{ + {Owner: "novox", Repo: "app", SHA: readyHead, Context: "some/other-check", State: "success"}, // another context + {Owner: "novox", Repo: "app", SHA: pushed, Context: "mesh/repo-check", State: "success"}, // another commit + {Owner: "novox", Repo: "app", SHA: readyHead, Context: "mesh/repo-check", State: "pending"}, // another state + } { + w.h.CommitStatus(e) + w.settleAll() + } + if len(w.forge.merges) != 1 { + t.Fatalf("asked again on a status that should change nothing: %v", w.forge.merges) + } + if w.delivery(IDOf("novox/app", readyHead)).Ready.Merged != nil { + t.Fatalf("merged on a status that should change nothing") + } +} + +// 1144/AC4: the stalled line for a refused approved head names the events that ask the merge again, and no longer +// says "each minute" (removed by issue 1115; false since then — there is no timer). +func TestTheStalledLineForARefusedApprovedHeadNamesTheEventsThatAskTheMergeAgain(t *testing.T) { + w := newWorld(t) + w.h.PullUpdated(pr("novox/app", 7, readyHead, "feat/x", "src/a.go")) + w.h.Checked(judgedAt("novox/app", 7, readyHead, "pass", "t1t1t1t1t1t1")) + w.forge.refuse = errors.New("the forge refused gitea_merge_pull_request: mesh/repo-check (pending)") + w.forge.open = []OpenPull{openPull("novox/app", 7, readyHead)} + w.h.SeeOpenPulls() + w.settleAll() + lines := w.h.Stalled() + if len(lines) != 1 { + t.Fatalf("wanted one stalled line, got %+v", lines) + } + h2 := lines[0].H2 + if strings.Contains(h2, "each minute") { + t.Fatalf("the stalled line still says 'each minute', false since issue 1115: %q", h2) + } + for _, want := range []string{"verdict", "Approve", "composed check", "status"} { + if !strings.Contains(h2, want) { + t.Fatalf("the stalled line does not name %q as asking the merge again: %q", want, h2) + } + } +} + +func TestAGroupMergesEveryMemberInOnePassOnlyOnceComposedAndApproved(t *testing.T) { + w := newWorld(t) + w.op.auto = false + const ctlHead, catHead = "c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1", "d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2" + w.h.PullUpdated(pr("novox/mesh-controller", 1, ctlHead, "feat/x", "internal/a.go")) + w.h.PullUpdated(pr("novox/mesh-catalog", 2, catHead, "feat/x", "modules/app/module.json")) + w.h.Checked(judgedAt("novox/mesh-controller", 1, ctlHead, "pass", "t1t1t1t1t1t1")) + w.h.Checked(judgedAt("novox/mesh-catalog", 2, catHead, "pass", "t3t3t3t3t3t3")) + w.settleAll() + if len(w.forge.merges) != 0 || w.op.count() != 0 { + t.Fatalf("a member was merged or asked before its group's composed check passed: %v, %d", w.forge.merges, w.op.count()) + } + w.h.Checked(CheckedEvent{Group: "feat/x", ID: "check-1", Verdict: "pass", Summary: "both compose"}) + w.settleAll() + if w.op.count() != 1 || len(w.forge.merges) != 0 { + t.Fatalf("composed, the group is asked once and nothing merges before the Approve: asks %d, merges %v", w.op.count(), w.forge.merges) + } + w.op.answer(w.op.open()[0], OptionApprove) + w.settleAll() + if len(w.forge.merges) != 2 { + t.Fatalf("both members are merged in the same pass on the one Approve: %v", w.forge.merges) + } + sort.Strings(w.forge.merges) + if w.forge.merges[0] != "novox/mesh-catalog#2@"+catHead || w.forge.merges[1] != "novox/mesh-controller#1@"+ctlHead { + t.Fatalf("merged %v, wanted both, each at its approved head", w.forge.merges) + } + for _, got := range w.forge.approvedHeads { + if len(got) != 2 || got[0] != "novox/mesh-catalog@"+catHead || got[1] != "novox/mesh-controller@"+ctlHead { + t.Fatalf("a member's merge was handed the approved heads %v", got) + } + } + if w.op.count() != 1 { + t.Fatalf("a second Approve was asked: %d", w.op.count()) + } +} + +// **A merge into another branch than main is its author's** (novox/hq ADR 0398 §4, issue 948): a green pull request +// whose base is not main is never asked or merged by this owner; it says so once on the pull request. +func TestAGreenPullRequestIntoAnotherBranchIsNotAskedOrMergedAndSaidOnce(t *testing.T) { + w := newWorld(t) + p := pr("novox/app", 7, readyHead, "feat/x", "src/a.go") + p.Base = "release/x" + w.h.PullUpdated(p) + w.h.Checked(judgedAt("novox/app", 7, readyHead, "pass", "t1t1t1t1t1t1")) + open := openPull("novox/app", 7, readyHead) + open.Base = "release/x" + w.forge.open = []OpenPull{open} + for range 3 { + w.h.SeeOpenPulls() + w.settleAll() + w.later(2 * time.Minute) + } + if len(w.forge.merges) != 0 || w.op.count() != 0 { + t.Fatalf("a pull request into release/x was merged %v or asked %d", w.forge.merges, w.op.count()) + } + if len(w.forge.comments) != 1 || !strings.Contains(w.forge.comments[0], "another branch") || + !strings.Contains(w.forge.comments[0], "0398") { + t.Fatalf("not said exactly once on the pull request: %v", w.forge.comments) + } +} diff --git a/modules/mesh-delivery/cmd/mesh-delivery/recheck_test.go b/modules/mesh-delivery/cmd/mesh-delivery/recheck_test.go new file mode 100644 index 00000000..97c843c8 --- /dev/null +++ b/modules/mesh-delivery/cmd/mesh-delivery/recheck_test.go @@ -0,0 +1,175 @@ +package main + +import ( + "strings" + "testing" + "time" +) + +// A recheck puts the merge check's statuses back to pending before it asks the check: the forge keeps the +// newest status of each context on the head, and a recheck asks of that same head, so the old verdict would +// stand — and branch protection would let the pull request merge on it — until the fresh one lands. + +// heldGreen is a head whose merge check passed, as the forge's holder set it from the verdict. +func heldGreen(t *testing.T, w *world, repo string, number int) string { + t.Helper() + w.h.PullUpdated(pr(repo, number, head, "feat/x", "src/a.go")) + owner, name, _ := strings.Cut(repo, "/") + _ = w.forge.Status(owner, name, head, ctxGate, "success", "pass: the gate said pass", "") + _ = w.forge.Status(owner, name, head, ctxRepo, "success", "pass: merge-check.sh passed", "") + w.forge.required = []string{ctxGate, ctxRepo} + w.h.Checked(verdict(repo, number, head, "pass")) + w.settleAll() + id := IDOf(repo, head) + if s := w.state(id); s != Ready { + t.Fatalf("a passing verdict left it %s", s) + } + return id +} + +func TestARecheckPutsTheMergeCheckBackToPendingBeforeItAsks(t *testing.T) { + w := newWorld(t) + id := heldGreen(t, w, "novox/app", 7) + at := "novox/app@" + head + " " + var whenAsked map[string]string + w.ctl.onCheck = func() { + w.forge.mu.Lock() + defer w.forge.mu.Unlock() + whenAsked = map[string]string{ctxGate: w.forge.statuses[at+ctxGate], ctxRepo: w.forge.statuses[at+ctxRepo]} + } + if _, err := w.h.Recheck(id, "main moved on", "a person"); err != nil { + t.Fatal(err) + } + if len(w.ctl.checks) != 1 { + t.Fatalf("its check was asked %v", w.ctl.checks) + } + for _, c := range []string{ctxGate, ctxRepo} { + if !strings.HasPrefix(whenAsked[c], "pending checking again: main moved on") { + t.Fatalf("as its check was asked, the forge held %s %q: the old verdict still lets it merge", c, whenAsked[c]) + } + } + checks, err := w.forge.Statuses("novox", "app", 0, head, "main") + if err != nil { + t.Fatal(err) + } + if checks.Merge.Mergeable == nil || *checks.Merge.Mergeable { + t.Fatalf("rechecked, the forge would still merge it: %+v", checks.Merge) + } + if s := w.state(id); s != Proposed { + t.Fatalf("rechecked, it is %s", s) + } +} + +// Only the statuses the head holds are put back: a verdict that says no repository check would leave a +// pending one standing for ever. +func TestARecheckLeavesAStatusTheHeadNeverHadAlone(t *testing.T) { + w := newWorld(t) + w.h.PullUpdated(pr("novox/app", 8, head, "feat/y", "src/a.go")) + _ = w.forge.Status("novox", "app", head, ctxGate, "failure", "fail: the gate said fail", "") + w.h.Checked(verdict("novox/app", 8, head, "fail")) + w.settleAll() + id := IDOf("novox/app", head) + if _, err := w.h.Recheck(id, "flaky", "a person"); err != nil { + t.Fatal(err) + } + if s := w.forge.statuses["novox/app@"+head+" "+ctxGate]; !strings.HasPrefix(s, "pending checking again: flaky") { + t.Fatalf("the gate is %q", s) + } + if s, set := w.forge.statuses["novox/app@"+head+" "+ctxRepo]; set { + t.Fatalf("a repository check the head never had was set: %q", s) + } +} + +// A forge that cannot be told refuses the recheck whole: nothing is asked on a stale verdict, and the +// delivery stays where it stood, for the person to ask again. +func TestARecheckTheForgeCannotTakeIsRefused(t *testing.T) { + w := newWorld(t) + id := heldGreen(t, w, "novox/app", 9) + w.forge.down = true + if _, err := w.h.Recheck(id, "main moved on", "a person"); err == nil { + t.Fatal("a recheck whose statuses could not be put back was taken") + } + if len(w.ctl.checks) != 0 { + t.Fatalf("its check was asked on a stale verdict: %v", w.ctl.checks) + } + if s := w.state(id); s != Ready { + t.Fatalf("refused, it moved to %s", s) + } +} + +// novox/hq issue 867, ADR 0323 §6: "the composed check runs again when a member is rechecked". A group whose composed +// check failed stayed rejected after a member was rechecked and passed, because the kept verdict was for the same heads +// and nothing marked it out of date. +func TestAMembersRecheckAsksTheGroupsComposedCheckAgain(t *testing.T) { + w := newWorld(t) + w.h.PullUpdated(pr("novox/a", 1, "a1a1a1a1a1a1a1", "feat/z")) + w.h.PullUpdated(pr("novox/b", 2, "b2b2b2b2b2b2b2", "feat/z")) + w.h.Checked(verdict("novox/a", 1, "a1a1a1a1a1a1a1", "pass")) + w.h.Checked(verdict("novox/b", 2, "b2b2b2b2b2b2b2", "pass")) + w.settleAll() + groupAsks := func() (n int) { + for _, c := range w.ctl.checks { + if strings.HasPrefix(c, "feat/z:") { + n++ + } + } + return + } + if groupAsks() != 1 { + t.Fatalf("the group was asked %v", w.ctl.checks) + } + w.h.Checked(CheckedEvent{Group: "feat/z", ID: "check-1", Verdict: "fail", Summary: "they do not compose"}) + w.settleAll() + if state, _ := GroupState(w.h.groups["feat/z"], w.h.membersOf(w.h.groups["feat/z"])); state != "rejected" { + t.Fatalf("the group is %s", state) + } + if _, err := w.h.Recheck(IDOf("novox/b", "b2b2b2b2b2b2b2"), "the judge was fixed", "a person"); err != nil { + t.Fatal(err) + } + // The member's own check passes again, at the same head, a while later. + w.later(5 * time.Minute) + w.h.Checked(verdict("novox/b", 2, "b2b2b2b2b2b2b2", "pass")) + w.settleAll() + if groupAsks() != 2 { + t.Fatalf("a member's recheck did not ask the group again: %v", w.ctl.checks) + } + if g := w.h.groups["feat/z"]; g.Check == nil || g.Check.Verdict != "" || g.Check.Asked == "check-1" { + t.Fatalf("the old verdict still stands: %+v", g.Check) + } +} + +// A recheck made while the group's composed check is still running is not lost: the old run's answer lands, and the +// group is still asked again once the member is ready. +func TestARecheckWhileTheComposedCheckRunsAsksTheGroupAgain(t *testing.T) { + w := newWorld(t) + w.h.PullUpdated(pr("novox/a", 1, "a1a1a1a1a1a1a1", "feat/z")) + w.h.PullUpdated(pr("novox/b", 2, "b2b2b2b2b2b2b2", "feat/z")) + w.h.Checked(verdict("novox/a", 1, "a1a1a1a1a1a1a1", "pass")) + w.h.Checked(verdict("novox/b", 2, "b2b2b2b2b2b2b2", "pass")) + w.settleAll() // the composed check check-1 is asked and running: no verdict yet + if _, err := w.h.Recheck(IDOf("novox/b", "b2b2b2b2b2b2b2"), "the judge was fixed", "a person"); err != nil { + t.Fatal(err) + } + // The old run answers. + w.h.Checked(CheckedEvent{Group: "feat/z", ID: "check-1", Verdict: "fail", Summary: "they do not compose"}) + w.later(5 * time.Minute) + w.h.Checked(verdict("novox/b", 2, "b2b2b2b2b2b2b2", "pass")) + w.settleAll() + n := 0 + for _, c := range w.ctl.checks { + if strings.HasPrefix(c, "feat/z:") { + n++ + } + } + if n != 2 { + t.Fatalf("the group was not asked again after a recheck made while its check ran: %v", w.ctl.checks) + } + if g := w.h.groups["feat/z"]; g.Check == nil || g.Check.Verdict != "" || g.Check.OutOfDate != "" { + t.Fatalf("the old run's answer stands: %+v", g.Check) + } + // The old run answering again is history. + w.h.Checked(CheckedEvent{Group: "feat/z", ID: "check-1", Verdict: "fail", Summary: "late"}) + if g := w.h.groups["feat/z"]; g.Check.Verdict != "" { + t.Fatalf("a late answer of the old run was taken: %+v", g.Check) + } +} diff --git a/modules/mesh-delivery/cmd/mesh-delivery/replay362_test.go b/modules/mesh-delivery/cmd/mesh-delivery/replay362_test.go new file mode 100644 index 00000000..7e83f79e --- /dev/null +++ b/modules/mesh-delivery/cmd/mesh-delivery/replay362_test.go @@ -0,0 +1,127 @@ +package main + +import ( + "strings" + "testing" + "time" +) + +// Replay R362 (novox/hq issue 362, mesh-lab's register): two merges of the catalogue 18 seconds apart, walked by +// one walk at the later merge's commit, which superseded the first — as the controller of 2026-10-10 said it, with +// no `merges` and the successor named only in the superseded walk's note. Written with only what mesh-delivery +// had before its fix (and fakes_test.go, covered_test.go's helpers), so it can be laid over the commit before. + +const ( + cat = "novox/mesh-catalog" + headCC = "cccccccccccc1111" // claude-code's head + headDu = "dddddddddddd2222" // dunst's head + mergeCC = "553b71910000aaaa" // merged first, 00:20:52 + mergeDu = "48bda4750000bbbb" // merged 18 s later, 00:21:09: contains mergeCC +) + +var ( + atCC = time.Date(2026, 10, 9, 22, 20, 52, 0, time.UTC) + atDu = atCC.Add(17 * time.Second) +) + +// twoMerges is the night of issue 362 up to the walks: both pull requests checked, both merged. +func twoMerges(t *testing.T) *world { + w := newWorldOver(t, newMemStore(), newFakeController(), newFakeForge(), atDu.Add(5*time.Second)) + cc := pr(cat, 101, headCC, "feat/claude-code", "modules/claude-code/module.json") + du := pr(cat, 102, headDu, "feat/dunst", "modules/dunst/module.json") + w.h.PullUpdated(cc) + w.h.PullUpdated(du) + w.h.Checked(verdictMoving(cat, 101, headCC, "claude-code")) + w.h.Checked(verdictMoving(cat, 102, headDu, "dunst")) + w.h.PullMerged(mergedAt(cc, mergeCC, atCC)) + w.h.PullMerged(mergedAt(du, mergeDu, atDu)) + return w +} + +// catWalk is a walk of the catalogue at a commit, waiting for its word, moving the modules given. +func catWalk(id, commit string, created time.Time, modules ...string) Walk { + w := Walk{ID: id, Repository: cat, Branch: "main", Commit: commit, Created: created, State: walkBuilding, + Tiers: [][]string{modules}, Modules: map[string]*WalkModule{}, Delivery: &WalkDelivery{Awaits: "mesh-delivery"}} + for _, m := range modules { + w.Modules[m] = &WalkModule{} + } + return w +} + +// done is a walk's record once every module passed on its first machine and was sent to the rest. +func done(w Walk, at time.Time) Walk { + w.State = walkDone + for name := range w.Modules { + judged := at + w.Modules[name] = &WalkModule{State: "built", First: []string{"anchor"}, FirstAt: &judged, SentAt: &judged, + Gate: &WalkGate{Machines: []string{"anchor"}, Verdict: "passed", JudgedAt: &judged}} + } + return w +} + +func supersededBy(old Walk, by string) Walk { + old.State = walkSuperseded + old.Note = "superseded at tier 0 by " + by + " (novox/mesh-catalog at 48bda475); dunst planned there again" + return old +} + +// The replay of issue 362, as today's controller says it: no `merges`, the successor named only in its note. In +// both orders the controller's two records can arrive in, the walk at the dunst commit delivers both merges +// without a person: claude-code covered, dunst delivered, the word asked once. +func TestReplay362(t *testing.T) { + for _, order := range []string{"newer walk first", "supersession first"} { + t.Run(order, func(t *testing.T) { + w := twoMerges(t) + w.forge.contains[mergeCC+" in "+mergeDu] = true + idCC, idDu := IDOf(cat, headCC), IDOf(cat, headDu) + first := w.ctl.put(catWalk("plan-1", mergeDu, atDu.Add(5*time.Second), "dunst")) + w.h.WalkMoved(first) + if s := w.state(idDu); s != Published { + t.Fatalf("dunst with its walk opened is %s", s) + } + second := catWalk("plan-2", mergeDu, atDu.Add(6*time.Second), "claude-code", "dunst") + if order == "newer walk first" { + w.h.WalkMoved(w.ctl.put(second)) + w.h.WalkMoved(w.ctl.put(supersededBy(first, "plan-2"))) + } else { + w.ctl.put(second) // kept by the controller, said after + w.h.WalkMoved(w.ctl.put(supersededBy(first, "plan-2"))) + } + w.settleAll() + for _, id := range []string{idCC, idDu} { + if d := w.delivery(id); d.State.Final() || d.State == Held { + t.Fatalf("%s is %s (%s)", id, d.State, d.HeldWhy) + } + } + if got := w.ctl.delivers; len(got) != 1 || got[0] != "plan-2" { + t.Fatalf("the word was asked %v, want once for plan-2", got) + } + for _, id := range []string{idCC, idDu} { + if d := w.delivery(id); d.State != Delivering || d.Walk.ID != "plan-2" { + t.Fatalf("%s is %s on %v", id, d.State, d.Walk) + } + } + // Ten minutes and a reconcile later nothing is held for a person. + w.later(11 * time.Minute) + if err := w.h.Reconcile(); err != nil { + t.Fatal(err) + } + w.settleAll() + w.h.WalkMoved(w.ctl.put(done(w.ctl.walk("plan-2"), w.now))) + w.settleAll() + cc, du := w.delivery(idCC), w.delivery(idDu) + if du.State != Delivered { + t.Fatalf("dunst is %s", du.State) + } + if cc.State != Covered || !strings.Contains(cc.Transitions[len(cc.Transitions)-1].Why, idDu) { + t.Fatalf("claude-code is %s: %+v", cc.State, cc.Transitions[len(cc.Transitions)-1]) + } + if len(w.h.Stalled()) != 0 { + t.Fatalf("stalled lists %+v", w.h.Stalled()) + } + if len(w.ctl.delivers) != 1 { + t.Fatalf("the word was asked %v", w.ctl.delivers) + } + }) + } +} diff --git a/modules/mesh-delivery/cmd/mesh-delivery/table.go b/modules/mesh-delivery/cmd/mesh-delivery/table.go new file mode 100644 index 00000000..b8ed6d08 --- /dev/null +++ b/modules/mesh-delivery/cmd/mesh-delivery/table.go @@ -0,0 +1,735 @@ +package main + +import ( + "errors" + "fmt" + "strings" + "time" +) + +// The state table (novox/hq ADR 0239 decision 2, to-be 47): every transition a delivery may take, each with +// its guard, compiled once. The code moves a delivery only through Apply, which finds the row for the +// delivery's state and the event, runs its guard and refuses anything else, naming what was asked. A test +// walks the table: every row it holds, and every pair it does not. + +// State is where a delivery is. +type State string + +// The states, in the order a delivery passes through them. None is a delivery not yet made: the rows from +// it are how one comes to exist. +const ( + None State = "" + Proposed State = "proposed" + Checked State = "checked" + Ready State = "ready" + Rejected State = "rejected" + Published State = "published" + Delivering State = "delivering" + Held State = "held" + Delivered State = "delivered" + Failed State = "failed" + Superseded State = "superseded" + Stopped State = "stopped" + // Retired is history: a merge this owner heard only from the forge's word, long after it happened — + // replayed from the bus when it first held the seat. It reached the machines before this owner existed, + // and nothing is delivered, released or put back for it (novox/hq issue 313). + Retired State = "retired" + // Covered is a merge a later one that contains it delivered: every module it moves was sent to every machine + // running it by the finished walk of a later merge into the same trunk, whose own check passed and which the + // forge says contains this one. It is not walked itself, and it is no failure: its pull request says covered, + // as a success (novox/hq ADR 0270). A revert merged in between is still possible: covered says what was + // delivered, not what runs. + Covered State = "covered" +) + +// AllStates are every state a delivery can be in, for the table's own test and its verb. +var AllStates = []State{Proposed, Checked, Ready, Rejected, Published, Delivering, Held, Delivered, Failed, + Superseded, Stopped, Retired, Covered} + +// Final is whether nothing follows a state. +func (s State) Final() bool { + return s == Delivered || s == Failed || s == Superseded || s == Stopped || s == Retired || s == Covered +} + +// Reached is whether a final state says the change reached the machines, or had nothing to reach them +// with: what a group counts as delivered. +func (s State) Reached() bool { return s == Delivered || s == Retired || s == Covered } + +// Event is what moves a delivery. +type Event string + +// The events. Observed ones are taken by settle whenever their guard holds; the others are acts — a +// person's, or healer H2's — taken only when asked. +const ( + EvAnnounced Event = "announced" // the forge announced a pull request's head + EvAppeared Event = "appeared" // a trunk commit the mesh is delivering, with no pull request known + EvAdopted Event = "adopted" // a walk already running at the switch, or on the controller's own path + EvChecked Event = "checked" // its check's verdict arrived + EvAccepted Event = "accepted" // the verdict passes + EvRefused Event = "refused" // the verdict fails, or the check could not run + EvRecheck Event = "recheck" // a person asked for it again, or its group changed + EvReask Event = "re-ask" // healer H2 asked a check asked and not answered once more + EvNewHead Event = "new-head" // a newer head of the same pull request + EvClosed Event = "closed" // the pull request closed unmerged + EvMerged Event = "merged" // it reached the trunk + EvMergedUnchecked Event = "merged-unchecked" // it reached the trunk without a passing check + EvGo Event = "go" // its walk started + EvHold Event = "hold" // something holds it for a person + EvRelease Event = "release" // a person's word that it goes on + EvDone Event = "done" // every machine of its deploy plan took it, or nothing was to be sent + EvFailed Event = "failed" // its walk failed: a gate, a build, a machine + EvSuperseded Event = "superseded" // a newer delivery to the same trunk took over its walk + EvStop Event = "stop" // a person stopped it, or its group did + EvHistory Event = "history" // made from the forge's word for a merge long past: history + EvRetire Event = "retire" // a person's word that a held delivery is history, never to be walked + EvCovered Event = "covered" // a later merge into the same trunk delivered every module it moves + EvLifted Event = "lifted" // what held it for a person is no longer so: its group's composed check passed, or a walk now answers its merge + EvRewalk Event = "rewalk" // the walk that carried a later commit for it failed, or was stopped for another merge (ADR 0299): it waits for the walk that walks it again + EvBackupFailed Event = "backup failed" // a backup before the send failed or stayed silent: nothing of its plan was sent (novox/hq ADR 0404) + EvTrunkMoved Event = "trunk moved" // another merge moved its trunk, and its verdict names the trunk commit before (novox/hq ADR 0323) +) + +// Facts are what a guard reads beyond the delivery itself: the moment, and who asks, for an act. +type Facts struct { + Now time.Time + By string + Why string +} + +// Row is one transition the table holds. +type Row struct { + From []State + Event Event + To State + // Guard says, in words, what must be true; Holds checks it, answering why not. + Guard string + Holds func(d *Delivery, f Facts) error + // Act says the row is taken only when asked — a person's or H2's — never by settle. + Act bool + // Says is the transition's why when nobody gave one: the fact that moved it, rather than the guard. + Says func(d *Delivery) string +} + +func unless(cond bool, why string) error { + if cond { + return nil + } + return fmt.Errorf("%s", why) +} + +var notFinal = []State{Proposed, Checked, Ready, Rejected, Published, Delivering, Held} + +// Table is every transition there is. The order within one state is the order settle tries them. +var Table = []Row{ + {From: []State{None}, Event: EvAnnounced, To: Proposed, + Guard: "a pull request's head the forge announced", + Holds: func(d *Delivery, f Facts) error { + return unless(d.Number > 0 && d.Commit != "", "no pull request's head") + }}, + {From: []State{None}, Event: EvAppeared, To: Held, + Guard: "a commit on the trunk, its walk waiting, and no pull request known: it waits for a person", + Holds: func(d *Delivery, f Facts) error { + return unless(d.MergedAs != "" && d.HeldWhy != "", "no trunk commit, or nothing says why it is held") + }}, + {From: []State{None}, Event: EvAdopted, To: Delivering, + Guard: "a walk already running: at the switch, or on the controller's own path", + Holds: func(d *Delivery, f Facts) error { + return unless(d.Walk != nil && d.Walk.Started(), "no running walk") + }}, + + // History is taken before anything else of a delivery off the trunk: a merge long past, heard only from + // the forge's word, is neither checked, held nor walked (novox/hq issue 313). + {From: []State{Proposed, Checked, Ready, Rejected}, Event: EvHistory, To: Retired, + Guard: "history: made from the forge's word for a merge on the trunk heard " + historyLate.String() + " or more " + + "after it happened — replayed when this owner first held the seat — and no walk of it open", + Holds: func(d *Delivery, f Facts) error { + if d.History == "" { + return errors.New("nothing says it is history") + } + return isHistory(d) + }}, + {From: []State{Held}, Event: EvRetire, To: Retired, Act: true, + Guard: "a person's word, with why, for history only: made from the forge's word for a merge on the trunk heard " + + historyLate.String() + " or more after it happened, and no walk of it open — never released or delivered", + Holds: func(d *Delivery, f Facts) error { + if f.Why == "" || f.By == "" { + return errors.New("a retire says who and why") + } + return isHistory(d) + }}, + + {From: []State{Proposed, Checked, Ready, Rejected}, Event: EvNewHead, To: Superseded, + Guard: "a newer head of the same pull request", + Holds: func(d *Delivery, f Facts) error { return unless(d.NewerHead != "", "no newer head") }}, + {From: []State{Proposed, Checked, Ready, Rejected}, Event: EvClosed, To: Stopped, + Guard: "the pull request closed unmerged", + Holds: func(d *Delivery, f Facts) error { return unless(d.ClosedUnmerged, "the pull request is open") }}, + {From: []State{Ready}, Event: EvMerged, To: Published, + Guard: "on the trunk its modules follow: merged there, and its walk opened — or nothing for a walk to move", + Holds: func(d *Delivery, f Facts) error { + return unless(d.MergedAs != "" && (d.Walk != nil || d.MovesNothing()), "not on the trunk yet") + }}, + // **A verdict counts only for the trunk's head** (novox/hq ADR 0323 §7): one naming a trunk commit that is no + // longer the trunk's head is kept and shown, and does not count, so the delivery stays proposed and the rows + // cannot loop with `trunk moved`; a recheck or the merge tool's ask judges it again. + {From: []State{Proposed}, Event: EvChecked, To: Checked, + Guard: "a verdict names this commit and the trunk's head as its trunk commit", + Holds: func(d *Delivery, f Facts) error { + if d.Check == nil { + return errors.New("no verdict") + } + return unless(d.outOfDate() == "", "its verdict is "+d.outOfDate()) + }}, + {From: []State{Checked}, Event: EvAccepted, To: Ready, + Guard: "the gate passed or warned, and the repository's own check did not fail", + Holds: func(d *Delivery, f Facts) error { return unless(d.Check.Passes(), "the verdict does not pass") }}, + {From: []State{Checked}, Event: EvRefused, To: Rejected, + Guard: "the gate failed or could not run, or the repository's own check failed", + Holds: func(d *Delivery, f Facts) error { return unless(!d.Check.Passes(), "the verdict passes") }}, + {From: []State{Proposed, Checked, Rejected, Held}, Event: EvDone, To: Delivered, + Guard: "merged, and its plan builds nothing for a walk to move — nothing at all, or only a new module the " + + "controller builds and sends nowhere: nothing to deliver and nothing for a person to decide, whatever its check said", + Holds: func(d *Delivery, f Facts) error { + return unless(d.MergedAs != "" && d.MovesNothing() && d.Walk == nil, "not merged, or a walk moves it") + }, + Says: func(d *Delivery) string { return "merged, and nothing for a walk to move: " + d.Plan.Summary }}, + {From: []State{Ready, Held}, Event: EvCovered, To: Covered, + Guard: "merged, no walk of its own open, and every module it moves sent to every machine running it by the " + + "finished walk of a later merge into the same trunk, whose own check passed and which the forge says contains " + + "it: a later merge that contains it was delivered", + Holds: func(d *Delivery, f Facts) error { + switch { + case d.MergedAs == "" || d.CoveredBy == "": + return errors.New("not merged, or nothing later carried its modules") + case d.Walk != nil && (d.Walk.State == walkBuilding || d.Walk.State == walkRolling): + return fmt.Errorf("its own walk %s is open: the controller delivers it", d.Walk.ID) + } + return nil + }, + Says: func(d *Delivery) string { + return "covered: a later merge that contains it was delivered: " + d.CoveredBy + }}, + // **A ready delivery whose verdict went out of date goes back to proposed** (novox/hq ADR 0323 §7): another + // merge moved its trunk. A rejected one stays rejected, its status on the forge saying the verdict is out of date. + {From: []State{Ready}, Event: EvTrunkMoved, To: Proposed, + Guard: "its verdict names a trunk commit that is not its trunk's head", + Holds: func(d *Delivery, f Facts) error { + return unless(d.MergedAs == "" && d.outOfDate() != "", "not merged, and its verdict is not out of date") + }, + Says: func(d *Delivery) string { return d.outOfDate() }}, + {From: []State{Proposed, Checked, Rejected}, Event: EvMergedUnchecked, To: Held, + Guard: "merged without a passing check — a verdict known with it is taken first: only a person decides that it goes on", + Holds: func(d *Delivery, f Facts) error { return unless(d.MergedAs != "", "not merged") }}, + {From: []State{Rejected, Ready}, Event: EvRecheck, To: Proposed, Act: true, + Guard: "a person asked, with why, or its group changed", + Holds: func(d *Delivery, f Facts) error { return unless(f.Why != "", "a recheck says why") }}, + {From: []State{Proposed}, Event: EvReask, To: Proposed, Act: true, + Guard: "its check was asked and no verdict came within its bound, and it was not re-asked before: healer H2 " + + "asks it once more, with why — after that it is the operator's", + Holds: func(d *Delivery, f Facts) error { + switch { + case f.Why == "" || f.By == "": + return errors.New("a re-ask says who and why") + case d.Check != nil: + return errors.New("its verdict is known") + case d.CheckAsk == nil: + return errors.New("its check was never asked: a look asks it, not H2") + case d.CheckAsk.Reasked > 0: + return errors.New("its check was re-asked once already: the operator's") + } + return nil + }}, + + {From: []State{Published}, Event: EvSuperseded, To: Superseded, + Guard: "its walk was superseded and names no walk that took it over: a walk that took it over answers its merge, and it follows that walk (novox/hq ADR 0276)", + Holds: func(d *Delivery, f Facts) error { + if d.Walk != nil && d.Walk.State == walkSuperseded && d.Walk.TakenOverBy != "" { + return fmt.Errorf("its walk was taken over by %s, which answers its merge: it follows that walk", d.Walk.TakenOverBy) + } + return unless(d.Walk != nil && d.Walk.State == walkSuperseded, "its walk is not superseded") + }}, + {From: []State{Published}, Event: EvDone, To: Delivered, + Guard: "nothing for a walk to move; or its walk, carrying its own commit, ended done before its word having built " + + "every module it moves or said it unchanged (novox/hq issue 474); or its walk said every module it moves " + + "unchanged, its build source holding none of the files changed, so it moves nothing (novox/hq issue 611, " + + "ADR 0270 §1)", + Holds: func(d *Delivery, f Facts) error { + if d.Walk == nil && d.MovesNothing() { + return nil + } + moved, _ := d.walkMovedItsModules() + return unless(d.endedUnstarted() && moved && (!d.carriedLater() || d.builtNoneOfIts()), "a walk moves it") + }, + Says: func(d *Delivery) string { + if d.Walk == nil { + return "nothing for a walk to move" + } + unchanged := d.unchangedOfIts() + switch { + case len(unchanged) == 0: + return fmt.Sprintf("its walk %s ended done before its word, having built every module it moves", d.Walk.ID) + case d.builtNoneOfIts(): + return fmt.Sprintf("its walk %s moved nothing: %s matched what runs, its build source holding none of "+ + "the files changed, so the build the mesh holds is its build (novox/hq issue 611)", d.Walk.ID, + strings.Join(unchanged, ", ")) + } + return fmt.Sprintf("its walk %s ended done before its word, having built every module it moves but %s, "+ + "which matched what runs (novox/hq issue 611)", d.Walk.ID, strings.Join(unchanged, ", ")) + }}, + {From: []State{Published, Ready}, Event: EvHold, To: Held, + Guard: "something holds it for a person: its group's composed check did not pass for the heads that merged, " + + "or it merged and no walk was opened for it", + Holds: func(d *Delivery, f Facts) error { + return unless(d.HeldWhy != "" && d.MergedAs != "", "nothing holds it, or it is not on the trunk") + }}, + {From: []State{Published}, Event: EvGo, To: Delivering, + Guard: "its walk started: let go by this owner in its group's order, by a person, or on the controller's own path " + + "— not a failed walk that carried a later commit for it, after which it waits for a walk of its own commit", + Holds: func(d *Delivery, f Facts) error { + if d.Walk != nil && d.Walk.State == walkFailed && d.carriedLater() { + return errors.New("its walk, which carried a later commit for it, failed: it waits for a walk of its own commit") + } + if d.walkStopped() { + return errors.New("its walk was stopped: it waits for the walk that walks its merge again, or ends stopped") + } + return unless(d.Walk != nil && d.Walk.Started(), "its walk waits") + }}, + // **A walk that ended before its word settles its delivery** (novox/hq issue 474): a batch's walk the + // controller judged to move nothing it holds is done when it is cut, and a merge it carried was left published, + // waiting for a start that cannot come. Covered when that walk, carrying a later merge for it, built its modules, + // or when a later delivered merge carried them (ADR 0270's covered test); stopped, saying why, when neither. + {From: []State{Published}, Event: EvCovered, To: Covered, + Guard: "its walk ended done before its word, and its modules were delivered by a later merge that contains it: " + + "that walk, carrying it, built them, or a later merge's finished walk sent them everywhere (ADR 0270)", + Holds: func(d *Delivery, f Facts) error { + if !d.endedUnstarted() { + return errors.New("its walk has not ended before its word") + } + moved, _ := d.walkMovedItsModules() + return unless(d.carriedLater() && moved || d.MergedAs != "" && d.CoveredBy != "", + "nothing later delivered its modules") + }, + Says: func(d *Delivery) string { + if moved, _ := d.walkMovedItsModules(); d.carriedLater() && moved { + return "covered: a later merge that contains it was delivered: " + d.Carrier + } + return "covered: a later merge that contains it was delivered: " + d.CoveredBy + }}, + {From: []State{Published}, Event: EvStop, To: Stopped, + Guard: "its walk ended done before its word without building every module it moves, and no later merge that " + + "contains it delivered them: no start can come, and nothing delivered it (novox/hq issue 474); or the walk " + + "it waits on was stopped through this owner and walks its merge again nowhere — left out, or the walk " + + "ended whole (novox/hq ADR 0299)", + Holds: func(d *Delivery, f Facts) error { + if d.walkStopped() && !d.walkedAgain() { + return nil + } + return unless(d.endedUnstarted() && d.Stranded != "", "its walk has not ended unstarted, or something may cover it") + }, + Says: func(d *Delivery) string { + if d.walkStopped() { + return d.stoppedWords() + } + return d.Stranded + }}, + {From: []State{Held}, Event: EvGo, To: Delivering, + Guard: "its walk started on a word that was not this owner's: the controller's own path, which waits for " + + "nobody, or a person's `plans go`", + Holds: func(d *Delivery, f Facts) error { + if d.Walk != nil && len(d.Walk.BackupsFailed) > 0 && d.Walk.Revision > d.BackupsAgainAfter { + return errors.New("a backup before the send failed: only a release asks the backups again") + } + return unless(d.Walk != nil && d.Walk.Started() && (!d.Walk.Waited() || d.Walk.LetGoByAPerson()), + "its walk was not started by a person or the controller's own path") + }}, + {From: []State{Held}, Event: EvLifted, To: Published, + Guard: "held only for its group's composed check, which has since passed or warned for the heads that merged, or " + + "held because no walk answered its merge, and one now does: it is published again and waits for its turn", + Holds: func(d *Delivery, f Facts) error { + return unless(d.Lifted != "" && d.HeldWhy == "" && d.MergedAs != "" && walkCanGoOn(d), "nothing lifted its hold") + }, + Says: func(d *Delivery) string { return "lifted: " + d.Lifted }}, + {From: []State{Held}, Event: EvRelease, To: Delivering, Act: true, + Guard: "a person's decision, with why", + Holds: func(d *Delivery, f Facts) error { + return unless(f.Why != "" && f.By != "", "a release is a person's word, with why") + }}, + + // **A failed backup holds the delivery while nothing was sent** (novox/hq ADR 0404 clause 4): the controller asks + // the backups before it asks or sends anything, and a failed or silent one holds its plan. Observed, from the + // plan's record; it does not lift by itself: a person's release asks the controller to take the backups again. + {From: []State{Delivering}, Event: EvBackupFailed, To: Held, + Guard: "its walk's record names a backup before the send that failed or stayed silent, and nothing of the plan " + + "was sent; not one the controller was already asked to take again", + Holds: func(d *Delivery, f Facts) error { + switch { + case d.Walk == nil || len(d.Walk.BackupsFailed) == 0: + return errors.New("no backup failed") + case len(d.Walk.Moved) > 0: + return errors.New("a module of the plan was sent already") + case d.Walk.Revision <= d.BackupsAgainAfter: + return errors.New("the controller was asked to take the backups again, and has not said so yet") + } + return nil + }, + Says: func(d *Delivery) string { return d.Walk.backupWords() }}, + {From: []State{Delivering}, Event: EvSuperseded, To: Superseded, + Guard: "its walk was superseded and names no walk that took it over: a walk that took it over answers its merge, and it follows that walk (novox/hq ADR 0276)", + Holds: func(d *Delivery, f Facts) error { + if d.Walk != nil && d.Walk.State == walkSuperseded && d.Walk.TakenOverBy != "" { + return fmt.Errorf("its walk was taken over by %s, which answers its merge: it follows that walk", d.Walk.TakenOverBy) + } + return unless(d.Walk != nil && d.Walk.State == walkSuperseded, "its walk is not superseded") + }}, + {From: []State{Delivering}, Event: EvDone, To: Delivered, + Guard: "its walk is done, carrying its own commit: every machine of its deploy plan passed, or was left as its policy says", + Holds: func(d *Delivery, f Facts) error { + return unless(d.Walk != nil && d.Walk.State == walkDone && !d.carriedLater(), "its walk is not done, or carried a later commit for it") + }}, + {From: []State{Delivering}, Event: EvCovered, To: Covered, + Guard: "its walk is done, carrying a later merge of its branch that contains its merge: answered by its successor in " + + "one walk (novox/hq ADR 0276)", + Holds: func(d *Delivery, f Facts) error { + return unless(d.Walk != nil && d.Walk.State == walkDone && d.carriedLater(), "its walk is not done, or carried its own commit") + }, + Says: func(d *Delivery) string { + return "covered: a later merge that contains it was delivered: " + d.Carrier + }}, + {From: []State{Delivering}, Event: EvStop, To: Stopped, + Guard: "its walk was stopped through this owner, and walks its merge again nowhere: left out, or the walk ended " + + "whole (novox/hq ADR 0299)", + Holds: func(d *Delivery, f Facts) error { + if d.walkedAgain() { + return fmt.Errorf("its walk was stopped without it, and %s walks its merge again", d.Walk.WalkedAgainBy) + } + return unless(d.walkStopped(), "its walk was not stopped") + }, + Says: func(d *Delivery) string { return d.stoppedWords() }}, + {From: []State{Delivering}, Event: EvFailed, To: Failed, + Guard: "its walk failed, carrying its own commit: a gate on a first machine (what it carried put back), a build, a machine", + Holds: func(d *Delivery, f Facts) error { + return unless(d.Walk != nil && d.Walk.State == walkFailed && d.Walk.StoppedBy == "" && !d.carriedLater(), + "its walk did not fail, or carried a later commit for it") + }}, + {From: []State{Delivering}, Event: EvRewalk, To: Published, + Guard: "its walk failed carrying a later merge of its branch for it: a failed walk marks no merge delivered, and this " + + "merge waits, published, for a walk of its own commit (novox/hq ADR 0276 decision 3); or its walk was stopped " + + "for another merge, and its merge is walked again without that one (novox/hq ADR 0299)", + Holds: func(d *Delivery, f Facts) error { + if d.walkedAgain() { + return nil + } + return unless(d.Walk != nil && d.Walk.State == walkFailed && d.Walk.StoppedBy == "" && d.carriedLater(), + "its walk did not fail, or carried its own commit") + }, + Says: func(d *Delivery) string { + if d.walkedAgain() { + return fmt.Sprintf("its walk %s was stopped for another merge (%s): %s walks its merge again without that one", + d.Walk.ID, d.Walk.StoppedWhy, d.Walk.WalkedAgainBy) + } + return fmt.Sprintf("its walk %s, which carried %s for it, failed: it waits for a walk of its own commit", + d.Walk.ID, shortOf(d.AnsweredBy, 12)) + }}, + + {From: notFinal, Event: EvStop, To: Stopped, Act: true, + Guard: "a person, with why, or its group stopped by a member before it", + Holds: func(d *Delivery, f Facts) error { return unless(f.Why != "" && f.By != "", "a stop says who and why") }}, +} + +// walkCanGoOn is whether a delivery published again would be delivered by what it has: its own walk still +// open, or nothing for a walk to move. +func walkCanGoOn(d *Delivery) bool { + if d.Walk == nil { + return d.MovesNothing() + } + return d.Walk.State == walkBuilding || d.Walk.State == walkRolling +} + +// historyLate is how long after its merge a merge heard only from the forge's word is history: the forge +// announces a merge within seconds, so one heard this late was replayed from the bus's history. +const historyLate = 15 * time.Minute + +// isHistory is whether a delivery is history, answering why not: made from the forge's word at its merge, on +// the trunk, its merge heard historyLate or more after it happened, and no walk of it open or waiting — a +// walk the controller keeps for it is a delivery, never history. +func isHistory(d *Delivery) error { + switch { + case d.MergedAs == "": + return errors.New("it is not on the trunk") + case !d.FromForgesWord(): + return errors.New("this owner heard its head: it is not history made from the forge's word") + case d.MergedAt.IsZero() || d.Created.Sub(d.MergedAt) < historyLate: + return fmt.Errorf("its merge was heard %s after it happened, within %s: a live merge, not history", + d.Created.Sub(d.MergedAt).Round(time.Second), historyLate) + case d.Walk != nil && (d.Walk.Waits || d.Walk.State == walkBuilding || d.Walk.State == walkRolling): + return fmt.Errorf("its walk %s is open: the controller delivers it", d.Walk.ID) + } + return nil +} + +// rowFor is the row a state and event name; nil when the table holds none. +func rowFor(from State, ev Event, act bool) *Row { + for i := range Table { + r := &Table[i] + if r.Event != ev || r.Act != act { + continue + } + for _, s := range r.From { + if s == from { + return r + } + } + } + return nil +} + +// ErrRefused is a transition the table does not hold, or whose guard does not. +type ErrRefused struct { + ID string + From State + Event Event + Why string +} + +func (e ErrRefused) Error() string { + from := string(e.From) + if from == "" { + from = "nothing" + } + return fmt.Sprintf("%s: %s from %s is refused — %s", e.ID, e.Event, from, e.Why) +} + +// Apply takes one transition, or refuses it. The delivery keeps the transition, bounded. +func Apply(d *Delivery, ev Event, act bool, f Facts) (Transition, error) { + r := rowFor(d.State, ev, act) + if r == nil { + kind := "observed" + if act { + kind = "asked" + } + return Transition{}, ErrRefused{ID: d.ID, From: d.State, Event: ev, + Why: "the table holds no such transition (" + kind + ")"} + } + if err := r.Holds(d, f); err != nil { + return Transition{}, ErrRefused{ID: d.ID, From: d.State, Event: ev, Why: "its guard does not hold: " + err.Error()} + } + why := f.Why + if why == "" && r.Says != nil { + why = r.Says(d) + } + if why == "" { + why = r.Guard + } + by := f.By + if by == "" && (ev == EvMerged || ev == EvDone) && d.MergedAs != "" { + // A merge this owner made on the operator's Approve names it as its author (novox/hq issue 722). + by = d.Ready.mergedBy() + } + t := Transition{At: f.Now, From: d.State, To: r.To, Event: ev, Why: why, By: by} + d.State, d.Since = r.To, f.Now + if r.Event == EvBackupFailed { + d.HeldWhy = why + } + d.Transitions = append(d.Transitions, t) + if len(d.Transitions) > keptTransitions { + d.Transitions = d.Transitions[len(d.Transitions)-keptTransitions:] + } + return t, nil +} + +// keptTransitions is how many transitions a delivery keeps; its note on the commit keeps every one. +const keptTransitions = 100 + +// settle takes every observed transition whose guard holds, in the table's order, until none does — so a +// delivery always stands where its facts put it, whatever order they arrived in. +func settle(d *Delivery, f Facts) []Transition { + var taken []Transition + for range len(Table) + 1 { + moved := false + for i := range Table { + r := &Table[i] + if r.Act || !contains(r.From, d.State) { + continue + } + if t, err := Apply(d, r.Event, false, f); err == nil { + taken = append(taken, t) + moved = true + break + } + } + if !moved { + break + } + } + return taken +} + +func contains(states []State, s State) bool { + for _, x := range states { + if x == s { + return true + } + } + return false +} + +// Bounds are how long a delivery may be in a state before `stalled` lists it, and what healer H2 may do +// then — always a transition the table holds, never one of its own (novox/hq ADR 0239 decision 9). +type Bound struct { + For time.Duration + // H2 is the transitions H2 may take after the bound, by event; empty when the state is the operator's. + H2 []Event + Says string +} + +// Bounds are every state's. +var Bounds = map[State]Bound{ + Proposed: {For: time.Hour, H2: []Event{EvReask}, Says: "its check was asked and not answered: H2 asks it once " + + "more, then the check's own watchdog (S6) and the operator speak for it"}, + Checked: {For: time.Minute, Says: "a verdict is decided at once"}, + Published: {For: 30 * time.Minute, H2: []Event{EvSuperseded, EvGo}, Says: "its walk waits for its turn or its word"}, + Held: {For: 24 * time.Hour, Says: "it waits for the operator"}, + Delivering: {For: 2 * time.Hour, H2: []Event{EvDone, EvSuperseded, EvFailed}, Says: "its walk runs"}, +} + +// boundSince is when a delivery's bound runs from: its state's start, except for a proposed delivery whose +// check this owner asked — its bound is "asked and not answered", so it runs from the ask. One never asked +// runs from its start: an ask that cannot be made is still listed. +func boundSince(d *Delivery) time.Time { + if d.State == Proposed { + if d.AskedAgain != nil && d.outOfDate() != "" { + return *d.AskedAgain + } + if d.CheckAsk != nil { + return d.CheckAsk.At + } + return d.Since + } + return d.Since +} + +// heldUnwalked is the bound of a delivery held because no walk answers its merge (novox/hq issue 362): none. It +// waited the ten minutes a walk opens within already, and nothing but a person moves it now. +var heldUnwalked = Bound{For: 0, Says: "merged, and no walk answers its merge: the operator's"} + +// rewalkBound is the bound of a merge whose walk carried a later commit for it and failed: a walk of its own +// commit opens within the ten minutes a walk opens within (ADR 0276 decision 3), or nothing but a person moves it. +var rewalkBound = Bound{For: unwalked, Says: "the walk that carried a later commit for it failed, and no walk of " + + "its own commit opened: the operator's"} + +// boundOf is a delivery's bound: its state's, or none for a delivery held because no walk answers it. +func boundOf(d *Delivery) (Bound, bool) { + if d.State == Proposed && d.outOfDate() != "" { + // Its check is out of date and not asked: nothing is running for it, and it costs nothing (ADR 0323 §7). + // Its bound starts when its check is asked: a recheck puts the verdict away; the merge tool's ask announces the + // head again, and its bound runs from then. + if d.AskedAgain == nil { + return Bound{}, false + } + b := Bounds[Proposed] + b.H2, b.Says = nil, "its verdict is "+d.outOfDate()+"; its check was asked again and not answered: the operator's" + return b, true + } + if d.State == Held && d.Unwalked && d.Walk == nil { + return heldUnwalked, true + } + if d.waitsForItsOwnWalk() { + return rewalkBound, true + } + if (d.State == Published || d.State == Delivering) && d.HeldBack != "" { + // Its walk carries a stopped merge and is not let go (ADR 0299): said with why, past the bound of its state. + b := Bounds[d.State] + b.For, b.Says = Bounds[Published].For, d.HeldBack // a walk not let go waits for a word, whatever its state + return b, true + } + b, bounded := Bounds[d.State] + return b, bounded +} + +// pastBound is whether a delivery has been where it is longer than its state's bound. +func pastBound(d *Delivery, now time.Time) bool { + b, bounded := boundOf(d) + return bounded && !d.State.Final() && (b.For == 0 || now.Sub(boundSince(d)) > b.For) +} + +// StepState is where one machine stands in a delivering walk. +type StepState string + +// The machine steps (ADR 0239 decision 2): sent, judged, passed or failed, and put back. +const ( + StepSent StepState = "sent" + StepJudging StepState = "judging" + StepPassed StepState = "passed" + StepFailed StepState = "failed" + StepRolledBack StepState = "rolled-back" +) + +// StepTable is every move a machine's step may make. From "" is a step first heard of. +var StepTable = map[StepState][]StepState{ + "": {StepSent, StepJudging, StepPassed, StepFailed, StepRolledBack}, + StepSent: {StepJudging, StepPassed, StepFailed}, + StepJudging: {StepPassed, StepFailed}, + StepFailed: {StepRolledBack}, +} + +// StepMay is whether a step may move from one state to another: directly, or through states the walk's +// record passed between two readings of it (sent, judged and failed and put back, read once). +func StepMay(from, to StepState) bool { + if from == to { + return true + } + seen := map[StepState]bool{} + next := []StepState{from} + for len(next) > 0 { + s := next[0] + next = next[1:] + for _, t := range StepTable[s] { + if t == to { + return true + } + if !seen[t] { + seen[t] = true + next = append(next, t) + } + } + } + return false +} + +// TableText is the table as the `table` verb answers it. +func TableText() map[string]any { + var rows []map[string]any + for _, r := range Table { + from := make([]string, 0, len(r.From)) + for _, s := range r.From { + if s == None { + from = append(from, "(none)") + continue + } + from = append(from, string(s)) + } + kind := "observed" + if r.Act { + kind = "asked" + } + rows = append(rows, map[string]any{"from": strings.Join(from, ", "), "event": r.Event, "to": r.To, + "guard": r.Guard, "taken": kind}) + } + bounds := map[string]any{} + for s, b := range Bounds { + var h2 []string + for _, e := range b.H2 { + h2 = append(h2, string(e)) + } + bounds[string(s)] = map[string]any{"bound": b.For.String(), "h2": h2, "says": b.Says} + } + steps := map[string][]StepState{} + for from, to := range StepTable { + name := string(from) + if name == "" { + name = "(first heard)" + } + steps[name] = to + } + return map[string]any{"transitions": rows, "bounds": bounds, "machine-steps": steps} +} diff --git a/modules/mesh-delivery/cmd/mesh-delivery/table_test.go b/modules/mesh-delivery/cmd/mesh-delivery/table_test.go new file mode 100644 index 00000000..573e4935 --- /dev/null +++ b/modules/mesh-delivery/cmd/mesh-delivery/table_test.go @@ -0,0 +1,260 @@ +package main + +import ( + "errors" + "testing" + "time" +) + +// The table walked (novox/hq ADR 0239 decision 2): every row it holds can be taken when its guard holds and +// is refused when it does not; every pair it does not hold is refused, by name; and the rules the table +// exists for are true of the table itself, not of the code around it. + +// aDeliveryFor is a delivery whose facts make the row's guard hold. +func aDeliveryFor(r Row) (*Delivery, Facts) { + now := time.Date(2026, 10, 6, 12, 0, 0, 0, time.UTC) + d := &Delivery{ID: "novox/app@c0ffee000000", Repository: "novox/app", Commit: "c0ffee000000", Number: 7} + f := Facts{Now: now} + switch r.Event { + case EvAppeared: + d.MergedAs, d.HeldWhy = "c0ffee000000", "no pull request" + case EvAdopted, EvGo: + d.Walk = &WalkSeen{ID: "plan-1", State: walkRolling} + if r.From[0] == Held { + d.Walk.WaitedFor, d.Walk.GoBy = "mesh-delivery", "a person (jochen)" + } + case EvNewHead: + d.NewerHead = "beef" + case EvClosed: + d.ClosedUnmerged = true + case EvMerged: + d.MergedAs, d.Walk = "aaaa1111", &WalkSeen{ID: "plan-1", State: walkBuilding, Waits: true, WaitedFor: "mesh-delivery"} + case EvMergedUnchecked: + d.MergedAs = "aaaa1111" + case EvHistory, EvRetire: + d.MergedAs, d.FromForge = "aaaa1111", true + d.MergedAt, d.Created = now.Add(-48*time.Hour), now + if r.Event == EvHistory { + d.History = "merged two days before this owner heard of it" + } else { + f.Why, f.By = "history adopted at the switch", "jochen" + } + case EvChecked, EvAccepted: + d.Check = &Verdict{Gate: "pass"} + case EvRefused: + d.Check = &Verdict{Gate: "fail"} + case EvRecheck, EvRelease, EvStop: + f.Why, f.By = "a person's reason", "jochen" + if r.Event == EvStop && !r.Act { + f = Facts{Now: now} + d.Walk = &WalkSeen{ID: "plan-1", State: walkFailed, StoppedBy: "mesh-delivery for jochen"} + if r.From[0] == Published { + // Its walk ended done before its word, and nothing delivered it (novox/hq issue 474). + d.Walk = &WalkSeen{ID: "plan-1", State: walkDone, Waits: true, WaitedFor: "mesh-delivery", MovedSaid: true} + d.Stranded = "its walk plan-1 ended done before its word without building app" + } + } + case EvReask: + f.Why, f.By = "delivery stalled", byH2 + d.CheckAsk = &CheckAsk{ID: "check-1", At: now.Add(-2 * time.Hour), By: byOwner} + case EvHold: + d.HeldWhy, d.MergedAs = "its group's check did not pass", "aaaa1111" + case EvDone: + switch { + case r.From[0] == Published: + d.Plan = &DeliveryPlan{Summary: "moves nothing"} + case r.From[0] == Proposed: + d.MergedAs, d.Plan = "aaaa1111", &DeliveryPlan{Summary: "builds nothing"} + default: + d.Walk = &WalkSeen{ID: "plan-1", State: walkDone} + } + case EvCovered: + d.MergedAs, d.CoveredBy = "aaaa1111", "delivered with novox/app@bbbb (merged as bbbb2222, walk plan-2): app" + if r.From[0] == Published { + // Its walk ended done before its word; a later merge delivered its modules (novox/hq issue 474). + d.Walk = &WalkSeen{ID: "plan-1", State: walkDone, Waits: true, WaitedFor: "mesh-delivery", MovedSaid: true} + } + if r.From[0] == Delivering { + // Answered by its successor in one walk (ADR 0276): the walk done, carrying a later merge. + d.CoveredBy, d.AnsweredBy, d.Carrier = "", "bbbb2222", "novox/app@bbbb (merged as bbbb2222, walk plan-2)" + d.Walk = &WalkSeen{ID: "plan-2", State: walkDone} + } + case EvRewalk: + d.MergedAs, d.AnsweredBy = "aaaa1111", "bbbb2222" + d.Walk = &WalkSeen{ID: "plan-2", State: walkFailed} + case EvTrunkMoved: + d.Check = &Verdict{Gate: "pass", Head: "c0ffee000000", Trunk: "d28ca95bbbbb"} + d.TrunkNow = "690966dccccccccc" + case EvLifted: + d.MergedAs, d.Lifted = "aaaa1111", "its group's composed check passed" + d.Walk = &WalkSeen{ID: "plan-1", State: walkBuilding, Waits: true, WaitedFor: "mesh-delivery"} + case EvBackupFailed: + d.Walk = &WalkSeen{ID: "plan-1", State: walkBuilding, Revision: 4, + BackupsFailed: []WalkBackupFailure{{Machine: "ace", Module: "pg", Words: "no space left on device"}}} + case EvFailed: + d.Walk = &WalkSeen{ID: "plan-1", State: walkFailed} + case EvSuperseded: + d.Walk = &WalkSeen{ID: "plan-1", State: walkSuperseded} + } + return d, f +} + +func TestEveryRowOfTheTableIsTakenWhenItsGuardHoldsAndRefusedWhenNot(t *testing.T) { + for _, r := range Table { + for _, from := range r.From { + d, f := aDeliveryFor(r) + d.State = from + got, err := Apply(d, r.Event, r.Act, f) + if err != nil { + t.Errorf("%s —%s→ %s was refused with its guard holding: %v", orNothing(from), r.Event, r.To, err) + continue + } + if d.State != r.To || got.From != from || got.To != r.To || len(d.Transitions) != 1 { + t.Errorf("%s —%s→ left it %s, transition %+v", orNothing(from), r.Event, d.State, got) + } + // The guard not holding: a delivery with no facts. + bare := &Delivery{ID: "novox/app@bare", State: from, Check: &Verdict{Gate: "pass"}} + if r.Event == EvAccepted || r.Event == EvChecked { + bare.Check = nil + if r.Event == EvAccepted { + bare.Check = &Verdict{Gate: "fail"} + } + } + if r.Event == EvRefused { + bare.Check = &Verdict{Gate: "pass"} + } + if _, err := Apply(bare, r.Event, r.Act, Facts{Now: f.Now}); err == nil { + t.Errorf("%s —%s→ %s was taken with no fact for its guard", orNothing(from), r.Event, r.To) + } else if bare.State != from || len(bare.Transitions) != 0 { + t.Errorf("a refused transition moved the delivery to %s", bare.State) + } + } + } +} + +func TestEveryPairTheTableDoesNotHoldIsRefusedByName(t *testing.T) { + events := []Event{EvAnnounced, EvAppeared, EvAdopted, EvChecked, EvAccepted, EvRefused, EvRecheck, EvReask, EvNewHead, + EvClosed, EvMerged, EvMergedUnchecked, EvGo, EvHold, EvRelease, EvDone, EvFailed, EvSuperseded, EvStop, + EvHistory, EvRetire, EvCovered, EvLifted, EvRewalk, EvTrunkMoved, EvBackupFailed} + for _, from := range append([]State{None}, AllStates...) { + for _, ev := range events { + for _, act := range []bool{false, true} { + if rowFor(from, ev, act) != nil { + continue + } + d := &Delivery{ID: "novox/app@c0ffee", State: from, MergedAs: "x", NewerHead: "y", Check: &Verdict{Gate: "pass"}, + Walk: &WalkSeen{ID: "p", State: walkDone}, HeldWhy: "h", ClosedUnmerged: true, Number: 1, Commit: "c"} + _, err := Apply(d, ev, act, Facts{Now: time.Now(), Why: "w", By: "b"}) + var refused ErrRefused + if !errors.As(err, &refused) || refused.Event != ev || refused.From != from { + t.Errorf("%s —%s (act %v)→ was not refused by name: %v", orNothing(from), ev, act, err) + } + if d.State != from { + t.Errorf("a pair the table does not hold moved %s to %s", orNothing(from), d.State) + } + } + } + } +} + +// The rules the table is for, read off the table. +func TestTheTableKeepsItsRules(t *testing.T) { + for _, r := range Table { + for _, from := range r.From { + // Nothing leaves a final state. + if from.Final() { + t.Errorf("%s is final and the table leaves it by %s", from, r.Event) + } + // Nothing goes back before the trunk once a delivery is on it. + if (from == Published || from == Delivering || from == Held) && + (r.To == Proposed || r.To == Checked || r.To == Ready || r.To == Rejected) { + t.Errorf("%s goes back to %s by %s", from, r.To, r.Event) + } + } + // The trunk rule: only a ready delivery is published, by its merge — or a held one published again when + // what held it is gone (issue 316); nothing off the trunk delivers. + // And a merge whose walk carried a later commit for it and failed waits there for a walk of its own (ADR 0276). + if r.To == Published && !(r.Event == EvMerged && len(r.From) == 1 && r.From[0] == Ready) && + !(r.Event == EvLifted && len(r.From) == 1 && r.From[0] == Held) && + !(r.Event == EvRewalk && len(r.From) == 1 && r.From[0] == Delivering) { + t.Errorf("%v —%s→ published: only a ready delivery's merge publishes", r.From, r.Event) + } + // Nothing a person did not decide ends a delivery as stopped or failed by observation from held. + if contains(r.From, Held) && !r.Act && (r.To == Stopped || r.To == Failed) { + t.Errorf("held —%s→ %s by observation", r.Event, r.To) + } + if r.To == Delivering { + for _, from := range r.From { + if from == Proposed || from == Checked || from == Ready || from == Rejected { + t.Errorf("%s delivers by %s without reaching the trunk", from, r.Event) + } + } + } + // Held is left only by a person or a walk that started on a word that was not this owner's. + if contains(r.From, Held) && r.To == Delivering && r.Event == EvRelease && !r.Act { + t.Error("a held delivery is released by observation") + } + } + // Every state that is not final has a bound or is the operator's to wait in, and every H2 transition is + // a row the table holds from that state: observed once the walk's record is read again, or H2's own ask. + for state, b := range Bounds { + for _, ev := range b.H2 { + if rowFor(state, ev, false) == nil && rowFor(state, ev, true) == nil { + t.Errorf("H2 may take %s from %s, which the table does not hold", ev, state) + } + } + } + // H2 re-asks a check once: a second re-ask is refused by the table itself. + d0 := &Delivery{ID: "novox/app@c0", State: Proposed, Number: 1, CheckAsk: &CheckAsk{ID: "check-1", Reasked: 1}} + if _, err := Apply(d0, EvReask, true, Facts{Now: time.Now(), By: byH2, Why: "stalled"}); err == nil { + t.Fatal("a check re-asked once was re-asked again") + } + // A ready delivery off the trunk is refused publication, whatever else is true of it. + d := &Delivery{ID: "novox/app@c0", State: Ready, Walk: &WalkSeen{ID: "plan-x", State: walkRolling}} + if _, err := Apply(d, EvMerged, false, Facts{Now: time.Now()}); err == nil { + t.Fatal("a commit not on the trunk was published") + } +} + +func TestTheMachineStepTable(t *testing.T) { + for _, c := range []struct { + from, to StepState + may bool + }{ + {"", StepSent, true}, {StepSent, StepJudging, true}, {StepJudging, StepPassed, true}, + {StepJudging, StepFailed, true}, {StepFailed, StepRolledBack, true}, {StepSent, StepFailed, true}, + {StepPassed, StepFailed, false}, {StepRolledBack, StepPassed, false}, {StepPassed, StepSent, false}, + {StepJudging, StepSent, false}, {StepFailed, StepPassed, false}, + } { + if StepMay(c.from, c.to) != c.may { + t.Errorf("%q → %q: may %v, wanted %v", c.from, c.to, !c.may, c.may) + } + } +} + +// settle stands a delivery where its facts put it, whatever order they came in. +func TestSettleIsTheOrderOfTheFactsNotOfTheirArrival(t *testing.T) { + now := time.Now() + d := &Delivery{ID: "novox/app@c0", Number: 1, Commit: "c0"} + if _, err := Apply(d, EvAnnounced, false, Facts{Now: now}); err != nil { + t.Fatal(err) + } + // The verdict, the merge and a finished walk, all known before anything settled. + d.Check = &Verdict{Gate: "pass"} + d.MergedAs = "m1" + d.Walk = &WalkSeen{ID: "plan-1", State: walkDone} + ts := settle(d, Facts{Now: now}) + var path []State + for _, t := range ts { + path = append(path, t.To) + } + want := []State{Checked, Ready, Published, Delivering, Delivered} + if len(path) != len(want) { + t.Fatalf("settled through %v, wanted %v", path, want) + } + for i := range want { + if path[i] != want[i] { + t.Fatalf("settled through %v, wanted %v", path, want) + } + } +} diff --git a/modules/mesh-delivery/cmd/mesh-delivery/timers_source_test.go b/modules/mesh-delivery/cmd/mesh-delivery/timers_source_test.go new file mode 100644 index 00000000..731c4863 --- /dev/null +++ b/modules/mesh-delivery/cmd/mesh-delivery/timers_source_test.go @@ -0,0 +1,85 @@ +package main + +import ( + "os" + "path/filepath" + "regexp" + "strconv" + "strings" + "testing" +) + +// **A timer in this module names the bounded step it retries** (novox/hq ADR 0406 "How it is checked", issue 975): the +// module acts on events, so a clock that fires on its own — a ticker, time.Tick — is a poll, and a one-shot timer +// (time.AfterFunc, time.NewTimer) is allowed only for a bounded step, said in a comment on the line above: +// `// retries: `. This reads the module's Go source, not its tests. + +var ( + pollingClock = regexp.MustCompile(`\btime\.(NewTicker|Tick)\(`) + oneShot = regexp.MustCompile(`\btime\.(AfterFunc|NewTimer)\(`) + retriesLine = regexp.MustCompile(`^\s*//\s*retries:\s*\S`) +) + +// timerFaults are the lines of a source file that make a clock of their own without naming the step it retries. +func timerFaults(name, src string) []string { + var faults []string + lines := strings.Split(src, "\n") + for i, line := range lines { + code, _, _ := strings.Cut(line, "//") + switch { + case pollingClock.MatchString(code): + faults = append(faults, name+":"+strconv.Itoa(i+1)+": a ticker polls; act on an event, or name a bounded step's one-shot timer: "+strings.TrimSpace(line)) + case oneShot.MatchString(code) && (i == 0 || !retriesLine.MatchString(lines[i-1])): + faults = append(faults, name+":"+strconv.Itoa(i+1)+": a timer with no `// retries: ` comment on the line above: "+strings.TrimSpace(line)) + } + } + return faults +} + +func TestNoTimerInTheModuleFiresWithoutNamingTheBoundedStepItRetries(t *testing.T) { + files, err := filepath.Glob("*.go") + if err != nil || len(files) == 0 { + t.Fatalf("no source read: %v", err) + } + timers := 0 + for _, f := range files { + if strings.HasSuffix(f, "_test.go") { + continue + } + raw, err := os.ReadFile(f) + if err != nil { + t.Fatal(err) + } + for _, fault := range timerFaults(f, string(raw)) { + t.Error(fault) + } + timers += len(oneShot.FindAllString(string(raw), -1)) + } + if timers == 0 { + t.Fatal("the test found no timer at all: it is not reading the source it should") + } +} + +// The checker itself: the tick this module had before it acted on events, and a timer that names nothing, fail; a +// named one passes. +func TestTheTimerCheckFailsOnATickerAndOnAnUnnamedTimer(t *testing.T) { + was := "\tgo func() {\n\t\ttick := time.NewTicker(10 * time.Second)\n\t\tdefer tick.Stop()\n\t}()\n" + if got := timerFaults("main.go", was); len(got) != 1 || !strings.Contains(got[0], "main.go:2") { + t.Fatalf("the old tick was not found: %v", got) + } + if got := timerFaults("x.go", "for range time.Tick(time.Second) {}\n"); len(got) != 1 { + t.Fatalf("time.Tick was not found: %v", got) + } + if got := timerFaults("x.go", "\ttime.AfterFunc(d, f)\n"); len(got) != 1 { + t.Fatalf("an unnamed timer was not found: %v", got) + } + if got := timerFaults("x.go", "\t// retries:\n\tt := time.NewTimer(d)\n"); len(got) != 1 { + t.Fatalf("a comment that names no step passed: %v", got) + } + if got := timerFaults("x.go", "\t// retries: the refused merge, after a minute\n\ttime.AfterFunc(d, f)\n"); len(got) != 0 { + t.Fatalf("a named timer failed: %v", got) + } + if got := timerFaults("x.go", "\t// time.NewTicker was here\n"); len(got) != 0 { + t.Fatalf("a comment about a ticker failed: %v", got) + } +} diff --git a/modules/mesh-delivery/cmd/mesh-delivery/timers_test.go b/modules/mesh-delivery/cmd/mesh-delivery/timers_test.go new file mode 100644 index 00000000..370d1127 --- /dev/null +++ b/modules/mesh-delivery/cmd/mesh-delivery/timers_test.go @@ -0,0 +1,212 @@ +package main + +import ( + "errors" + "strings" + "testing" + "time" +) + +// **A step that waits on the clock says when it is looked at again** (novox/hq ADR 0406 §1, issue 975): there is no +// tick to find it, so each grace, bound and retry is registered (lookAt) when a look meets it, and the one timer fires +// a look for the earliest. A look that meets nothing waiting registers nothing. + +func (w *world) wantedAt() (time.Time, string) { + w.h.wanted.mu.Lock() + defer w.h.wanted.mu.Unlock() + return w.h.wanted.at, w.h.wanted.step +} + +func TestALookThatMeetsNothingWaitingOnTheClockWantsNoOtherLook(t *testing.T) { + w := newWorld(t) + w.h.PullUpdated(pr("novox/app", 7, readyHead, "feat/x", "src/a.go")) + w.h.Checked(judgedAt("novox/app", 7, readyHead, "pass", "t1t1t1t1t1t1")) + w.h.React() + if at, step := w.wantedAt(); !at.IsZero() { + t.Fatalf("a look with nothing waiting on the clock wants another at %s for %q", at, step) + } +} + +func TestAProposedDeliveryWithNoVerdictHasItsCheckAskedAtOnceWithNoGrace(t *testing.T) { + w := newWorld(t) + w.ctl.down = false + w.h.PullUpdated(pr("novox/app", 7, readyHead, "feat/x", "src/a.go")) + id := IDOf("novox/app", readyHead) + w.h.React() + if w.delivery(id).CheckAsk == nil || len(w.ctl.checks) != 1 { + t.Fatalf("the first look did not ask the check: %v", w.ctl.checks) + } +} + +func TestAProposedDeliveryWhoseHeadHoldsAPassingVerdictAtTheForgeAdoptsItAndAsksNoCheck(t *testing.T) { + w := newWorld(t) + w.h.PullUpdated(pr("novox/app", 7, readyHead, "feat/x", "src/a.go")) + id := IDOf("novox/app", readyHead) + op := openPull("novox/app", 7, readyHead) + op.Checks = map[string]HeadCheck{"mesh/merge-gate": {State: "success"}, "mesh/repo-check": {State: "success"}} + w.forge.open = []OpenPull{op} + w.h.React() + w.settleAll() + if len(w.ctl.checks) != 0 { + t.Fatalf("a check was asked for a head whose passing verdict stands: %v", w.ctl.checks) + } + if d := w.delivery(id); d.Check == nil || !d.Check.Passes() || d.State != Ready { + t.Fatalf("the standing verdict was not adopted: %+v (%s)", d.Check, d.State) + } +} + +// 1144/AC3: no timer asks a refused merge again; only an event does (here, mergeAgainNow called directly). +func TestAMergeTheForgeRefusedIsAskedAgainOnlyOnAnEventWithNoTimer(t *testing.T) { + w := newWorld(t) + w.h.PullUpdated(pr("novox/app", 7, readyHead, "feat/x", "src/a.go")) + w.h.Checked(judgedAt("novox/app", 7, readyHead, "pass", "t1t1t1t1t1t1")) + w.forge.refuse = errors.New("the forge refused gitea_merge_pull_request: not mergeable yet") + w.forge.open = []OpenPull{openPull("novox/app", 7, readyHead)} + w.h.wantForge() + w.settleAll() + if len(w.forge.merges) != 1 { + t.Fatalf("merges %v", w.forge.merges) + } + if at, step := w.wantedAt(); !at.IsZero() { + t.Fatalf("the refused merge wants a timer at %s for %q; it waits for the next event", at, step) + } + w.forge.refuse = nil + w.settleAll() + if len(w.forge.merges) != 1 { + t.Fatalf("asked again at a look with no event: %v", w.forge.merges) + } + again := w.delivery(IDOf("novox/app", readyHead)) + w.h.mu.Lock() + w.h.mergeAgainNow(again) + w.h.mu.Unlock() + w.settleAll() + if len(w.forge.merges) != 2 || w.delivery(IDOf("novox/app", readyHead)).Ready.Merged == nil { + t.Fatalf("the refused merge was not asked again at the timer: %v", w.forge.merges) + } +} + +func TestAMergeNoWalkOpenedForReadsTheControllersWalksOnceAtItsBound(t *testing.T) { + w := newWorldOver(t, newMemStore(), newFakeController(), newFakeForge(), atCC) + cc := pr(cat, 101, headCC, "feat/claude-code", "modules/claude-code/module.json") + w.h.PullUpdated(cc) + w.h.Checked(verdictMoving(cat, 101, headCC, "claude-code")) + w.h.PullMerged(mergedAt(cc, mergeCC, atCC)) + w.h.WalkMoved(w.ctl.put(catWalk("plan-5", "eeeeeeeeeeee5555", atCC.Add(time.Minute), "dunst"))) + w.h.React() + if at, step := w.wantedAt(); at.IsZero() || at.After(atCC.Add(unwalked)) { + t.Fatalf("the look wanted at %s for %q; wanted by the bound, %s", at, step, atCC.Add(unwalked)) + } + reads := w.ctl.walkReads + w.later(unwalked + time.Second) + w.settleAll() + w.settleAll() + id := IDOf(cat, headCC) + if d := w.delivery(id); d.State != Held || !strings.Contains(d.HeldWhy, "plan-5") { + t.Fatalf("it is %s: %q", d.State, d.HeldWhy) + } + if got := w.ctl.walkReads - reads; got != 1 { + t.Fatalf("the controller's walks were read %d times at the bound; wanted once", got) + } +} + +func TestAnEffectTheForgeCouldNotTakeIsTriedAgainAtItsTimeByTheTimerAlone(t *testing.T) { + w := newWorld(t) + w.forge.down = true + w.h.PullUpdated(pr("novox/app", 7, readyHead, "feat/x", "src/a.go")) + w.h.React() + at, step := w.wantedAt() + if at.IsZero() || !strings.Contains(step, "owed") || at.After(w.now.Add(retryFirst)) { + t.Fatalf("the look wanted at %s for %q; wanted within %s for the owed effect", at, step, retryFirst) + } + w.forge.down = false + w.later(retryFirst + time.Second) + w.h.React() + if len(w.forge.notes) == 0 { + t.Fatal("the owed effect was not tried again at its time") + } +} + +func TestAnAskTheControllerCouldNotAnswerIsAskedAgainAfterAWhileByTheTimerAlone(t *testing.T) { + w := newWorld(t) + w.h.PullUpdated(pr("novox/app", 7, readyHead, "feat/x", "src/a.go")) + w.h.Checked(judgedAt("novox/app", 7, readyHead, "pass", "t1t1t1t1t1t1")) + w.settleAll() + w.ctl.down = true + w.h.mu.Lock() + d := w.h.deliveries[IDOf("novox/app", readyHead)] + d.Plan = nil + w.h.mu.Unlock() + w.h.React() + at, step := w.wantedAt() + if want := w.now.Add(askAgainAfter); !at.Equal(want) || !strings.Contains(step, "plan") { + t.Fatalf("the look wanted at %s for %q; wanted %s for the plan that could not be asked", at, step, want) + } +} + +func TestAStandingGateWithNoRepoCheckIsNotAdoptedAndTheCheckIsAsked(t *testing.T) { + w := newWorld(t) + w.h.PullUpdated(pr("novox/app", 7, readyHead, "feat/x", "src/a.go")) + id := IDOf("novox/app", readyHead) + op := openPull("novox/app", 7, readyHead) + op.Required = []string{"mesh/merge-gate", "mesh/repo-check"} + op.Checks = map[string]HeadCheck{"mesh/merge-gate": {State: "success"}} + w.forge.open = []OpenPull{op} + w.settleAll() + if d := w.delivery(id); d.Check != nil || d.State != Proposed { + t.Fatalf("a gate with no repo-check was adopted: %+v (%s)", d.Check, d.State) + } + if len(w.ctl.checks) != 1 { + t.Fatalf("its check was not asked: %v", w.ctl.checks) + } +} + +func TestAKeptAskWithNoIdIsNoAskAndAStandingVerdictIsAdopted(t *testing.T) { + w := newWorld(t) + w.op.auto = false + w.h.PullUpdated(pr("novox/app", 7, readyHead, "feat/x", "src/a.go")) + id := IDOf("novox/app", readyHead) + w.h.mu.Lock() + w.h.deliveries[id].CheckAsk = &CheckAsk{At: w.now, By: byOwner} + w.h.mu.Unlock() + op := openPull("novox/app", 7, readyHead) + op.Required = []string{"mesh/merge-gate", "mesh/repo-check"} + op.Checks = map[string]HeadCheck{"mesh/merge-gate": {State: "success"}, "mesh/repo-check": {State: "success"}} + w.forge.open = []OpenPull{op} + w.settleAll() + if d := w.delivery(id); d.State != Ready || d.Check == nil { + t.Fatalf("the standing verdict was not adopted over an ask with no id: %s", d.State) + } + if len(w.ctl.checks) != 0 || w.op.count() != 1 { + t.Fatalf("checks %v, Approves asked %d; wanted none and one", w.ctl.checks, w.op.count()) + } +} + +func TestAKeptAskWithNoIdAndNoStandingVerdictIsAskedAgainAtOnce(t *testing.T) { + w := newWorld(t) + w.h.PullUpdated(pr("novox/app", 7, readyHead, "feat/x", "src/a.go")) + id := IDOf("novox/app", readyHead) + w.h.mu.Lock() + w.h.deliveries[id].CheckAsk = &CheckAsk{At: w.now, By: byOwner} + w.h.mu.Unlock() + w.settleAll() + if d := w.delivery(id); d.CheckAsk == nil || d.CheckAsk.ID == "" || len(w.ctl.checks) != 1 { + t.Fatalf("not asked again: %+v %v", d.CheckAsk, w.ctl.checks) + } +} + +func TestAnEmptyAnswerToACheckAskIsARefusalAskedAtMostOnceAMinute(t *testing.T) { + w := newWorld(t) + w.ctl.emptyCheck = true + w.h.PullUpdated(pr("novox/app", 7, readyHead, "feat/x", "src/a.go")) + id := IDOf("novox/app", readyHead) + w.settleAll() + w.settleAll() + if len(w.ctl.checks) != 1 || w.delivery(id).CheckAsk != nil || len(w.forge.notesOn(readyHead)) > 1 { + t.Fatalf("asked %v, ask kept %v; wanted one ask and none kept", w.ctl.checks, w.delivery(id).CheckAsk) + } + w.later(time.Minute) + w.settleAll() + if len(w.ctl.checks) != 2 { + t.Fatalf("not asked again after the minute: %v", w.ctl.checks) + } +} diff --git a/modules/mesh-delivery/cmd/mesh-delivery/times.go b/modules/mesh-delivery/cmd/mesh-delivery/times.go new file mode 100644 index 00000000..f2ec2319 --- /dev/null +++ b/modules/mesh-delivery/cmd/mesh-delivery/times.go @@ -0,0 +1,373 @@ +package main + +import ( + "fmt" + "sort" + "strings" + "time" +) + +// Delivery time and delivery budgets (novox/hq ADR 0282 decisions 1 and 6, issue 382): a delivery's time runs +// from its merge to every machine of its walk's rest reporting the build applied. The controller works out the +// walk's phases (its `delivery-walks` and `plan-moved` say them); this owner keeps, on each delivery, its own +// delivery time from its own merge, its class and its phases, and `times` answers the week's median and worst +// per class against its budget. Measurement only: nothing here holds, releases or changes a delivery. + +// The budgets by class (ADR 0282 decision 1, the leaf's as ADR 0304 decision 4 changed it). +var budgets = map[string]time.Duration{"leaf": 3 * time.Minute, "core": 10 * time.Minute} + +// classes in the order `times` says them. +var classes = []string{"leaf", "core"} + +// WalkPhases is a walk's time phase by phase, as the controller says it (mesh-controller inventory.WalkPhases). +type WalkPhases struct { + Class string `json:"class,omitempty"` + From *time.Time `json:"from,omitempty"` + End *time.Time `json:"end,omitempty"` + TotalMS int64 `json:"total_ms,omitempty"` + UnknownMS int64 `json:"unknown_ms,omitempty"` + Phases []WalkPhase `json:"phases"` + Silent []string `json:"silent,omitempty"` + Said string `json:"said"` + Merges []MergeStart `json:"merges,omitempty"` +} + +// MergeStart is one merge a walk answers and when it was made. +type MergeStart struct { + Repository string `json:"repository"` + Commit string `json:"commit"` + Merged time.Time `json:"merged"` +} + +// WalkPhase is one phase of a walk: measured, unknown (never a zero), none (it did not happen) or open. +type WalkPhase struct { + Name string `json:"name"` + Tier int `json:"tier"` + State string `json:"state"` + Start *time.Time `json:"start,omitempty"` + End *time.Time `json:"end,omitempty"` + TookMS int64 `json:"took_ms,omitempty"` + Took string `json:"took,omitempty"` + Said string `json:"said,omitempty"` +} + +const ( + phaseMeasured = "measured" + phaseUnknown = "unknown" +) + +// DeliveryTime is a delivery's time, kept on it once its walk's end is known. +type DeliveryTime struct { + Walk string `json:"walk"` + Class string `json:"class"` + // From is its own merge; End when every machine of its walk's rest reported the build applied. + From time.Time `json:"from"` + End time.Time `json:"end"` + TookMS int64 `json:"took_ms"` + Took string `json:"took"` + BudgetMS int64 `json:"budget_ms,omitempty"` + Over bool `json:"over,omitempty"` + Longest string `json:"longest,omitempty"` + UnknownMS int64 `json:"unknown_ms,omitempty"` + Phases []WalkPhase `json:"phases"` + Silent []string `json:"silent,omitempty"` + // Unreliable says why its moments cannot be trusted (out of order, or its merge after its walk's end): its + // time is then no number — TookMS zero, never counted, said among the unknown by `times`. + Unreliable string `json:"unreliable,omitempty"` +} + +// timeOf is a delivery's time from its walk's phases; nil while the walk's end is not known. +// +// The walk's phases run back to back from its earliest merge to its end: each phase with an end covers the span +// from the end before it, measured or unknown. This delivery's time runs from its own merge, so every span is cut +// there: a phase that ended before its merge is said none ("before its merge"), the phase its merge falls in +// starts at its merge, and an unknown span counts only after it. A merge before the walk's earliest adds that wait +// as unknown time. The measured phases and the unknown time then add up to the delivery time exactly. +// +// Moments out of order (a phase ending before the one before it, or the delivery's merge after the walk's end) +// are a clock or a record that cannot be trusted: the time is kept as unreliable, with why, and no number is said. +func timeOf(d *Delivery, w Walk) *DeliveryTime { + p := w.Phases + if p == nil || p.End == nil || p.From == nil { + return nil + } + from := d.MergedAt + for _, m := range p.Merges { + if from.IsZero() && strings.EqualFold(m.Repository, d.Repository) && sameCommit(m.Commit, d.MergedAs) { + from = m.Merged + } + } + if from.IsZero() { + from = *p.From + } + from = from.Truncate(time.Millisecond) + t := &DeliveryTime{Walk: w.ID, Class: p.Class, From: from, End: *p.End, Silent: p.Silent} + unreliable := func(why string) *DeliveryTime { + // Its phases are named, never timed: a moment out of order makes every number in them suspect. + t.Unreliable, t.Phases = why, nil + for _, ph := range p.Phases { + t.Phases = append(t.Phases, WalkPhase{Name: ph.Name, Tier: ph.Tier, State: phaseUnknown, Said: "unreliable"}) + } + return t + } + if from.After(*p.End) { + return unreliable(fmt.Sprintf("its merge (%s) is after its walk's end (%s)", from.UTC().Format(time.RFC3339), + p.End.UTC().Format(time.RFC3339))) + } + var unknown int64 + if from.Before(*p.From) { + unknown += p.From.Sub(from).Milliseconds() // merged before the walk's earliest merge: waited, unmeasured + } + prev := *p.From + for _, ph := range p.Phases { + if ph.End == nil { + t.Phases = append(t.Phases, ph) + continue + } + end := *ph.End + if end.Before(prev) { + return unreliable(fmt.Sprintf("its walk's %s ends before the phase before it: its moments are out of order", ph.Name)) + } + start := prev + prev = end + if end.Before(from) { + ph.State, ph.Start, ph.TookMS, ph.Took, ph.Said = "none", nil, 0, "", "before its merge" + t.Phases = append(t.Phases, ph) + continue + } + if start.Before(from) { + start = from + } + span := end.Sub(start) + if ph.State == phaseMeasured { + s := start + ph.Start, ph.TookMS, ph.Took = &s, span.Milliseconds(), span.Round(100*time.Millisecond).String() + } else { + unknown += span.Milliseconds() + } + t.Phases = append(t.Phases, ph) + } + if !prev.Equal(*p.End) { + return unreliable("its walk's phases do not reach its end") + } + t.UnknownMS = unknown + took := t.End.Sub(from) + t.TookMS, t.Took = took.Milliseconds(), took.Round(time.Second).String() + if b, ok := budgets[t.Class]; ok { + t.BudgetMS, t.Over = b.Milliseconds(), took > b + } + t.Longest = longestOf(t.Phases) + return t +} + +// longestOf is the longest measured phase, as words: "judgement (tier 0) 2m29s". +func longestOf(phases []WalkPhase) string { + best := -1 + for i, ph := range phases { + if ph.State == phaseMeasured && (best < 0 || ph.TookMS > phases[best].TookMS) { + best = i + } + } + if best < 0 { + return "" + } + ph := phases[best] + name := ph.Name + if ph.Tier >= 0 { + name = fmt.Sprintf("%s (tier %d)", ph.Name, ph.Tier) + } + return name + " " + (time.Duration(ph.TookMS) * time.Millisecond).Round(time.Second).String() +} + +// timeWalk keeps, on every delivery the walk answers, its delivery time once the walk's end is known; a delivery +// whose time did not change is not written again (issue 326). +func (h *Holder) timeWalk(w Walk) { + if s, ok := h.walks[w.ID]; ok && s.walk.Revision > w.Revision { + w = s.walk // an older record arriving late: the newest says the time + } + if w.Phases == nil || w.Phases.End == nil { + return + } + for _, d := range h.deliveries { + if d.Walk == nil || d.Walk.ID != w.ID { + continue + } + t := timeOf(d, w) + if t == nil || (d.Time != nil && d.Time.TookMS == t.TookMS && d.Time.Unreliable == t.Unreliable && d.Time.End.Equal(t.End) && d.Time.Walk == t.Walk && + d.Time.Class == t.Class && d.Time.UnknownMS == t.UnknownMS && d.Time.Longest == t.Longest) { + continue + } + d.Time = t + h.keep(d) + } +} + +// TimesClass is one class's line of `times`. +type TimesClass struct { + Class string `json:"class"` + Budget string `json:"budget"` + BudgetMS int64 `json:"budget_ms"` + Count int `json:"count"` + Median string `json:"median,omitempty"` + MedianMS int64 `json:"median_ms,omitempty"` + Worst string `json:"worst,omitempty"` + WorstMS int64 `json:"worst_ms,omitempty"` + WorstID string `json:"worst_id,omitempty"` + Within int `json:"within"` + Over int `json:"over"` + Latest *TimesLatest `json:"latest,omitempty"` +} + +// TimesLatest is a delivery as `times` names it: the newest of a class, or one over its budget. +type TimesLatest struct { + ID string `json:"id"` + Class string `json:"class,omitempty"` + TookMS int64 `json:"took_ms"` + Took string `json:"took"` + Over bool `json:"over,omitempty"` + Longest string `json:"longest,omitempty"` + Landed string `json:"landed"` +} + +// TimesAnswer is `times`. +type TimesAnswer struct { + Days int `json:"days"` + Line string `json:"line"` + Classes []TimesClass `json:"classes"` + Over []TimesLatest `json:"over,omitempty"` + // Unknown are the deliveries delivered in the window whose time is not known, and why it is not: at most + // unknownShown of them, UnknownCount all. + Unknown []string `json:"unknown,omitempty"` + UnknownCount int `json:"unknown_count,omitempty"` +} + +// unknownShown is how many deliveries with no known time `times` names. +const unknownShown = 20 + +// Times is the `times` verb: each class's budget, the last days' median and worst delivery time, and every +// delivery over its budget with its longest phase; or, given a delivery's id, its time phase by phase. +func (h *Holder) Times(days int, id string) (any, error) { + h.mu.Lock() + defer h.mu.Unlock() + if id != "" { + d := h.deliveries[id] + if d == nil { + return nil, fmt.Errorf("no delivery %q: `deliveries` lists them", id) + } + if d.Time == nil { + why := "its walk's end is not known yet" + if d.Walk == nil { + why = "no walk answers it" + } else if s, ok := h.walks[d.Walk.ID]; ok && s.walk.Phases != nil { + why = s.walk.Phases.Said + } + return map[string]any{"delivery": id, "state": d.State, "time": nil, "why": why}, nil + } + return map[string]any{"delivery": id, "state": d.State, "time": d.Time}, nil + } + return h.times(days), nil +} + +// times is the answer for the last days, under the lock. +func (h *Holder) times(days int) TimesAnswer { + if days <= 0 { + days = 7 + } + since := h.Now().Add(-time.Duration(days) * 24 * time.Hour) + a := TimesAnswer{Days: days} + took := map[string][]*Delivery{} + for _, d := range h.deliveries { + if d.Time == nil { + if (d.State == Delivered || d.State == Covered) && d.Since.After(since) { + why := "its walk's end is not known" + if d.Walk != nil { + if s, ok := h.walks[d.Walk.ID]; ok && s.walk.Phases != nil && s.walk.Phases.Said != "" { + why = s.walk.Phases.Said + } + } + a.Unknown = append(a.Unknown, d.ID+": "+why) + } + continue + } + if d.Time.End.Before(since) { + continue + } + if d.Time.Unreliable != "" { + a.Unknown = append(a.Unknown, d.ID+": its time is unreliable: "+d.Time.Unreliable) + continue + } + if _, ok := budgets[d.Time.Class]; !ok { + a.Unknown = append(a.Unknown, fmt.Sprintf("%s: took %s, its class %q has no budget", d.ID, d.Time.Took, d.Time.Class)) + continue + } + took[d.Time.Class] = append(took[d.Time.Class], d) + } + sort.Strings(a.Unknown) + a.UnknownCount = len(a.Unknown) + if len(a.Unknown) > unknownShown { + a.Unknown = a.Unknown[:unknownShown] + } + var line []string + for _, class := range classes { + c := TimesClass{Class: class, Budget: budgets[class].String(), BudgetMS: budgets[class].Milliseconds()} + ds := took[class] + sort.Slice(ds, func(i, j int) bool { return ds[i].Time.TookMS < ds[j].Time.TookMS }) + c.Count = len(ds) + for _, d := range ds { + if d.Time.Over { + c.Over++ + a.Over = append(a.Over, latestOf(d)) + } else { + c.Within++ + } + if c.Latest == nil || d.Time.End.After(mustLanded(c.Latest)) { + l := latestOf(d) + c.Latest = &l + } + } + if len(ds) > 0 { + med := median(ds) + c.MedianMS, c.Median = med.Milliseconds(), med.Round(time.Second).String() + worst := ds[len(ds)-1] + c.WorstMS, c.Worst, c.WorstID = worst.Time.TookMS, worst.Time.Took, worst.ID + line = append(line, fmt.Sprintf("%s: median %s, worst %s, %d of %d within %s", class, c.Median, c.Worst, + c.Within, c.Count, c.Budget)) + } else { + line = append(line, fmt.Sprintf("%s: no delivery with a known time", class)) + } + a.Classes = append(a.Classes, c) + } + sort.Slice(a.Over, func(i, j int) bool { return a.Over[i].Landed > a.Over[j].Landed }) + a.Line = fmt.Sprintf("delivery time over %d day(s) — %s", days, strings.Join(line, "; ")) + if a.UnknownCount > 0 { + a.Line += fmt.Sprintf("; %d delivered with no known time", a.UnknownCount) + } + return a +} + +func latestOf(d *Delivery) TimesLatest { + return TimesLatest{ID: d.ID, Class: d.Time.Class, TookMS: d.Time.TookMS, Took: d.Time.Took, Over: d.Time.Over, + Longest: d.Time.Longest, Landed: d.Time.End.UTC().Format(time.RFC3339)} +} + +func mustLanded(l *TimesLatest) time.Time { + t, _ := time.Parse(time.RFC3339, l.Landed) + return t +} + +// median of deliveries sorted by their time. +func median(ds []*Delivery) time.Duration { + n := len(ds) + ms := ds[n/2].Time.TookMS + if n%2 == 0 { + ms = (ds[n/2-1].Time.TookMS + ds[n/2].Time.TookMS) / 2 + } + return time.Duration(ms) * time.Millisecond +} + +// TimesLine is the one line `deliveries` heads its list with. +func (h *Holder) TimesLine() string { + h.mu.Lock() + defer h.mu.Unlock() + return h.times(7).Line +} diff --git a/modules/mesh-delivery/cmd/mesh-delivery/times_test.go b/modules/mesh-delivery/cmd/mesh-delivery/times_test.go new file mode 100644 index 00000000..ddf39756 --- /dev/null +++ b/modules/mesh-delivery/cmd/mesh-delivery/times_test.go @@ -0,0 +1,261 @@ +package main + +import ( + "strings" + "testing" + "time" +) + +// novox/hq ADR 0282 (issue 382): a delivery keeps its delivery time, class and phases from its walk, from its +// own merge; `times` answers each class's median and worst against its budget. Measurement only. + +func ms(t time.Time) *time.Time { return &t } + +// phasesOf is a walk's phases as the controller says them: its earliest merge at from, then each phase of the +// given lengths one after the other, the last ending the walk. +func phasesOf(class string, from time.Time, lengths ...struct { + name string + d time.Duration +}) *WalkPhases { + p := &WalkPhases{Class: class, From: ms(from)} + at := from + for _, l := range lengths { + start := at + at = at.Add(l.d) + p.Phases = append(p.Phases, WalkPhase{Name: l.name, Tier: -1, State: phaseMeasured, Start: ms(start), End: ms(at), + TookMS: l.d.Milliseconds()}) + } + p.End = ms(at) + p.TotalMS = at.Sub(from).Milliseconds() + return p +} + +type ph = struct { + name string + d time.Duration +} + +// walkedOnce is a merge walked to the end, the walk's phases given: its delivery's id. +func walkedOnce(w *world, number int, head, merge string, phases *WalkPhases) string { + w.t.Helper() + repo := "novox/mesh-catalog" + p := pr(repo, number, head, "feat/x") + w.h.PullUpdated(p) + w.h.Checked(verdictMoving(repo, number, head, "app")) + w.h.PullMerged(mergedAt(p, merge, phases.From.Add(10*time.Second))) + walk := aWalk("plan-"+merge[:6], repo, merge, false) + walk.Delivery = &WalkDelivery{Merges: []WalkMerge{{Repository: repo, Commit: merge}}} + w.h.WalkMoved(w.ctl.put(walk)) + w.settleAll() + walk = w.ctl.walk(walk.ID) + walk.State, walk.Revision = walkDone, walk.Revision+1 + walk.Modules = map[string]*WalkModule{"app": {State: "built", SentAt: phases.End}} + walk.Phases = phases + w.h.WalkMoved(w.ctl.put(walk)) + w.settleAll() + return IDOf(repo, head) +} + +// A delivery's time runs from its own merge — ten seconds after the walk's earliest — to its walk's end, and its +// phases still add up to it. +func TestADeliveryKeepsItsTimeFromItsOwnMerge(t *testing.T) { + w := newWorld(t) + from := t0.Add(time.Minute) + id := walkedOnce(w, 7, "head7", "merge7aaaa", phasesOf("leaf", from, ph{"window", 90 * time.Second}, + ph{"queued", 5 * time.Second}, ph{"build", 25 * time.Second}, ph{"judgement", 150 * time.Second}, + ph{"apply", 20 * time.Second})) + d := w.delivery(id) + if d.Time == nil { + t.Fatalf("no delivery time kept: %+v", d) + } + if d.Time.TookMS != (280*time.Second).Milliseconds() || d.Time.Class != "leaf" || !d.Time.Over { + t.Fatalf("time %dms (want 280000, from its own merge, over a leaf's three minutes), class %s, over %v", d.Time.TookMS, d.Time.Class, d.Time.Over) + } + var sum int64 + for _, p := range d.Time.Phases { + if p.State == phaseMeasured { + sum += p.TookMS + } + } + if sum+d.Time.UnknownMS != d.Time.TookMS { + t.Fatalf("the phases add up to %dms + %dms unknown, its time is %dms", sum, d.Time.UnknownMS, d.Time.TookMS) + } + if d.Time.Phases[0].TookMS != 80000 || !strings.HasPrefix(d.Time.Longest, "judgement") { + t.Fatalf("its window starts at its own merge (80s), longest judgement: %+v %s", d.Time.Phases[0], d.Time.Longest) + } + // Its own merge after a first phase whose start is unknown: the unknown time moves, the sum holds. + p := phasesOf("leaf", from, ph{"window", 90 * time.Second}, ph{"build", 30 * time.Second}) + p.Phases[0] = WalkPhase{Name: "window", Tier: -1, State: phaseUnknown, End: p.Phases[0].End} + p.UnknownMS = 90000 + dt := timeOf(&Delivery{MergedAt: from.Add(10 * time.Second)}, Walk{ID: "w", Phases: p}) + if dt.UnknownMS != 80000 || dt.TookMS != 110000 { + t.Fatalf("unknown first phase: %+v", dt) + } + // Merged after the walk's first phase ended (a later merge carried by the walk, its window already running): that + // phase is before its merge, the next starts at its merge, and nothing is negative. + p = phasesOf("leaf", from, ph{"window", 30 * time.Second}, ph{"queued", 60 * time.Second}, ph{"build", 30 * time.Second}) + p.Phases[0].State, p.Phases[0].Start, p.Phases[0].TookMS = phaseUnknown, nil, 0 + dt = timeOf(&Delivery{MergedAt: from.Add(45 * time.Second)}, Walk{ID: "w", Phases: p}) + if dt.Unreliable != "" || dt.UnknownMS != 0 || dt.TookMS != 75000 || dt.Phases[0].State != "none" || + dt.Phases[1].TookMS != 45000 || dt.Phases[2].TookMS != 30000 { + t.Fatalf("merged inside the second phase: %+v", dt) + } + assertSums(t, dt) + // Merged before the walk's earliest merge: the wait is unknown time, and the sum holds. + dt = timeOf(&Delivery{MergedAt: from.Add(-20 * time.Second)}, Walk{ID: "w", Phases: p}) + if dt.UnknownMS != 50000 || dt.TookMS != 140000 { + t.Fatalf("merged before the walk's first merge: %+v", dt) + } + assertSums(t, dt) + // Moments out of order: unreliable, and no number. + bad := phasesOf("leaf", from, ph{"window", 30 * time.Second}, ph{"build", 30 * time.Second}) + early := from.Add(10 * time.Second) + bad.Phases[1].End = &early + bad.End = &early + dt = timeOf(&Delivery{MergedAt: from}, Walk{ID: "w", Phases: bad}) + if dt.Unreliable == "" || dt.TookMS != 0 { + t.Fatalf("out of order is unreliable, with no time: %+v", dt) + } + for _, ph := range dt.Phases { + if ph.TookMS != 0 || ph.Start != nil || ph.State != phaseUnknown { + t.Fatalf("an unreliable time says no phase's time: %+v", ph) + } + } + // The walk's phases not reaching its end: unreliable. + short := phasesOf("leaf", from, ph{"window", 30 * time.Second}) + later := from.Add(time.Minute) + short.End = &later + if dt := timeOf(&Delivery{MergedAt: from}, Walk{ID: "w", Phases: short}); dt.Unreliable == "" || dt.TookMS != 0 { + t.Fatalf("phases short of the walk's end: %+v", dt) + } + // Merged after the walk's end: unreliable. + if dt := timeOf(&Delivery{MergedAt: from.Add(time.Hour)}, Walk{ID: "w", Phases: p}); dt.Unreliable == "" || dt.TookMS != 0 { + t.Fatalf("merged after its walk's end: %+v", dt) + } + // Exactly its budget is within it. + p = phasesOf("leaf", from, ph{"build", 3*time.Minute + 10*time.Second}) + if dt := timeOf(&Delivery{MergedAt: from.Add(10 * time.Second)}, Walk{ID: "w", Phases: p}); dt.Over || dt.TookMS != 180000 { + t.Fatalf("three minutes is within a leaf's budget: %+v", dt) + } + p = phasesOf("leaf", from, ph{"build", 3*time.Minute + 11*time.Second}) + if dt := timeOf(&Delivery{MergedAt: from.Add(10 * time.Second)}, Walk{ID: "w", Phases: p}); !dt.Over { + t.Fatalf("a second past three minutes is over a leaf's budget: %+v", dt) + } + // Seen again, unchanged: not written again (issue 326). + before := w.store.puts + w.h.Reconcile() + w.settleAll() + if w.store.puts != before { + t.Fatalf("a delivery whose time did not change was written again") + } +} + +// A walk whose end is not known gives no time; a phase unknown in the walk stays unknown, never a zero. +func TestNoEndNoTime(t *testing.T) { + w := newWorld(t) + p := phasesOf("core", t0, ph{"window", time.Minute}, ph{"build", time.Minute}) + p.End, p.TotalMS = nil, 0 + p.Phases = append(p.Phases, WalkPhase{Name: "apply", Tier: -1, State: phaseUnknown, Said: "waiting for ace"}) + p.Said = "its end is unknown: apply (waiting for ace)" + id := walkedOnce(w, 8, "head8", "merge8bbbb", p) + if d := w.delivery(id); d.Time != nil { + t.Fatalf("a time without an end: %+v", d.Time) + } + got, err := w.h.Times(0, id) + if err != nil || !strings.Contains(mustString(got), "waiting for ace") { + t.Fatalf("the delivery's time says why it is not known: %v %v", got, err) + } + a := w.h.times(7) + if len(a.Unknown) != 1 || !strings.Contains(a.Unknown[0], "waiting for ace") { + t.Fatalf("a delivered delivery with no known time is said: %+v", a.Unknown) + } +} + +// `times`: each class's median, worst, within and over its budget, its newest delivery, and every delivery over. +func TestTimesSaysEachClassAgainstItsBudget(t *testing.T) { + w := newWorld(t) + leaf := func(n int, d time.Duration, at time.Duration) string { + return walkedOnce(w, n, "head"+string(rune('a'+n)), "merge"+string(rune('a'+n))+"zzzz", + phasesOf("leaf", t0.Add(at), ph{"window", 10 * time.Second}, ph{"judgement", d})) + } + leaf(1, 1*time.Minute, 0) + leaf(2, 2*time.Minute, time.Hour) + over := leaf(3, 6*time.Minute, 2*time.Hour) + walkedOnce(w, 4, "headcore", "mergekore1", phasesOf("core", t0.Add(3*time.Hour), ph{"judgement", 9 * time.Minute})) + w.later(4 * time.Hour) + a := w.h.times(7) + if len(a.Classes) != 2 || a.Classes[0].Class != "leaf" { + t.Fatalf("classes: %+v", a.Classes) + } + l := a.Classes[0] + // Each delivery's time is from its own merge, ten seconds after the walk's: 1m, 2m, 6m. + if l.Count != 3 || l.Within != 2 || l.Over != 1 || l.MedianMS != (2*time.Minute).Milliseconds() || + l.WorstMS != (6*time.Minute).Milliseconds() || l.WorstID != over { + t.Fatalf("leaf: %+v", l) + } + if l.Latest == nil || l.Latest.ID != over || !l.Latest.Over || !strings.HasPrefix(l.Latest.Longest, "judgement") { + t.Fatalf("the newest leaf delivery: %+v", l.Latest) + } + if c := a.Classes[1]; c.Count != 1 || c.Over != 0 || c.Latest == nil || c.Latest.Over { + t.Fatalf("core within ten minutes: %+v", c) + } + if len(a.Over) != 1 || a.Over[0].ID != over { + t.Fatalf("over: %+v", a.Over) + } + if !strings.Contains(a.Line, "leaf: median 2m0s, worst 6m0s, 2 of 3 within 3m0s") { + t.Fatalf("the line: %s", a.Line) + } + // Past the window, nothing. + w.later(8 * 24 * time.Hour) + if a := w.h.times(7); a.Classes[0].Count != 0 || a.Classes[0].Latest != nil { + t.Fatalf("deliveries older than the window are not counted: %+v", a.Classes[0]) + } +} + +func mustString(v any) string { return string(mustJSON(v)) } + +// assertSums checks a delivery time's measured phases and unknown time add up to it, and none is negative. +func assertSums(t *testing.T, dt *DeliveryTime) { + t.Helper() + sum := dt.UnknownMS + if dt.UnknownMS < 0 { + t.Fatalf("negative unknown time: %+v", dt) + } + for _, p := range dt.Phases { + if p.TookMS < 0 { + t.Fatalf("negative phase %s: %+v", p.Name, p) + } + if p.State == phaseMeasured { + sum += p.TookMS + } + } + if sum != dt.TookMS { + t.Fatalf("the phases add up to %dms, the delivery time is %dms: %+v", sum, dt.TookMS, dt) + } +} + +// An unreliable time is said among the unknown, never in a median, an over count or the newest of its class. +func TestAnUnreliableTimeIsNeverCounted(t *testing.T) { + w := newWorld(t) + good := walkedOnce(w, 11, "headgood", "mergegood1", phasesOf("leaf", t0, ph{"build", 2 * time.Minute})) + bad := phasesOf("leaf", t0.Add(time.Hour), ph{"window", 30 * time.Second}, ph{"build", 9 * time.Minute}) + early := t0.Add(time.Hour + 10*time.Second) + bad.Phases[1].End, bad.End = &early, &early + unreliable := walkedOnce(w, 12, "headbad", "mergebad01", bad) + if d := w.delivery(unreliable); d.Time == nil || d.Time.Unreliable == "" { + t.Fatalf("the out-of-order walk gave a time: %+v", d.Time) + } + w.later(2 * time.Hour) + a := w.h.times(7) + l := a.Classes[0] + if l.Count != 1 || l.Over != 0 || l.Latest == nil || l.Latest.ID != good || l.WorstID != good { + t.Fatalf("an unreliable time was counted: %+v", l) + } + found := false + for _, u := range a.Unknown { + found = found || strings.HasPrefix(u, unreliable+": its time is unreliable") + } + if !found { + t.Fatalf("the unreliable delivery is not said among the unknown: %v", a.Unknown) + } +} diff --git a/modules/mesh-delivery/cmd/mesh-delivery/unchecked.go b/modules/mesh-delivery/cmd/mesh-delivery/unchecked.go new file mode 100644 index 00000000..bbcbf204 --- /dev/null +++ b/modules/mesh-delivery/cmd/mesh-delivery/unchecked.go @@ -0,0 +1,305 @@ +package main + +import ( + "fmt" + "slices" + "strings" + "time" +) + +// A pull request the mesh was never asked to check (novox/hq issue 347). +// +// The forge's announcer is the only way the mesh hears of a pull request's head, and on 2026-10-09 it missed one: a +// pull request opened after its push moved nothing the announcer read, and waited unchecked, unable to merge, with +// nothing saying so. This owner asks the forge itself for every open pull request when it starts and when an event says +// what the forge shows may have changed — a merge check's status, a new head (events.go), not a poll +// (novox/hq ADR 0406 §1, issue 975) — and once more at a head's bound, and one whose +// base requires the merge check — mesh/merge-gate or mesh/repo-check — with neither on its head for UncheckedAfter +// since this owner first saw it so is said as a delivery held: a `stalled` line in the state Unannounced, which the +// controller raises as `delivery..stalled` (D14) like any other delivery past its bound. The delivery domain's own +// word, so no second condition and no second judge: the id is the one its delivery will have once announced. +// +// A head whose check started and never answered is said the same way (novox/hq issue 438). Counting any merge-check +// status as checked let a head that got only a pending mesh/merge-gate, or never got mesh/repo-check, wait unable to +// merge with nothing saying so. So a head is checked only when every merge check its base requires has answered — +// success, failure, error or warning, a failure being a verdict the pull request shows itself — and one with a +// required check pending or never set is said once that has lasted the delivery table's bound for a check asked and +// not answered (Bounds[Proposed]): an announced head would be said after that bound too, and it is longer than the +// controller lets one check run (its checkTimeout, 45 minutes), so a slow check or a status read a moment behind +// (issue 423) is never said. Such a head was announced — the controller set its status — so its line is not in the +// state Unannounced but Unanswered, with words of its own. +// +// A head whose delivery is held in ready or rejected is said by the same rule (novox/hq issue 441). Those states have +// no bound of their own — the verdict is known here — but the forge's status is what lets the pull request merge or +// shows why not, and a status write that lagged or was lost (issue 423), or a check asked again after the verdict, +// leaves a required check pending or never set there: the pull request waits on it, and nothing here asks again. So +// such a head is kept here like one without a delivery, and said on the same bound, as Unanswered — it was announced, +// so never Unannounced, even with no merge check on its head at all. A delivery in any other state is not: proposed +// and checked wait under bounds of their own, and the rest are past the merge, their pull request no longer open. + +// Unannounced is a pull request's head the forge never announced: not a state of the table — no delivery exists — +// only what a stalled line says. +const Unannounced State = "unannounced" + +// Unanswered is a pull request's head whose merge check started and never answered, and no delivery of it is held +// here (novox/hq issue 438): not a state of the table either, only what a stalled line says. The word is the table's +// own for the proposed state's bound: its check was asked and not answered. +const Unanswered State = "unanswered" + +// outOfDatePrefix begins the words the forge's holder puts a merge check back to pending with when its trunk moved +// (novox/hq ADR 0323 §4): the gitea module's outOfDateWords. +const outOfDatePrefix = "out of date:" + +// UncheckedAfter is how long a head may go without a merge check before it is said: the announcer passes every half +// minute and the check is asked at once, so ten minutes is a fault, never a queue. +const UncheckedAfter = 10 * time.Minute + +// mergeChecks are the statuses that are the mesh's merge check (novox/hq ADR 0237): a base requiring either is +// one the mesh is meant to check. +var mergeChecks = []string{"mesh/merge-gate", "mesh/repo-check"} + +// OpenPull is an open pull request as the forge says it (the gitea module's gitea_open_pulls): its head, its base, +// the contexts the base's protection requires, the contexts its head has a status of, in any state, and each of +// those statuses with its state. +type OpenPull struct { + Owner string `json:"owner"` + Repo string `json:"repo"` + Number int `json:"number"` + Base string `json:"base"` + HeadSHA string `json:"head_sha"` + HTMLURL string `json:"html_url,omitempty"` + Required []string `json:"required"` + Statuses []string `json:"statuses"` + // Checks are the head's statuses by context, the newest of each (novox/hq issue 438). A forge module older + // than that says none: the head is then read by Statuses alone, as before, and a pending check is not told + // from an answered one. + Checks map[string]HeadCheck `json:"checks,omitempty"` +} + +// HeadCheck is one status on a head as the forge keeps it: pending, success, failure, error or warning. +type HeadCheck struct { + State string `json:"state"` + Description string `json:"description,omitempty"` + UpdatedAt string `json:"updated_at,omitempty"` +} + +// unchecked is a head seen without an answered merge check, what it lacks, and when this owner first saw it so — +// its own time, since the forge's update time moves on every comment. +type unchecked struct { + pull OpenPull + waiting []string + // pending is when each required check that is pending was set so, as the forge says it: what tells a check + // asked again from one still waiting on the same ask. + pending map[string]string + first time.Time + // read is when the forge was read that showed it so. + read time.Time + // held is the state of the delivery of this head, ready or rejected, when one is held here (novox/hq issue + // 441); empty for a head with no delivery. + held State +} + +// saidByTheForge is whether a delivery's head is said here when its required check has not answered on the forge: +// one in ready or rejected, whose state has no bound of its own (novox/hq issue 441). No delivery at all is too. +func saidByTheForge(d *Delivery) bool { + return d == nil || d.State == Ready || d.State == Rejected +} + +// unanswered are the merge checks the base requires that the head has no verdict of, each as it reads — "never +// set", or pending and since when — when each pending one was set so, and whether the head has any merge check at +// all: its check started. A head on a base that requires none, or without a head, has nothing unanswered. +func (p OpenPull) unanswered() (waiting []string, pending map[string]string, started bool) { + if p.HeadSHA == "" { + return nil, nil, false + } + pending = map[string]string{} + started = slices.ContainsFunc(mergeChecks, func(c string) bool { + return slices.Contains(p.Statuses, c) || p.Checks[c].State != "" + }) + for _, c := range mergeChecks { + if !slices.Contains(p.Required, c) { + continue + } + got, ok := p.Checks[c] + switch { + case p.Checks == nil && started: + // A forge that says no states: a head with any merge check counts as checked, as before issue 438, + // since nothing here says whether the one it has answered. + case !ok || got.State == "": + waiting = append(waiting, c+" never set") + case got.State == "pending" && strings.HasPrefix(got.Description, outOfDatePrefix): + // Out of date because its trunk moved, and not asked (novox/hq ADR 0323 §4, §7): nothing runs for it and + // it costs nothing, so it is not a check asked and not answered. Asked again, it is pending anew. + case got.State == "pending": + pending[c] = utcOf(got.UpdatedAt) + waiting = append(waiting, c+" pending since "+orUnknown(pending[c])) + } + } + return waiting, pending, started +} + +// utcOf is a forge time in UTC, as this module's other lines give their times; one it cannot read stays as given. +func utcOf(at string) string { + t, err := time.Parse(time.RFC3339, at) + if err != nil { + return at + } + return t.UTC().Format(time.RFC3339) +} + +func orUnknown(at string) string { + if at == "" { + return "a time the forge did not say" + } + return at +} + +// askedAgain is whether a required check is pending on a newer ask than when this owner last saw the head: set +// pending for the first time, set pending anew, or gone back to pending after it answered. A check that answers +// never is: the head waits on what is left from the same first sight (novox/hq issue 438). +func askedAgain(was, now map[string]string) bool { + for c, at := range now { + if before, ok := was[c]; !ok || before != at { + return true + } + } + return false +} + +// SeeOpenPulls asks the forge for every open pull request and keeps the heads with a required merge check not +// answered. A forge that cannot be asked changes nothing: what was seen stands, and nothing is guessed. +func (h *Holder) SeeOpenPulls() error { + open, err := h.Forge.OpenPulls() + if err != nil { + h.Logf("[mesh-delivery] the forge's open pull requests cannot be read: %v", err) + return err + } + h.mu.Lock() + defer h.mu.Unlock() + now := h.Now() + seen := map[string]unchecked{} + for _, p := range open { + waiting, pending, _ := p.unanswered() + if len(waiting) == 0 { + continue + } + id := IDOf(p.Owner+"/"+p.Repo, p.HeadSHA) + d := h.deliveries[id] + if !saidByTheForge(d) { + continue // a delivery with its own bound, or past its merge + } + var held State + if d != nil { + held = d.State + } + // The clock runs from when this owner first saw the head waiting, and starts again only on a newer ask: a + // check that started, or was asked again — its pending status set anew — is given its whole bound from then. + // A check that answers does not start it again, so a head is said on the bound from its first sight. + u := unchecked{pull: p, waiting: waiting, pending: pending, first: now, read: now, held: held} + if was, ok := h.unchecked[id]; ok && !askedAgain(was.pending, pending) { + u.first = was.first + } else { + h.Logf("[mesh-delivery] %s/%s#%d's head %s %s; said if so in %s", p.Owner, p.Repo, p.Number, + short(p.HeadSHA), u.what(), u.bound()) + } + seen[id] = u + } + h.unchecked = seen + return nil +} + +// started is whether the head's check started, rather than was never asked: it has any merge check, or a delivery +// of it is held here, which the forge announced and the controller checked (novox/hq issue 441). +func (u unchecked) started() bool { + _, _, started := u.pull.unanswered() + return started || u.held != "" +} + +// bound is how long the head may wait as it does: UncheckedAfter for a check never asked, the table's bound for a +// check asked and not answered for one that started (novox/hq issue 438). +func (u unchecked) bound() time.Duration { + if u.started() { + return Bounds[Proposed].For + } + return UncheckedAfter +} + +// what is what the head lacks, in words. +func (u unchecked) what() string { + if u.started() { + return "has a merge check that has not answered: " + strings.Join(u.waiting, ", ") + } + return "has no merge check" +} + +// uncheckedLines are the heads unchecked past their bound, as stalled lines; under the lock. +func (h *Holder) uncheckedLines(now time.Time) []StalledLine { + var out []StalledLine + for id, u := range h.unchecked { + d := h.deliveries[id] + if !saidByTheForge(d) || (d != nil && d.State != u.held) || now.Sub(u.first) < u.bound() { + continue // moved on since the forge was last read: the next reading says it anew + } + h2 := "none: the forge never announced it, so there is no delivery to close — the operator's" + says := fmt.Sprintf("%s/%s#%d is open on %s, which requires the merge check, and its head has had no merge "+ + "check: the mesh was never asked to check it. A new commit on its branch announces it", + u.pull.Owner, u.pull.Repo, u.pull.Number, u.pull.Base) + state, waiting := Unannounced, []string(nil) + if u.started() { + state, waiting = Unanswered, u.waiting + says = fmt.Sprintf("%s/%s#%d is open on %s, which requires the merge check, and its head's check started "+ + "and never answered: %s. No delivery of it is held here, so nothing asks it again. A new commit on "+ + "its branch announces it and asks its check", u.pull.Owner, u.pull.Repo, u.pull.Number, u.pull.Base, + strings.Join(u.waiting, ", ")) + h2 = "none: no delivery of it is held here, so there is none to close — the operator's" + } + if u.held != "" { + // A delivery is held, its verdict known, and the forge shows another (novox/hq issue 441). + says = fmt.Sprintf("%s/%s#%d is open on %s, which requires the merge check, and its delivery is %s here, "+ + "but on the forge its check has not answered: %s. The pull request waits on that status and nothing "+ + "asks it again. A recheck of the delivery asks its check again, as does a new commit on its branch", + u.pull.Owner, u.pull.Repo, u.pull.Number, u.pull.Base, u.held, strings.Join(u.waiting, ", ")) + h2 = "none: its delivery is " + string(u.held) + ", a state the operator moves — the operator's" + } + out = append(out, StalledLine{ID: id, Number: u.pull.Number, State: state, For: now.Sub(u.first).Round(time.Second).String(), + Bound: u.bound().String(), Held: u.held, + H2: h2, Says: says, Waiting: waiting}) + } + return out +} + +func short(sha string) string { + if len(sha) > 12 { + return sha[:12] + } + return sha +} + +// uncheckedDue is whether a head seen waiting has reached the bound it is said at without the forge being read since: it +// is read once more then, to say it only if it still waits. A head not yet at its bound is looked at again when it is. +// Where the poll used to find a status that never came, an event does not exist to find it; this read is bounded to one +// per head and its bound. Called by React. +func (h *Holder) uncheckedDue() bool { + h.mu.Lock() + defer h.mu.Unlock() + now, due := h.Now(), false + for _, u := range h.unchecked { + at := u.first.Add(u.bound()) + switch { + case now.Before(at): + h.lookAt(at, "a head whose merge check has not answered, read from the forge once more at its bound") + case u.read.Before(at): + due = true + } + } + return due +} + +// forgetUnchecked drops the heads seen waiting of a pull request that closed or merged. +func (h *Holder) forgetUnchecked(owner, repo string, number int) { + for id, u := range h.unchecked { + if strings.EqualFold(u.pull.Owner, owner) && strings.EqualFold(u.pull.Repo, repo) && u.pull.Number == number { + delete(h.unchecked, id) + } + } +} diff --git a/modules/mesh-delivery/cmd/mesh-delivery/unchecked_events_test.go b/modules/mesh-delivery/cmd/mesh-delivery/unchecked_events_test.go new file mode 100644 index 00000000..b2999e7c --- /dev/null +++ b/modules/mesh-delivery/cmd/mesh-delivery/unchecked_events_test.go @@ -0,0 +1,95 @@ +package main + +import ( + "testing" + "time" +) + +// **A head nobody checks is found from events, not from a poll** (novox/hq ADR 0406 §1, issue 975, issues 347 and 722): +// the forge's open pull requests are read when this owner starts and when an event says something they show may have +// changed — a merge check's status, a new head, a ready mark — and once more at the bound a head's check is said. + +func statusEvent(sha, context, state string) ForgeStatusEvent { + return ForgeStatusEvent{Owner: "novox", Repo: "hq", SHA: sha, Context: context, State: state} +} + +func TestAMergeCheckStatusReadsTheOpenPullRequestsAndAnotherContextDoesNot(t *testing.T) { + w := newWorld(t) + before := w.forge.openCalls + hear(t, w, "gitea.forge.status", statusEvent("cccc", "mesh/delivery", "success")) + if w.forge.openCalls != before { + t.Fatalf("a status of a context that is no merge check read the forge %d times", w.forge.openCalls-before) + } + hear(t, w, "gitea.forge.status", statusEvent("cccc", "mesh/merge-gate", "pending")) + if w.forge.openCalls != before+1 { + t.Fatalf("a merge check's status read the forge %d times, wanted once", w.forge.openCalls-before) + } +} + +func TestAHeadWhoseCheckNeverAnsweredIsSaidFromTheEventsAndOneReadAtTheBound(t *testing.T) { + w := newWorld(t) + w.forge.open = []OpenPull{{Owner: "novox", Repo: "hq", Number: 1, Base: "main", HeadSHA: "cccc", + Required: []string{"mesh/merge-gate"}, Statuses: []string{"mesh/merge-gate"}, + Checks: map[string]HeadCheck{"mesh/merge-gate": {State: "pending", UpdatedAt: "2026-10-06T12:00:00Z"}}}} + hear(t, w, "gitea.forge.status", statusEvent("cccc", "mesh/merge-gate", "pending")) + if len(w.h.Stalled()) != 0 { + t.Fatal("a head just seen waiting is said already") + } + reads := w.forge.openCalls + // Minutes pass with no event: nothing polls. At the bound, one read confirms it, and it is said. + for range 3 { + w.later(5 * time.Minute) + w.h.React() + } + w.later(Bounds[Proposed].For) + w.h.React() + w.h.React() + if got := w.forge.openCalls - reads; got != 1 { + t.Fatalf("the forge's open pull requests were read %d times between the status and the bound's end; wanted once, at the bound", got) + } + if lines := w.h.Stalled(); len(lines) != 1 || lines[0].State != Unanswered { + t.Fatalf("the head whose check never answered is not said: %+v", lines) + } +} + +func TestANewlyAnnouncedHeadReadsTheForgeAndAKnownOneDoesNot(t *testing.T) { + w := newWorld(t) + before := w.forge.openCalls + hear(t, w, "gitea.pull.updated", pr("novox/app", 7, readyHead, "feat/x", "src/a.go")) + if w.forge.openCalls != before+2 { // the unchecked heads, and the head's standing verdict before its check is asked (issue 1115) + t.Fatalf("a new head read the forge %d times, wanted twice", w.forge.openCalls-before) + } + hear(t, w, "gitea.pull.updated", pr("novox/app", 7, readyHead, "feat/x", "src/a.go")) + if w.forge.openCalls != before+3 { // the look of a delivery awaiting its verdict reads its standing one + t.Fatalf("the same head announced again read the forge again (%d reads)", w.forge.openCalls-before) + } +} + +func TestNothingReadsTheForgeWithoutAnEvent(t *testing.T) { + w := newWorld(t) + w.forge.open = []OpenPull{openPull("novox/app", 7, readyHead)} + before := w.forge.openCalls + for range 6 { + w.later(time.Minute) + w.settleAll() + } + if w.forge.openCalls != before { + t.Fatalf("the forge's open pull requests were read %d times with no event", w.forge.openCalls-before) + } +} + +func TestAClosedOrMergedPullRequestIsNoLongerSaidUnchecked(t *testing.T) { + w := newWorld(t) + w.forge.open = []OpenPull{{Owner: "novox", Repo: "hq", Number: 1, Base: "main", HeadSHA: "cccc", + Required: []string{"mesh/merge-gate"}}} + w.h.SeeOpenPulls() + w.later(11 * time.Minute) + if len(w.h.Stalled()) != 1 { + t.Fatal("not said") + } + closed := pr("novox/hq", 1, "cccc", "feat/x") + hear(t, w, "gitea.pull.closed", closed) + if lines := w.h.Stalled(); len(lines) != 0 { + t.Fatalf("a closed pull request is still said: %+v", lines) + } +} diff --git a/modules/mesh-delivery/cmd/mesh-delivery/unchecked_test.go b/modules/mesh-delivery/cmd/mesh-delivery/unchecked_test.go new file mode 100644 index 00000000..c1ce9718 --- /dev/null +++ b/modules/mesh-delivery/cmd/mesh-delivery/unchecked_test.go @@ -0,0 +1,80 @@ +package main + +import ( + "strings" + "testing" + "time" +) + +// **A pull request the mesh was never asked to check is a delivery held, and said** (novox/hq issue 347): one open +// on a base that requires the merge check — mesh/merge-gate or mesh/repo-check, as hq requires only the second — +// whose head has had neither for ten minutes since this owner first saw it so, is a stalled line: the controller +// raises it as `delivery..stalled` (D14), the delivery domain's own word for a delivery held past its bound. +func TestAPullRequestNobodyCheckedIsAStalledDelivery(t *testing.T) { + w := newWorld(t) + head := "bfe82315f42c8f4509d0598c32569a202d5eada6" + w.forge.open = []OpenPull{ + {Owner: "novox", Repo: "hq", Number: 243, Base: "main", HeadSHA: head, Required: []string{"mesh/repo-check"}}, + {Owner: "novox", Repo: "sdk", Number: 14, Base: "main", HeadSHA: "aaaa", Required: []string{"mesh/merge-gate", "mesh/repo-check"}, + Statuses: []string{"mesh/merge-gate"}}, + {Owner: "novox", Repo: "free", Number: 1, Base: "main", HeadSHA: "bbbb", Required: nil}, + } + w.h.SeeOpenPulls() + if lines := w.h.Stalled(); len(lines) != 0 { + t.Fatalf("a head just seen is stalled already: %+v", lines) + } + w.later(6 * time.Minute) + w.h.SeeOpenPulls() // seen again: its own first-seen time stands, whatever the forge's update time says + w.later(5 * time.Minute) + lines := w.h.Stalled() + if len(lines) != 1 || lines[0].ID != IDOf("novox/hq", head) || lines[0].State != Unannounced || + !strings.Contains(lines[0].Says, "no merge check") || !strings.HasPrefix(lines[0].H2, "none") { + t.Fatalf("an unchecked pull request on a base requiring only mesh/repo-check reads %+v", lines) + } + if lines[0].Number != 243 { + t.Fatalf("the line does not name the pull request's number: %+v", lines[0]) + } + if lines[0].For != "11m0s" { + t.Fatalf("its time is not counted from when this owner first saw it: %s", lines[0].For) + } + + // Announced at last: a delivery of its own, with its own bound — no longer this line. + w.h.PullUpdated(pr("novox/hq", 243, head, "decision/x")) + w.h.SeeOpenPulls() + for _, l := range w.h.Stalled() { + if l.State == Unannounced { + t.Fatalf("an announced pull request is still said unannounced: %+v", l) + } + } +} + +// One that gets a check, or closes, is dropped; one the forge cannot be asked about is not guessed. +func TestAnUncheckedPullRequestThatGetsACheckOrClosesIsDropped(t *testing.T) { + w := newWorld(t) + w.forge.open = []OpenPull{{Owner: "novox", Repo: "hq", Number: 1, Base: "main", HeadSHA: "cccc", + Required: []string{"mesh/merge-gate"}}} + w.h.SeeOpenPulls() + w.later(11 * time.Minute) + if len(w.h.Stalled()) != 1 { + t.Fatal("not said") + } + w.forge.open[0].Statuses = []string{"mesh/merge-gate"} + w.h.SeeOpenPulls() + if len(w.h.Stalled()) != 0 { + t.Fatal("a pull request with a check is still said") + } + w.forge.open[0].Statuses = nil + w.h.SeeOpenPulls() + w.later(11 * time.Minute) + w.forge.openDown = true + w.h.SeeOpenPulls() // the forge away: what was seen stands, nothing new is guessed + if len(w.h.Stalled()) != 1 { + t.Fatal("a forge that could not be asked dropped or invented a line") + } + w.forge.openDown = false + w.forge.open = nil + w.h.SeeOpenPulls() + if len(w.h.Stalled()) != 0 { + t.Fatal("a closed pull request is still said") + } +} diff --git a/modules/mesh-delivery/cmd/mesh-delivery/verbs.go b/modules/mesh-delivery/cmd/mesh-delivery/verbs.go new file mode 100644 index 00000000..3a7dbce7 --- /dev/null +++ b/modules/mesh-delivery/cmd/mesh-delivery/verbs.go @@ -0,0 +1,851 @@ +package main + +import ( + "errors" + "fmt" + "sort" + "strings" + "time" +) + +// The seat's verbs (novox/hq ADR 0239, the mesh-delivery seat in the controller's set): six that read, and +// the acts — a person's recheck, release and stop, and healer H2's close, each only by a transition the table +// holds. + +// Line is one delivery as `deliveries` lists it. +type Line struct { + ID string `json:"id"` + State State `json:"state"` + Since string `json:"since"` + For string `json:"for"` + Group string `json:"group,omitempty"` + Number int `json:"number,omitempty"` + Waits string `json:"waits,omitempty"` + Plan string `json:"plan,omitempty"` + Walk string `json:"walk,omitempty"` + Owed int `json:"owed,omitempty"` + Stalled bool `json:"stalled,omitempty"` +} + +// waitsFor is what a delivery waits for, in words. +func (h *Holder) waitsFor(d *Delivery) string { + if d.HeldBack != "" && !d.State.Final() { + return d.HeldBack + } + switch d.State { + case Proposed: + return "its check" + case Ready: + if b, batched := h.batchOf(d); batched { + return batchSaid(b, h.Now()) + } + if d.MergedAs != "" { + return "a walk that answers its merge" + } + return "its merge" + case Rejected: + return "a new head, or a recheck" + case Published: + if d.waitsForItsOwnWalk() { + return "a walk of its own commit: its walk " + d.Walk.ID + ", which carried " + shortOf(d.AnsweredBy, 8) + + " for it, failed" + } + if d.Walk != nil && d.Walk.TakenOverBy != "" { + return "the walk " + d.Walk.TakenOverBy + ", which took its walk " + d.Walk.ID + " over" + } + if d.endedUnstarted() { + // Never "its walk's start": a done walk has none (novox/hq issue 474). + if _, known := d.walkMovedItsModules(); !known { + return "its plan or its walk's record, to read what its walk " + d.Walk.ID + ", ended before its word, built" + } + return "a later delivered merge to carry " + strings.Join(d.notMovedBy(), ", ") + ": its walk " + d.Walk.ID + + " ended without building it, and it is stopped " + strandedAfter.String() + " after that walk ended otherwise" + } + if d.Group != "" { + return "its group " + d.Group + ": every member merged, and the plan's word" + } + return "its walk's start" + case Held: + return "a person: " + d.HeldWhy + case Delivering: + if d.Walk != nil { + return "its walk: " + clip(d.Walk.Note, 120) + } + case Stopped: + if h.stopLapsed(d) { + return "a later walk of its repository that contains its merge: its group stopped it for " + d.stoppedFor() + + ", whose change was delivered since (novox/hq issue 747)" + } + if strandedUncovered(d) { + return "a later delivered merge to carry " + strings.Join(d.notMovedBy(), ", ") + ": it is stopped, and " + + "listed by stalled until one does (novox/hq issue 480)" + } + } + return "" +} + +// Deliveries is the `deliveries` verb. +func (h *Holder) Deliveries(state, repository, group string, all bool) []Line { + h.mu.Lock() + defer h.mu.Unlock() + now := h.Now() + var out []Line + for _, d := range h.deliveries { + if state != "" && string(d.State) != state || repository != "" && !strings.EqualFold(d.Repository, repository) || + group != "" && d.Group != group { + continue + } + // A final delivery `stalled` still lists stays, however old: the two never disagree (novox/hq issue 480). + if d.State.Final() && !all && now.Sub(d.Since) > 24*time.Hour && !strandedUncovered(d) { + continue + } + l := Line{ID: d.ID, State: d.State, Since: d.Since.UTC().Format(time.RFC3339), For: now.Sub(d.Since).Round(time.Second).String(), + Group: d.Group, Number: d.Number, Waits: h.waitsFor(d), Owed: len(d.Owed)} + if d.Plan != nil { + l.Plan = d.Plan.Summary + } + if d.Walk != nil { + l.Walk = d.Walk.ID + } + l.Stalled = pastBound(d, now) || strandedUncovered(d) + out = append(out, l) + } + sort.Slice(out, func(i, j int) bool { + if out[i].State.Final() != out[j].State.Final() { + return !out[i].State.Final() + } + return out[i].Since > out[j].Since + }) + return out +} + +// Show is the `show` verb: a delivery whole, or a group with its members. +func (h *Holder) Show(id string) (any, error) { + h.mu.Lock() + defer h.mu.Unlock() + if d := h.deliveries[id]; d != nil { + out := map[string]any{"delivery": d, "waits": h.waitsFor(d)} + if d.coveredStop() { + out["reads"] = d.coveredStopSaid() // stopped, and its change reached the machines (novox/hq issue 480) + } + if g := h.groups[d.Group]; g != nil { + state, why := GroupState(g, h.membersOf(g)) + out["group"] = map[string]any{"group": g, "state": state, "why": why} + } + return out, nil + } + if g := h.groups[id]; g != nil { + members := h.membersOf(g) + state, why := GroupState(g, members) + return map[string]any{"group": g, "state": state, "why": why, "members": members}, nil + } + return nil, fmt.Errorf("no delivery or group %q: `deliveries` lists them", id) +} + +// GroupLine is one group as `groups` lists it. +type GroupLine struct { + ID string `json:"id"` + State string `json:"state"` + Why string `json:"why"` + // ComposedGate says the group's gate is the composed check's: each member's own pull request check runs the + // repository's check alone, and the composed verdict sets its mesh/merge-gate (novox/hq ADR 0403 §1, issue 978). + ComposedGate bool `json:"composed_gate"` + Check string `json:"composed,omitempty"` + Members []string `json:"members"` + // Details are the members again, one object each, with what a reader needs to find the pull request's + // siblings: its number, full head, base branch and whether it is open (novox/hq ADR 0382 §1). Additive: + // `members` keeps its text form for older readers. + Details []MemberLine `json:"member_details,omitempty"` +} + +// MemberLine is one member of a group, as `groups` lists it beside its text line. +type MemberLine struct { + ID string `json:"id"` + Repository string `json:"repository"` + State State `json:"state"` + Number int `json:"number"` + Head string `json:"head"` + Base string `json:"base"` + Open bool `json:"open"` +} + +// Groups is the `groups` verb. +func (h *Holder) Groups(all bool) []GroupLine { + h.mu.Lock() + defer h.mu.Unlock() + var out []GroupLine + for _, g := range h.groups { + members := h.membersOf(g) + if len(members) == 0 { + continue + } + state, why := GroupState(g, members) + if !all && (state == "delivered" || state == "failed" || state == "stopped") { + continue + } + l := GroupLine{ID: g.ID, State: state, Why: why, ComposedGate: true} + for _, d := range members { + l.Members = append(l.Members, fmt.Sprintf("%s (%s)", d.ID, d.State)) + l.Details = append(l.Details, MemberLine{ID: d.ID, Repository: d.Repository, State: d.State, + Number: d.Number, Head: d.Commit, Base: d.Base, + Open: !d.State.Final() && d.MergedAs == "" && !d.ClosedUnmerged}) + } + if g.Check != nil { + l.Check = g.Check.Verdict + if l.Check == "" { + l.Check = "asked as " + g.Check.Asked + } + } + out = append(out, l) + } + sort.Slice(out, func(i, j int) bool { return out[i].ID < out[j].ID }) + return out +} + +// StalledLine is one delivery past its state's bound. +type StalledLine struct { + ID string `json:"id"` + // Number is the pull request's, for a head the forge never announced: the condition names it (novox/hq issue 347). + Number int `json:"number,omitempty"` + State State `json:"state"` + // Walk is the walk a line of the state Unheld is about (novox/hq issue 362). + Walk string `json:"walk,omitempty"` + // Waiting are the merge checks a line of the state Unanswered waits on, each as it reads — "mesh/repo-check + // never set", "mesh/merge-gate pending since " — so the controller's words name them without reading + // Says (novox/hq issue 438). + Waiting []string `json:"waiting,omitempty"` + // Checks are the same merge checks as data — context, state, since when — so the controller says their times in + // the operator's own zone, which it cannot do inside a finished sentence (novox/hq issue 443). Waiting stays + // beside them for a controller that does not read them yet. + Checks []WaitingCheck `json:"checks,omitempty"` + // Held is the state of the delivery a line of the state Unanswered is about, ready or rejected, when one is held + // here (novox/hq issues 441, 451): its check answered once, so the controller advises a recheck of it rather than + // the build queue, reading the state here and never from Says. Empty for a head with no delivery. + Held State `json:"held,omitempty"` + For string `json:"for"` + Bound string `json:"bound"` + H2 string `json:"h2"` + Says string `json:"says"` + // Severity is "urgent" for a group that is held or half merged (issue 1275); empty otherwise. + Severity string `json:"severity,omitempty"` +} + +// Stalled is the `stalled` verb: what the controller's self-check raises as `delivery..stalled`. +func (h *Holder) Stalled() []StalledLine { + h.mu.Lock() + defer h.mu.Unlock() + now := h.Now() + var out []StalledLine + for _, d := range h.deliveries { + if strandedUncovered(d) { + // Stopped, so past no bound, and still loud (review of novox/hq issue 474): listed until a later + // delivered merge carries its modules. + out = append(out, StalledLine{ID: d.ID, State: d.State, For: now.Sub(d.Since).Round(time.Second).String(), + Bound: "0s", H2: "none: a merge that delivers its modules ends it — the operator's", Says: d.Stranded}) + continue + } + if !pastBound(d, now) { + continue + } + b, _ := boundOf(d) + h2 := "none: the state is the operator's" + switch { + case d.State == Proposed && d.AskedAgain != nil && d.outOfDate() != "": + h2 = "none: its verdict is out of date and its check, asked again, has not answered — the operator's" + case d.State == Proposed && d.CheckAsk == nil: + h2 = "none: its check could not be asked yet — the controller does not answer delivery-check" + case d.State == Proposed && d.CheckAsk.Reasked > 0: + h2 = "none: its check was re-asked once already and still not answered — the operator's" + case d.State == Proposed: + h2 = "close, by re-ask: its check asked once more" + case len(b.H2) > 0: + var evs []string + for _, e := range b.H2 { + evs = append(evs, string(e)) + } + h2 = "close, by " + strings.Join(evs, " or ") + ", when the walk's record says so" + } + out = append(out, StalledLine{ID: d.ID, State: d.State, For: now.Sub(boundSince(d)).Round(time.Second).String(), + Bound: b.For.String(), H2: h2, Says: b.Says}) + } + out = append(out, h.withChecks(h.uncheckedLines(now))...) + out = append(out, h.orphanLines(now)...) + out = append(out, h.readyLines(now)...) + out = append(out, h.groupLines(now)...) + sort.Slice(out, func(i, j int) bool { return out[i].ID < out[j].ID }) + return out +} + +// Recheck is the `recheck` verb: a rejected or ready delivery proposed again, and its check asked again. +// +// A recheck asks of the head the forge already judged, and the forge keeps the newest status of each +// context on it: left alone, the old verdict stands, and a branch that requires it lets the pull request merge +// on it while the fresh check runs. So the merge check's statuses are put back to pending first — by the +// forge's holder, which keeps them — and only then is the check asked. A forge that cannot be told refuses the +// recheck whole: the delivery stays where it stood, and nothing is asked on a stale verdict. +func (h *Holder) Recheck(id, why, by string) (string, error) { + h.mu.Lock() + d := h.deliveries[id] + if d == nil { + h.mu.Unlock() + return "", fmt.Errorf("no delivery %q", id) + } + // Whether the table takes it, tried on a copy: nothing is said on the forge for a recheck it refuses. + probe := *d + probe.Transitions = append([]Transition(nil), d.Transitions...) + if _, err := Apply(&probe, EvRecheck, true, Facts{Now: h.Now(), Why: why, By: by}); err != nil { + h.mu.Unlock() + return "", err + } + owner, repo, commit, page := d.Owner(), d.Repo(), d.Commit, d.HTMLURL + h.mu.Unlock() + if err := h.checkingAgain(owner, repo, commit, page, why); err != nil { + return "", fmt.Errorf("%s is not rechecked: %v; its merge check must say pending before it is asked again, "+ + "so nothing merges on the old verdict — ask again once the forge answers", id, err) + } + h.mu.Lock() + t, err := Apply(d, EvRecheck, true, Facts{Now: h.Now(), Why: why, By: by}) + if err != nil { + h.mu.Unlock() + return "", fmt.Errorf("%v (its merge check says pending; the forge's next announcement asks it)", err) + } + d.Check = nil + // The composed check of its group goes out of date (novox/hq ADR 0323 §6, issue 867): it was judged for these very + // heads (running or answered: a run's answer is taken by its id and must not stand for the recheck), so without this the kept verdict (a failure above all) stands and the group is never asked again. It is + // asked again once the member is judged again, when every member is ready. + if g := h.groups[d.Group]; g != nil && g.Check != nil { + now := h.Now() + g.Check.OutOfDate = fmt.Sprintf("composed check out of date: %s was rechecked (%s)", d.ID, why) + g.Check.OutOfDateAt = now + g.Updated = now + h.keepGroup(g) + } + h.moved(d, []Transition{t}) + m := MemberOf(d) + h.mu.Unlock() + asked, err := h.Controller.Check("", []Member{m}) + if err != nil { + return "", fmt.Errorf("%s is proposed again and its check could not be asked yet (%v); the forge's next "+ + "announcement asks it", id, err) + } + return fmt.Sprintf("%s is proposed again, its merge check pending; its check was asked (%s)", id, asked), nil +} + +// checkingAgain puts a head's merge check statuses — those a branch's protection may require — back to +// pending, so the forge refuses the merge until the fresh verdict sets them. The gate always, since every +// verdict sets it; the repository's own check only when the head holds one, since a verdict that says none +// would leave a pending one standing for ever. Asked of the forge's holder, which keeps them; it takes +// pending on them and nothing else, a verdict being the controller's alone (novox/hq issue 293). +func (h *Holder) checkingAgain(owner, repo, commit, page, why string) error { + if h.Forge == nil { + return errors.New("no forge") + } + contexts := []string{ctxGate, ctxRepo} + if held, err := h.Forge.Statuses(owner, repo, 0, commit, ""); err == nil { + contexts = []string{ctxGate} + for _, s := range held.Statuses { + if s.Context == ctxRepo { + contexts = append(contexts, ctxRepo) + } + } + } + line := clip(oneLine("checking again: "+why), 140) + for _, c := range contexts { + if err := h.Forge.Status(owner, repo, commit, c, "pending", line, page); err != nil { + return fmt.Errorf("the forge did not set %s pending on %s: %w", c, shortOf(commit, 8), err) + } + } + return nil +} + +// Release is the `release` verb: a held delivery goes on, on a person's word. +func (h *Holder) Release(id, why, by string) (string, error) { + h.mu.Lock() + d := h.deliveries[id] + if d == nil { + h.mu.Unlock() + return "", fmt.Errorf("no delivery %q", id) + } + t, err := Apply(d, EvRelease, true, Facts{Now: h.Now(), Why: why, By: by}) + if err != nil { + h.mu.Unlock() + return "", err + } + // Held for a failed backup before the send: the controller is asked to take the backups again (novox/hq ADR + // 0404 clause 4); its record at this revision or before still holds the failure, and does not hold it again. + backupsOf, againBefore := "", d.BackupsAgainAfter + if d.Walk != nil && len(d.Walk.BackupsFailed) > 0 { + backupsOf, d.BackupsAgainAfter = d.Walk.ID, d.Walk.Revision + } + d.HeldWhy = "" + h.moved(d, []Transition{t}) + if backupsOf != "" { + h.mu.Unlock() + if err := h.Controller.BackupsAgain(backupsOf, "released by "+by+": "+why); err != nil { + // The controller did not clear the failure: its record still holds it, so the delivery is held for it again now. + h.mu.Lock() + if d := h.deliveries[id]; d != nil { + d.BackupsAgainAfter = againBefore + h.settleAndKeep(d) + } + h.mu.Unlock() + h.again("a walk's backups the controller refused or did not answer") + return "", fmt.Errorf("%s was not released: the controller could not be asked to take the backups of %s again "+ + "(%v), so it is held again for the failed backup", id, backupsOf, err) + } + return fmt.Sprintf("%s is delivering on %s's word; the backups of its walk %s are asked again", id, by, backupsOf), nil + } + var walk string + if d.Walk != nil && d.Walk.Waits { + walk = d.Walk.ID + } + var stopped []string + if walk != "" { + stopped, _ = h.carriesStopped(walk) + } + h.mu.Unlock() + if walk == "" { + return fmt.Sprintf("%s is delivering, on %s's word", id, by), nil + } + if len(stopped) > 0 { + // Never the word for a walk that carries a stopped change (ADR 0299): asked to leave it out first. + return "", fmt.Errorf("%s is delivering on %s's word, and its walk %s is not let go yet: it carries %s, whose "+ + "delivery was stopped, and is asked to leave it out first", id, by, walk, strings.Join(stopped, ", ")) + } + if err := h.Controller.Deliver(walk, "released by "+by+": "+why); err != nil { + h.again("a walk's word the controller refused or did not answer") + return "", fmt.Errorf("%s is delivering on %s's word, and its walk %s could not be let go yet (%v): it is "+ + "asked again in %s", id, by, walk, err, askAgainAfter) + } + h.mu.Lock() + if d := h.deliveries[id]; d != nil { + d.LetGo, d.LetGoWhy = h.Now(), "released by "+by + if d.Walk != nil && d.Walk.ID == walk { + d.Walk.Waits, d.Walk.GoBy = false, byOwner + } + h.keep(d) + } + h.mu.Unlock() + return fmt.Sprintf("%s is delivering on %s's word; its walk %s is let go", id, by, walk), nil +} + +// Stop is the `stop` verb with no way given: see StopWay. +func (h *Holder) Stop(id, why, by string) (string, error) { return h.StopWay(id, why, by, "") } + +// The ways a person's stop of a delivery on a shared walk takes (novox/hq ADR 0299). +const ( + wayAgain = "again" // the walk ended, its other merges walked again without this one + wayWalk = "walk" // the whole walk stopped, every delivery on it with it +) + +// StopWay is the `stop` verb: a delivery stopped, its walk ended through the controller, and in a group every +// member after it stopped too. +// +// **A stop never takes other deliveries down quietly** (novox/hq issue 459, ADR 0299): one walk carries every merge +// of its batch. When the delivery's open walk carries anything else, the stop does nothing at first: it says what +// ending the walk would take down — each other delivery, and which of them contain this merge and end either way — +// and the two ways, given with `way`: `again` ends the walk and walks the others again without this merge and the +// merges that contain it; `walk` stops the whole walk, and every delivery on it ends stopped, naming this stop. +func (h *Holder) StopWay(id, why, by, way string) (string, error) { + if way != "" && way != wayAgain && way != wayWalk { + return "", fmt.Errorf("way %q: %q or %q", way, wayAgain, wayWalk) + } + h.mu.Lock() + d := h.deliveries[id] + if d == nil { + h.mu.Unlock() + return "", fmt.Errorf("no delivery %q", id) + } + // Its guard is read on a copy: the delivery is stopped only once its walk is (review of #221). + trial := *d + if _, err := Apply(&trial, EvStop, true, Facts{Now: h.Now(), Why: why, By: by}); err != nil { + h.mu.Unlock() + return "", err + } + var walk string + if d.Walk != nil && (d.Walk.State == walkBuilding || d.Walk.State == walkRolling) { + walk = d.Walk.ID + } + merge := d.Repository + "@" + d.MergedAs + // A merge that waits in a batch not yet cut: the batch is asked to leave it out once it is stopped (ADR 0299). + var batch string + if b, batched := h.batchOf(d); batched && walk == "" { + batch = b.ID + } + var others []string + if walk != "" { + others = h.othersOn(walk, []string{merge}) + } + if len(others) > 0 && way == "" { + said := h.takenDown(d, walk, others) + h.mu.Unlock() + return "", errors.New(said) + } + h.mu.Unlock() + + var did string + switch { + case walk == "": + case len(others) == 0 || way == wayWalk: + whole := why + if len(others) > 0 { + whole = fmt.Sprintf("%s was stopped, and its whole walk with it: %s", id, why) + } + if err := h.Controller.Stop(walk, whole, by); err != nil { + return "", fmt.Errorf("%s is not stopped: its walk %s could not be ended yet: %v — `plans stop %s --why …` "+ + "ends it by hand", id, walk, err, walk) + } + did = fmt.Sprintf("; its walk %s is ended", walk) + if len(others) > 0 { + did = fmt.Sprintf("; its walk %s is ended whole, and every delivery on it with it: %s", walk, + strings.Join(others, ", ")) + } + default: + a, err := h.Controller.StopWithout(walk, why, by, []string{merge}) + if err != nil { + return "", fmt.Errorf("%s is not stopped: its walk %s could not be ended without it: %v — `way: %s` stops "+ + "the whole walk, and `plans stop %s --why …` ends it by hand", id, walk, err, wayWalk, walk) + } + did = fmt.Sprintf("; its walk %s is ended without it, and %s", walk, againWords(a)) + } + + h.mu.Lock() + d = h.deliveries[id] + if d == nil || d.State.Final() { + h.mu.Unlock() + return fmt.Sprintf("%s ended meanwhile%s", id, did), nil + } + t, err := Apply(d, EvStop, true, Facts{Now: h.Now(), Why: why, By: by}) + if err != nil { + h.mu.Unlock() + return "", err + } + h.moved(d, []Transition{t}) + var asks []func() + if g := h.groups[d.Group]; g != nil && g.Closed { + asks = h.groupWord(g, h.membersOf(g)) + } else if g != nil { + h.leaveGroup(d) + } + h.mu.Unlock() + for _, a := range asks { + a() + } + if batch != "" { + // Asked now, not at the next look; and the batch's walk is never let go while it carries the merge. + did = fmt.Sprintf("; its merge waits in the batch %s, which was asked now to leave it out and is never let go "+ + "while it carries it: %s", batch, h.stopLeaving(batch, why, by, []string{merge})) + } + return fmt.Sprintf("%s is stopped, on %s's word%s", id, by, did), nil +} + +// takenDown is what ending a delivery's shared walk would take down, and the two ways: for a caller holding the +// lock (ADR 0299). +func (h *Holder) takenDown(d *Delivery, walk string, others []string) string { + var ride, contain []string + for _, o := range others { + od := h.deliveries[o] + if od != nil && containsMerge(od, d) { + contain = append(contain, o) + continue + } + ride = append(ride, o) + } + s := fmt.Sprintf("%s is not stopped: its walk %s carries other changes, and ending the walk would take them down: %s.", + d.ID, walk, orNone(strings.Join(ride, ", "))) + if len(contain) > 0 { + s += fmt.Sprintf(" These contain its merge and end with it either way: %s.", strings.Join(contain, ", ")) + } + // A module is built at its branch, which holds this merge: a delivery moving a module this one moves would + // deliver it, and the controller leaves it out too (ADR 0299). Its plan is what is known here; the controller + // decides, and its answer names each. + var share []string + for _, o := range ride { + if od := h.deliveries[o]; od != nil && od.Plan != nil && d.Plan != nil { + if both := sharedModules(od.Plan.Moved, d.Plan.Moved); len(both) > 0 { + share = append(share, fmt.Sprintf("%s (moves %s)", o, strings.Join(both, ", "))) + } + } + } + if len(share) > 0 { + s += fmt.Sprintf(" With `way: %s` these are left out too, for moving a module this merge moves: %s.", wayAgain, + strings.Join(share, ", ")) + } + s += fmt.Sprintf(" With `way: %s` the controller may leave out more for sharing a module, and its answer names each.", + wayAgain) + return s + fmt.Sprintf(" What the walk already sent stays where it was sent; the stop's answer names it."+ + " Choose a way: `way: %s` ends the walk and walks the others again without this merge; "+ + "`way: %s` stops the whole walk and every delivery on it.", wayAgain, wayWalk) +} + +// sharedModules are the modules in both lists, sorted. +func sharedModules(a, b []string) []string { + var out []string + for _, m := range a { + if containsString(b, m) && !containsString(out, m) { + out = append(out, m) + } + } + sort.Strings(out) + return out +} + +// containsMerge is whether o is a later merge of d's repository and branch, which contains d's merge on a linear +// trunk: merged after it, or the merge the walk carries for it. +func containsMerge(o, d *Delivery) bool { + if o.ID == d.ID || !strings.EqualFold(o.Repository, d.Repository) || !strings.EqualFold(o.Base, d.Base) || o.MergedAs == "" { + return false + } + return o.MergedAt.After(d.MergedAt) || d.AnsweredBy != "" && sameCommit(d.AnsweredBy, o.MergedAs) && + !sameCommit(d.AnsweredBy, d.MergedAs) +} + +// Close is healer H2's verb: for a delivery past its bound, the walk's record is read again and the +// delivery settled; only a transition the table names for that state counts, any other answer is refused. +func (h *Holder) Close(id, why string) (string, error) { + h.mu.Lock() + d := h.deliveries[id] + if d == nil { + h.mu.Unlock() + return "", fmt.Errorf("no delivery %q", id) + } + b, bounded := boundOf(d) + if !bounded || len(b.H2) == 0 { + h.mu.Unlock() + return "", fmt.Errorf("%s is %s: H2 has no transition there — the state is the operator's", id, d.State) + } + if !pastBound(d, h.Now()) { + h.mu.Unlock() + return "", fmt.Errorf("%s has been %s for %s, within its bound of %s: nothing to close", id, d.State, + h.Now().Sub(boundSince(d)).Round(time.Second), b.For) + } + if d.State == Proposed { + return h.reask(d, why) // unlocks + } + if d.Walk == nil { + h.mu.Unlock() + return "", fmt.Errorf("%s has no walk on record: H2 has nothing to read", id) + } + walk, was := d.Walk.ID, d.State + h.mu.Unlock() + _, walks, err := h.Controller.Walks(walk) + if err != nil { + return "", fmt.Errorf("the walk %s cannot be read: %w", walk, err) + } + for _, w := range walks { + h.WalkMoved(w) + } + h.mu.Lock() + defer h.mu.Unlock() + d = h.deliveries[id] + if d.State == was { + return "", errors.New(id + " is still " + string(was) + ": its walk's record does not say it moved — the condition is the operator's") + } + return fmt.Sprintf("%s: %s → %s, as its walk's record says (%s)", id, was, d.State, why), nil +} + +// reask is H2's close of a proposed delivery past its bound: its check, asked and not answered, asked once +// more — the table's re-ask, which holds only once. The row is tried first on a copy, so an ask the controller +// does not take spends nothing. Called under the lock; it unlocks. +func (h *Holder) reask(d *Delivery, why string) (string, error) { + id := d.ID + trial := *d + trial.Transitions = nil + if _, err := Apply(&trial, EvReask, true, Facts{Now: h.Now(), By: byH2, Why: why}); err != nil { + h.mu.Unlock() + return "", err + } + m := MemberOf(d) + h.mu.Unlock() + asked, err := h.Controller.Check("", []Member{m}) + if err != nil { + return "", fmt.Errorf("%s's check could not be re-asked yet: %w", id, err) + } + h.mu.Lock() + defer h.mu.Unlock() + d = h.deliveries[id] + if d == nil { + return "", fmt.Errorf("no delivery %q", id) + } + t, err := Apply(d, EvReask, true, Facts{Now: h.Now(), By: byH2, Why: why + " — re-asked as " + asked}) + if err != nil { + return "", err + } + d.CheckAsk.Reasked++ + d.CheckAsk.ID, d.CheckAsk.At, d.CheckAsk.By, d.CheckAsk.Why = asked, h.Now(), byH2, why + h.moved(d, []Transition{t}) + return fmt.Sprintf("%s: its check re-asked as %s, once (%s); if that is not answered either, the operator's", id, + asked, why), nil +} + +// WhatIf is the `what-if` verb. +func (h *Holder) WhatIf(repository, base string, paths []string) (*DeliveryPlan, error) { + if base == "" { + base = "main" + } + return h.Controller.Plan(repository, base, "", paths, nil, nil) +} + +// Status is the module's own `delivery_status`. +func (h *Holder) Status(listening string) map[string]any { + h.mu.Lock() + defer h.mu.Unlock() + count := map[State]int{} + owed := 0 + for _, d := range h.deliveries { + count[d.State]++ + owed += len(d.Owed) + } + for _, g := range h.groups { + owed += len(g.Owed) + } + byState := map[string]int{} + for s, n := range count { + byState[string(s)] = n + } + out := map[string]any{"deliveries": byState, "groups": len(h.groups), "owed": owed, "unkept": len(h.dirty), + "walks-unmatched": len(h.unmatched), "events": listening, "started": h.started.Format(time.RFC3339), + "seat-on-record": h.held} + // The owed effects tried since this owner started (novox/hq issue 407): done, failed and to be tried again, + // or given up, and when the last was tried. A count since the start, said with that start: two readings in + // one run give a window's tries. + tried := map[string]any{"since": h.started.Format(time.RFC3339), "done": h.tried.done, "failed": h.tried.failed, + "given-up": h.tried.givenUp, "tried": h.tried.done + h.tried.failed + h.tried.givenUp} + if !h.tried.last.IsZero() { + tried["last"] = h.tried.last.Format(time.RFC3339) + } + out["owed-tried"] = tried + // Each held delivery by name, with why and the verb that releases it (novox/hq issue 529): a count alone + // pointed a reader at nothing. + var held []string + for _, d := range h.deliveries { + if d.State == Held { + held = append(held, HeldLine(d)) + } + } + if len(held) > 0 { + sort.Strings(held) + out["held"] = held + } + if !h.reconciled.IsZero() { + out["walks-read"] = h.reconciled.Format(time.RFC3339) + } + if h.unreached != "" { + out["controller-unreached"] = h.unreached + } + if len(h.refusals) > 0 { + from := 0 + if len(h.refusals) > 10 { + from = len(h.refusals) - 10 + } + out["refused"] = h.refusals[from:] + } + if len(h.backlog) > 0 { + var ids []string + for _, w := range h.backlog { + ids = append(ids, w.ID+" "+w.State) + } + out["backlog-walks"] = ids + } + return out +} + +// RetiredAnswer is what `retire-history` answers. +type RetiredAnswer struct { + Retired int `json:"retired"` + ByRepository map[string]int `json:"by-repository,omitempty"` + Dry bool `json:"dry,omitempty"` + // LeftHeld are the held deliveries the rule does not take, with why — a person's still. + LeftHeld int `json:"left-held"` + Left []string `json:"left,omitempty"` + Says string `json:"says"` +} + +// RetireHistory is the `retire-history` verb (novox/hq issue 313): held deliveries that are history — made from +// the forge's word for a merge on the trunk heard long after it happened, with no walk of theirs open — retired +// on a person's word, never released or delivered. One by its id, refused when the rule does not take it; or +// every held one, of one repository when named, each the rule does not take left held and named. Dry answers +// what it would do and changes nothing. +func (h *Holder) RetireHistory(id, repository, why, by string, dry bool) (*RetiredAnswer, error) { + if why == "" { + return nil, errors.New("a retire says why") + } + h.mu.Lock() + defer h.mu.Unlock() + now := h.Now() + var held []*Delivery + if id != "" { + d := h.deliveries[id] + if d == nil { + return nil, fmt.Errorf("no delivery %q", id) + } + held = []*Delivery{d} + } else { + for _, d := range h.deliveries { + if d.State == Held && (repository == "" || strings.EqualFold(d.Repository, repository)) { + held = append(held, d) + } + } + sort.Slice(held, func(i, j int) bool { return held[i].ID < held[j].ID }) + } + out := &RetiredAnswer{ByRepository: map[string]int{}, Dry: dry} + for _, d := range held { + // Tried on a copy first: a refused retire changes nothing, and a dry one nothing either. + probe := *d + probe.Transitions = append([]Transition(nil), d.Transitions...) + f := Facts{Now: now, By: by, Why: why} + if _, err := Apply(&probe, EvRetire, true, f); err != nil { + if id != "" { + return nil, err + } + out.LeftHeld++ + if len(out.Left) < 25 { + out.Left = append(out.Left, err.Error()) + } + continue + } + out.Retired++ + out.ByRepository[d.Repository]++ + if dry { + continue + } + t, _ := Apply(d, EvRetire, true, f) + d.History = "retired by " + by + ": " + why + d.HeldWhy = "" + // Its pull request is not told: the forge may no longer hold it, or holds another under its number. + d.Owed = withoutKinds(d.Owed, EffectView, EffectStatus) + h.moved(d, []Transition{t}) + } + verb := "retired" + if dry { + verb = "would retire" + } + out.Says = fmt.Sprintf("%s %d held delivery(ies) as history; %d held left for a person", verb, out.Retired, out.LeftHeld) + h.Logf("[mesh-delivery] retire-history (%s): %s", why, out.Says) + return out, nil +} + +// withoutKinds is an owed list without the effects of the kinds named. +func withoutKinds(owed []Effect, kinds ...string) []Effect { + var out []Effect + for _, e := range owed { + if !containsString(kinds, e.Kind) { + out = append(out, e) + } + } + return out +} + +// HeldLine says a held delivery in one line: which, since when, why, its walk when one waits, and the verb that +// releases it (novox/hq issue 529). +func HeldLine(d *Delivery) string { + why := d.HeldWhy + if why == "" { + why = lastWhyOf(d) + } + s := fmt.Sprintf("%s held since %s: %s", d.ID, d.Since.UTC().Format(time.RFC3339), why) + if d.Walk != nil && d.Walk.Waits { + s += "; its walk " + d.Walk.ID + " waits for it" + } + return s + " — a person decides: `mesh-delivery.release` (id " + d.ID + ", why) lets it go, `mesh-delivery.stop` ends it" +} diff --git a/modules/mesh-delivery/cmd/mesh-delivery/waiting_checks.go b/modules/mesh-delivery/cmd/mesh-delivery/waiting_checks.go new file mode 100644 index 00000000..1da0f356 --- /dev/null +++ b/modules/mesh-delivery/cmd/mesh-delivery/waiting_checks.go @@ -0,0 +1,67 @@ +package main + +import ( + "slices" + "time" +) + +// The merge checks an unanswered line waits on, as data (novox/hq issue 443). The line's Waiting says them as finished +// words, with a pending check's time in UTC; the operator reads the controller's words, which give every time in the +// operator's own zone, and the controller cannot say a time again inside a sentence it did not write. So each check +// is given too as its context, its state and since when, and the controller words it. + +// WaitingCheck is one merge check a stalled line of the state Unanswered waits on. +type WaitingCheck struct { + Context string `json:"context"` + // State is CheckPending or CheckNeverSet. + State string `json:"state"` + // Since is when a pending check was set so, in RFC 3339 and UTC; empty when the forge did not say, or said what + // cannot be read — never a time made up. + Since string `json:"since,omitempty"` +} + +const ( + CheckPending = "pending" + CheckNeverSet = "never-set" +) + +// waitingChecks are the checks unanswered names, in its order and by its rule, as data. +func (p OpenPull) waitingChecks() []WaitingCheck { + if p.HeadSHA == "" { + return nil + } + started := slices.ContainsFunc(mergeChecks, func(c string) bool { + return slices.Contains(p.Statuses, c) || p.Checks[c].State != "" + }) + var out []WaitingCheck + for _, c := range mergeChecks { + if !slices.Contains(p.Required, c) { + continue + } + got, ok := p.Checks[c] + switch { + case p.Checks == nil && started: + // Read as checked, as unanswered does: nothing says whether the check it has answered. + case !ok || got.State == "": + out = append(out, WaitingCheck{Context: c, State: CheckNeverSet}) + case got.State == "pending": + w := WaitingCheck{Context: c, State: CheckPending} + if t, err := time.Parse(time.RFC3339, got.UpdatedAt); err == nil { + w.Since = t.UTC().Format(time.RFC3339) + } + out = append(out, w) + } + } + return out +} + +// withChecks gives each unanswered line the checks it waits on as data, from the head it was said of; under the lock. +// It reads every line of the state, whichever rule said it, so a line said for another reason carries them too. +func (h *Holder) withChecks(lines []StalledLine) []StalledLine { + for i, l := range lines { + if u, ok := h.unchecked[l.ID]; ok && l.State == Unanswered && len(l.Waiting) > 0 { + lines[i].Checks = u.pull.waitingChecks() + } + } + return lines +} diff --git a/modules/mesh-delivery/go.mod b/modules/mesh-delivery/go.mod new file mode 100644 index 00000000..5df693f9 --- /dev/null +++ b/modules/mesh-delivery/go.mod @@ -0,0 +1,22 @@ +module mesh-delivery + +go 1.25.0 + +require ( + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261013182924-5933eb757d44 + github.com/nats-io/nats-server/v2 v2.11.17 + github.com/nats-io/nats.go v1.51.0 +) + +require ( + github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op // indirect + github.com/google/go-tpm v0.9.8 // indirect + github.com/klauspost/compress v1.18.5 // indirect + github.com/minio/highwayhash v1.0.4 // indirect + github.com/nats-io/jwt/v2 v2.8.1 // indirect + github.com/nats-io/nkeys v0.4.15 // indirect + github.com/nats-io/nuid v1.0.1 // indirect + golang.org/x/crypto v0.50.0 // indirect + golang.org/x/sys v0.43.0 // indirect + golang.org/x/time v0.15.0 // indirect +) diff --git a/modules/mesh-delivery/go.sum b/modules/mesh-delivery/go.sum new file mode 100644 index 00000000..f84463ba --- /dev/null +++ b/modules/mesh-delivery/go.sum @@ -0,0 +1,27 @@ +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261013182924-5933eb757d44 h1:r5xIo5qWOXQdCSMxvqJy8kV4U+P/QWXFK6971Ha6wU8= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261013182924-5933eb757d44/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY= +github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= +github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= +github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= +github.com/google/go-tpm v0.9.8/go.mod h1:h9jEsEECg7gtLis0upRBQU+GhYVH6jMjrFxI8u6bVUY= +github.com/klauspost/compress v1.18.5 h1:/h1gH5Ce+VWNLSWqPzOVn6XBO+vJbCNGvjoaGBFW2IE= +github.com/klauspost/compress v1.18.5/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= +github.com/minio/highwayhash v1.0.4 h1:asJizugGgchQod2ja9NJlGOWq4s7KsAWr5XUc9Clgl4= +github.com/minio/highwayhash v1.0.4/go.mod h1:GGYsuwP/fPD6Y9hMiXuapVvlIUEhFhMTh0rxU3ik1LQ= +github.com/nats-io/jwt/v2 v2.8.1 h1:V0xpGuD/N8Mi+fQNDynXohVvp7ZztevW5io8CUWlPmU= +github.com/nats-io/jwt/v2 v2.8.1/go.mod h1:nWnOEEiVMiKHQpnAy4eXlizVEtSfzacZ1Q43LIRavZg= +github.com/nats-io/nats-server/v2 v2.11.17 h1:GKEghcFK6A+aFx11Yf1LjgLC3txAwvyhnYzhBIQZA8I= +github.com/nats-io/nats-server/v2 v2.11.17/go.mod h1:B1sFVz4StNosQ903ak4N1G01Fl/9f8e06mXpFIE2K24= +github.com/nats-io/nats.go v1.51.0 h1:ByW84XTz6W03GSSsygsZcA+xgKK8vPGaa/FCAAEHnAI= +github.com/nats-io/nats.go v1.51.0/go.mod h1:26HypzazeOkyO3/mqd1zZd53STJN0EjCYF9Uy2ZOBno= +github.com/nats-io/nkeys v0.4.15 h1:JACV5jRVO9V856KOapQ7x+EY8Jo3qw1vJt/9Jpwzkk4= +github.com/nats-io/nkeys v0.4.15/go.mod h1:CpMchTXC9fxA5zrMo4KpySxNjiDVvr8ANOSZdiNfUrs= +github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw= +github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= +golang.org/x/crypto v0.50.0 h1:zO47/JPrL6vsNkINmLoo/PH1gcxpls50DNogFvB5ZGI= +golang.org/x/crypto v0.50.0/go.mod h1:3muZ7vA7PBCE6xgPX7nkzzjiUq87kRItoJQM1Yo8S+Q= +golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.43.0 h1:Rlag2XtaFTxp19wS8MXlJwTvoh8ArU6ezoyFsMyCTNI= +golang.org/x/sys v0.43.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= +golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= diff --git a/modules/mesh-delivery/module.json b/modules/mesh-delivery/module.json new file mode 100644 index 00000000..1a40896b --- /dev/null +++ b/modules/mesh-delivery/module.json @@ -0,0 +1,95 @@ +{ + "module": "mesh-delivery", + "version": "1", + "slug": "deliver", + "claims": [ + { + "name": "mesh-delivery", + "scope": "mesh", + "serves": [ + "deliveries", + "times", + "show", + "groups", + "what-if", + "checks", + "table", + "stalled", + "recheck", + "release", + "stop", + "close", + "retire-history" + ] + } + ], + "consumes": [ + "gitea.pull.updated", + "gitea.pull.merged", + "gitea.pull.closed", + "gitea.forge.status", + "gitea.forge.pull-request.edited", + "gitea.forge.pull-request.closed", + "mesh-controller.checked", + "mesh-controller.plan-moved" + ], + "emits": [ + "transition", + "group", + "merged" + ], + "invokes": [ + "seat:mesh-controller.delivery-plan", + "seat:mesh-controller.delivery-check", + "seat:mesh-controller.deliver", + "seat:mesh-controller.delivery-backups-again", + "seat:mesh-controller.delivery-stop", + "seat:mesh-controller.delivery-walks", + "seat:mesh-controller.forget-checks", + "gitea.gitea_note_append", + "gitea.gitea_delivery_view", + "gitea.gitea_commit_status", + "gitea.gitea_commit_statuses", + "gitea.gitea_contains", + "gitea.gitea_open_pulls", + "gitea.gitea_merge_pull_request", + "gitea.gitea_add_comment", + "seat:operator-channel.notify" + ], + "uses": [ + "operator-channel" + ], + "state": [ + "deliveries", + "groups", + "asked", + "pulls" + ], + "tools": [ + "delivery_status" + ], + "resources": [ + { + "id": "state", + "type": "directory", + "mode": "0700", + "place": ".", + "unknown": "program" + } + ], + "build": { + "artifacts": [ + { + "name": "tools", + "kind": "bundle", + "language": "go", + "system": "arch", + "from": "cmd/mesh-delivery", + "binary": "mesh-delivery", + "loads": [ + "mesh-delivery" + ] + } + ] + } +}