diff --git a/modules/mesh-delivery-gate/README.md b/modules/mesh-delivery-gate/README.md new file mode 100644 index 00000000..23bcd87a --- /dev/null +++ b/modules/mesh-delivery-gate/README.md @@ -0,0 +1,42 @@ +# mesh-delivery-gate + +The `mesh-delivery-gate` module of the mesh catalogue: the gate of design 47. It claims the seat `mesh-delivery`, whose +name stays (ADR 0437 §11), and takes over from the module `mesh-delivery` (ADR 0437 §3, ADR 0450 §6). Its buckets are +`mesh-delivery-gate_*`. + +## Files it does not know + +Each directory or archive this module declares says how it treats a file it did not write (ADR 0422, ADR 0416). + +- `${dir:state}`: everything in it is the program's; the mesh moves nothing out (ADR 0422 §2). + +## A backup that failed before the send + +The controller backs up the data of the modules a delivery moves before it asks or sends anything (ADR 0404). When its +walk's record, in `plan-moved`, names a backup that failed or stayed silent and nothing of the plan was sent, the +delivery goes from delivering to held (the observed row `backup failed`); the held reason names the machine, the +module and the backup's words, and the pull request's comment shows it. The hold does not lift by itself. `release` +of such a delivery asks the controller (`delivery-backups-again`) to take the backups again and the delivery goes on +delivering; a record from before the controller cleared the failure does not hold it again. A held delivery already +ends as covered or delivered when a later merge covers it. + +## Taking the seat over from today's module + +The gate claims the seat `mesh-delivery` beside the module `mesh-delivery`, which the catalogue still holds, unchanged. The +controller refuses a merge that removes a module that is assigned, so today's module stays in the catalogue until it is +unassigned, and goes by a handover of verbs after the merge, never by hand on a machine (novox/hq ADR 0449 §4, ADR 0450 §6): + +1. assign `mesh-delivery-gate` to the machine that holds the seat (`mesh-controller.assign`); +2. check that `mesh-controller.seats` still names `mesh-delivery`, held by the running module; +3. unassign `mesh-delivery` from that machine (`mesh-controller.unassign`) and push it (`mesh-controller.push`); +4. the seat passes: the gate reads the controller's assignments and the node-engine's have keys, finds the module on no machine, + carries its deliveries, groups and pull keys over once (the key `carried` in its `deliveries` says it did), reads the controller's + walks and the forge's open pull requests, and acts. + +Until then the gate is silent: it takes no event, writes no key, asks nothing and answers no verb but `delivery_status`, which +says why (`holds-the-seat`, `seat-waits`). + +After the handover, the removal of `modules/mesh-delivery` follows as a pull request of its own, merged by the gate; the catalogue +then has one claimant of the seat. Until then the catalogue holds exactly these two, and the names of the old module that +the handover needs, and no others: the three buckets the gate carries +over, and the event the issue tracker and the tool runner hear until the module is forgotten (`catalogue_test.go`). diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/adr350_test.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/adr350_test.go new file mode 100644 index 00000000..05075466 --- /dev/null +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/adr350_test.go @@ -0,0 +1,50 @@ +package main + +import ( + "reflect" + "strings" + "testing" + "time" +) + +// One build per change (novox/hq ADR 0350): a pull request closed unmerged has its check builds let go of, and a +// walk's record says per module whether its build reused a check's build or built, and why. + +// A pull request closed unmerged asks the controller to let go of what its checks built for its merge (§5); one +// the controller cannot answer is said, never fatal. +func TestAPullRequestClosedUnmergedLetsGoOfItsCheckBuilds(t *testing.T) { + w := newWorldOver(t, newMemStore(), newFakeController(), newFakeForge(), atCC) + cc := pr(cat, 101, headCC, "feat/claude-code", "modules/claude-code/module.json") + w.h.PullUpdated(cc) + w.h.PullClosed(cc) + if want := []string{"novox/mesh-catalog#101"}; !reflect.DeepEqual(w.ctl.forgot, want) { + t.Fatalf("closed unmerged, the controller was asked to let go of %v", w.ctl.forgot) + } + if s := w.state(IDOf(cat, headCC)); s != Stopped { + t.Fatalf("closed unmerged it is %s", s) + } + + w.ctl.down = true + du := pr(cat, 102, headDu, "feat/dunst", "modules/dunst/module.json") + w.h.PullUpdated(du) + w.h.PullClosed(du) + if s := w.state(IDOf(cat, headDu)); s != Stopped { + t.Fatalf("with the controller down, a closed delivery is %s", s) + } +} + +// The walk's record says per module what its build did, and the delivery keeps it beside its walk (§4). +func TestAWalksRecordSaysWhetherEachModuleWasReusedOrBuilt(t *testing.T) { + walk := catWalk("plan-1", mergeCC, atCC.Add(time.Minute), "claude-code", "dunst") + walk.Modules["claude-code"].Made = "reused from the check of novox/mesh-catalog#101 at head cccccccccccc " + + "(merged into trunk 553b71910000): tools sha256:aa" + walk.Modules["dunst"].Made = "built: no check of novox/mesh-catalog#101 built it" + seen := walk.Seen() + if !strings.HasPrefix(seen.Built["claude-code"], "reused from the check of novox/mesh-catalog#101") || + seen.Built["dunst"] != "built: no check of novox/mesh-catalog#101 built it" { + t.Fatalf("the delivery keeps %v", seen.Built) + } + if catWalk("plan-2", mergeCC, atCC, "dunst").Seen().Built != nil { + t.Fatal("a walk from a controller that says nothing of its builds reads as if it said something") + } +} diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/answers.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/answers.go new file mode 100644 index 00000000..9fb1a14d --- /dev/null +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/answers.go @@ -0,0 +1,672 @@ +package main + +import ( + "fmt" + "sort" + "strings" + "time" +) + +// Which merges a walk answers (novox/hq issue 362, ADR 0276). +// +// On 2026-10-10 two merges of the catalogue 18 seconds apart were walked by one walk at the later merge's +// commit, which superseded the first. This owner found the walk a delivery follows by the walk's id or by the +// commit it was merged as, final or not: the surviving walk matched the dunst delivery, which the supersession +// had already ended, so nothing gave it its word, and the claude-code delivery, merged as another commit, +// matched no walk and was held for a person after ten minutes, for a reason that named neither. +// +// A walk answers every merge it carries: the merges it names (`delivery.merges`, said by a controller that +// batches merges), the merge of its own commit, a merge whose walk it took over, and a merge of its repository +// and branch the forge says its commit contains. A delivery follows the newest walk that answers it, and only a +// delivery not yet ended is ever tied to a walk by what it answers: an ended delivery is moved only by the +// record of its own walk, which no longer moves it. + +// seenWalk is a walk's last record and when it was heard. +type seenWalk struct { + walk Walk + at time.Time +} + +// keptWalkFor is how long a walk no longer open is kept for a merge heard after it ended; a batch is kept as +// long, past its window's latest close. +const keptWalkFor = 10 * time.Minute + +// answer is a delivery a walk answers and the commit of its repository the walk carries for it. +type answer struct { + d *Delivery + carried string +} + +// walkAnswers is whether a walk answers a delivery's merge, and the commit it carries for it. +func walkAnswers(w Walk, d *Delivery) (string, bool) { + if w.Release != nil || w.IsBatch() || d.MergedAs == "" { + return "", false + } + if c, ok := w.carriedFor(d.Repository, d.MergedAs); ok { + return c, true + } + if d.Walk != nil && d.Walk.TakenOverBy == w.ID && namesNoMerges(w) && strings.EqualFold(w.Repository, d.Repository) && + sameBranch(w.Branch, d.Base) { + // The walk that took its walk over, from a controller that names no merges: it answers every merge the one + // it took over answered, by its commit of the same repository and branch. A walk that names its merges + // answers only those (carriedFor, above). + return w.Commit, true + } + return "", false +} + +// containedAnswer is, for a controller that does not name the merges a walk answers, whether an open walk +// answers a ready merged delivery by containment: of its repository and branch, made for a merge after it, the +// forge says its commit contains this merge, and it moves every module this merge's plan moves. A later walk that +// moves only what its own merge moved does not deliver this one, however much of the trunk it contains. +func (h *Holder) containedAnswer(w Walk, d *Delivery) (string, bool) { + if !h.mayContain(w, d) || !d.ContainedIn[w.Commit] { + return "", false + } + return w.Commit, true +} + +// mayContain is whether an open walk is one to ask the forge about for a ready merged delivery (containedAnswer). +func (h *Holder) mayContain(w Walk, d *Delivery) bool { + if w.Release != nil || w.IsBatch() || !w.Open() || !namesNoMerges(w) || d.State != Ready || d.MergedAs == "" || d.Walk != nil || + d.Plan == nil || len(d.Plan.Moved) == 0 || !strings.EqualFold(w.Repository, d.Repository) || + !sameBranch(w.Branch, d.Base) || sameCommit(w.Commit, d.MergedAs) { + return false + } + for _, m := range d.Plan.Moved { + if _, moves := w.Modules[m]; !moves { + return false + } + } + // Made for a merge after this one: the walk's own merge, where this owner holds it, or else the walk itself. + after := w.Created + for _, o := range h.deliveries { + if strings.EqualFold(o.Repository, w.Repository) && sameCommit(o.MergedAs, w.Commit) && !o.MergedAt.IsZero() { + after = o.MergedAt + } + } + return after.After(d.MergedAt) +} + +// namesNoMerges is whether a walk is from a controller that does not name the merges it answers. +func namesNoMerges(w Walk) bool { return w.Delivery == nil || len(w.Delivery.Merges) == 0 } + +func sameBranch(a, b string) bool { return a == "" || b == "" || strings.EqualFold(a, b) } + +// follows is whether a delivery takes a walk that answers it as the one it follows: it has none, it is that walk, +// that walk took its own over, or that walk is newer than the one it follows. A walk record kept before its +// creation was read is followed past only once it is superseded or failed. +func follows(d *Delivery, w Walk) bool { + switch { + case d.Walk == nil || d.Walk.ID == w.ID || d.Walk.TakenOverBy == w.ID: + return true + case d.Walk.Created.IsZero(): + return d.Walk.State == walkSuperseded || d.Walk.State == walkFailed + } + return w.Created.After(d.Walk.Created) +} + +// deliveriesWalking are the deliveries a walk record moves: every delivery that already follows it — ended or +// not, its own record — and every delivery not ended that it answers and that follows it now. In id order. +func (h *Holder) deliveriesWalking(w Walk) []answer { + var out []answer + for _, d := range h.deliveries { + if d.Walk != nil && d.Walk.ID == w.ID { + carried := d.AnsweredBy + if c, ok := walkAnswers(w, d); ok { + carried = c + } + out = append(out, answer{d: d, carried: carried}) + continue + } + if d.State.Final() || !follows(d, w) { + continue + } + if c, ok := walkAnswers(w, d); ok { + out = append(out, answer{d: d, carried: c}) + } + } + sort.Slice(out, func(i, j int) bool { return out[i].d.ID < out[j].d.ID }) + return out +} + +// accounted is whether any delivery, ended or not, follows the walk or is answered by it: a walk known for +// what it answers is never made a delivery of its own. +func (h *Holder) accounted(w Walk) bool { + for _, d := range h.deliveries { + if d.Walk != nil && d.Walk.ID == w.ID { + return true + } + if _, ok := walkAnswers(w, d); ok { + return true + } + } + return false +} + +// held is whether a live delivery follows the walk, or a live delivery it answers follows a newer one. A walk whose +// word only stops that lapsed give (lapsedOn, novox/hq issue 753) is not held: it stays under the orphans' timer, so +// past the grace without its word it is listed by `stalled`, saying why it was not given. +func (h *Holder) heldWalk(w Walk) bool { + for _, d := range h.deliveries { + if d.State.Final() { + continue + } + if d.Walk != nil && d.Walk.ID == w.ID { + return true + } + if _, ok := walkAnswers(w, d); ok { + return true + } + } + return false +} + +// pickWalk is the newest walk heard that answers a merged delivery with no walk yet; nil for none. +func (h *Holder) pickWalk(d *Delivery) (Walk, string, bool) { + var best Walk + var carried string + found := false + for _, s := range h.walks { + if !follows(d, s.walk) { + continue + } + c, ok := walkAnswers(s.walk, d) + if !ok { + c, ok = h.containedAnswer(s.walk, d) + } + if !ok { + continue + } + if !found || s.walk.Created.After(best.Created) || s.walk.Created.Equal(best.Created) && s.walk.ID > best.ID { + best, carried, found = s.walk, c, true + } + } + return best, carried, found +} + +// linkAnswering ties a merged delivery with no walk to the newest walk heard that answers it; under the lock. +func (h *Holder) linkAnswering(d *Delivery) bool { + if d.State.Final() || d.MergedAs == "" || d.Walk != nil { + return false + } + w, carried, ok := h.pickWalk(d) + if !ok { + return false + } + h.linkWalk(d, w, carried) + delete(h.unmatched, w.ID) + return true +} + +// containsAsks are the forge's asks whether an open walk of a merged delivery's repository and branch, at +// another commit, contains its merge: for a controller that does not name the merges a walk answers (issue 362), +// asked once per pair, the answer kept on the delivery. +func (h *Holder) containsAsks(d *Delivery, now time.Time) []func() { + var asks []func() + for _, s := range h.walks { + w := s.walk + if !h.mayContain(w, d) { + continue + } + if _, known := d.ContainedIn[w.Commit]; known { + continue + } + if a := h.askContains(d, w.Commit, now); a != nil { + asks = append(asks, a) + } + } + return asks +} + +// batchOf is the batch a merged delivery's merge is in, when the controller said one; false for none. +func (h *Holder) batchOf(d *Delivery) (Walk, bool) { + if d.MergedAs == "" { + return Walk{}, false + } + for _, s := range h.walks { + if !s.walk.IsBatch() || s.walk.Delivery == nil { + continue + } + for _, m := range s.walk.Delivery.Merges { + if strings.EqualFold(m.Repository, d.Repository) && sameCommit(m.Commit, d.MergedAs) { + return s.walk, true + } + } + } + return Walk{}, false +} + +// batchSaid is what a batch holding a delivery's merge waits for, in words. +func batchSaid(b Walk, now time.Time) string { + said := "its batch " + b.ID + ": " + b.State + bt := b.Delivery.Batch + switch { + case b.State == walkQueued && bt != nil && bt.Behind != "": + said += " behind " + bt.Behind + case b.State == walkAssembling && bt != nil && bt.ClosesAt != nil: + left := bt.ClosesAt.Sub(now).Round(time.Second) + if left < 0 { + left = 0 + } + said += fmt.Sprintf(", %s left", left) + if bt.AtMost != nil { + said += " (at the latest " + bt.AtMost.Local().Format("15:04:05") + ")" + } + } + return said + "; plan not yet calculated" +} + +// unwalkedWhy is why a merged delivery no walk answers is held: the open walks of its repository and that none +// answers its merge, or that the controller holds none — what happened, not what might have. +func (h *Holder) unwalkedWhy(d *Delivery) string { + var open []string + for _, s := range h.walks { + w := s.walk + if w.Release == nil && !w.IsBatch() && w.Open() && strings.EqualFold(w.Repository, d.Repository) { + open = append(open, fmt.Sprintf("%s at %s", w.ID, shortOf(w.Commit, 8))) + } + } + sort.Strings(open) + head := fmt.Sprintf("merged as %s, and no walk answers its merge within %s: ", shortOf(d.MergedAs, 8), unwalked) + if len(open) == 0 { + return head + "the controller holds no open walk of " + d.Repository + " — its merge was not heard, or " + + "nothing the mesh holds follows " + orNone(d.Base) + ": a person decides" + } + return head + "the open walk(s) of " + d.Repository + " — " + strings.Join(open, ", ") + " — name no merge of " + + "it and the forge did not say they contain it: a person decides" +} + +func orNone(s string) string { + if s == "" { + return "its branch" + } + return s +} + +// orphanLines are the waiting walks no live delivery holds, past the grace a merge is given to be heard, as +// stalled lines in the state Unheld: nothing would ever give such a walk its word (issue 362). Under the lock. +func (h *Holder) orphanLines(now time.Time) []StalledLine { + var out []StalledLine + for id, first := range h.orphans { + s, known := h.walks[id] + if !known || now.Sub(first) < matchGrace { + continue + } + w := s.walk + if len(h.lapsedOn(w)) > 0 { + // Owed its word by stops that lapsed, and not given it past the grace (review of issue 753): said, why. + says := fmt.Sprintf("the walk %s of %s at %s waits for this owner's word, which the stopped deliveries whose "+ + "merge it answers owe it (novox/hq issue 753), and it has not started", w.ID, w.Repository, shortOf(w.Commit, 8)) + if why := h.lapsedSaid(w); why != "" { + says += ": " + why + } + out = append(out, StalledLine{ID: IDOf(w.Repository, w.Commit), State: Published, Walk: w.ID, + For: now.Sub(first).Round(time.Second).String(), Bound: matchGrace.String(), + H2: "none: its word is given at its turn — the operator's past the bound", Says: says}) + continue + } + out = append(out, StalledLine{ID: IDOf(w.Repository, w.Commit), State: Unheld, Walk: w.ID, + For: now.Sub(first).Round(time.Second).String(), Bound: matchGrace.String(), + H2: "none: no delivery holds the walk, so nothing gives it its word — the operator's", + Says: fmt.Sprintf("the walk %s of %s at %s waits for this owner's word, and no delivery that has not ended "+ + "follows it or is answered by it: `mesh-delivery.show` for the deliveries of %s, and `plans go %s --why …` "+ + "starts it by hand", w.ID, w.Repository, shortOf(w.Commit, 8), w.Repository, w.ID)}) + out[len(out)-1].Says += h.endedSaid(w) + if hold := h.coverHeld(id); hold != "" { + out[len(out)-1].Says += "; it is held back meanwhile: " + hold + } + } + return out +} + +// Unheld is a waiting walk no live delivery holds: not a state of the table — only what a stalled line says. +const Unheld State = "unheld" + +// seeOrphans keeps when each waiting walk no live delivery holds was first seen so, and forgets the rest; a +// walk still unmatched waits for its merge or its adoption instead. Under the lock. +func (h *Holder) seeOrphans(now time.Time) { + for id, s := range h.walks { + w := s.walk + _, unmatched := h.unmatched[id] + if unmatched || w.Release != nil || w.IsBatch() || !w.Open() || !w.Seen().Waits || h.heldWalk(w) { + delete(h.orphans, id) + continue + } + if _, was := h.orphans[id]; !was { + h.orphans[id] = now + h.Logf("[mesh-delivery] the walk %s waits for this owner's word and no live delivery holds it: said once "+ + "it is %s so", id, matchGrace) + } + } + for id := range h.orphans { + if _, known := h.walks[id]; !known { + delete(h.orphans, id) + } + } +} + +// forgetWalks drops a walk record no longer open after keptWalkFor, and a batch heard of that long past its +// window's latest close. Under the lock. +func (h *Holder) forgetWalks(now time.Time) { + for id, s := range h.walks { + w := s.walk + switch { + case w.IsBatch(): + end := s.at + if w.Delivery != nil && w.Delivery.Batch != nil && w.Delivery.Batch.AtMost != nil && w.Delivery.Batch.AtMost.After(end) { + end = *w.Delivery.Batch.AtMost + } + if now.Sub(end) > 3*time.Hour { + delete(h.walks, id) + } + case !w.Open() && now.Sub(s.at) > keptWalkFor: + delete(h.walks, id) + delete(h.deliverErr, id) + } + } +} + +// --- the word, once per walk ------------------------------------------------------------------------------ + +// turnWant is a walk whose turn came for some of the deliveries it carries, and why. +type turnWant struct { + ids []string + whys []string +} + +// endedSaid names the ended deliveries a waiting walk answers or that follow it, with their states, so a walk no live +// delivery holds says what it carries instead of only that nothing holds it (novox/hq issue 753); empty for none. +func (h *Holder) endedSaid(w Walk) string { + var ended []string + for _, d := range h.deliveries { + if !d.State.Final() { + continue + } + _, answers := walkAnswers(w, d) + if answers || d.Walk != nil && d.Walk.ID == w.ID { + ended = append(ended, fmt.Sprintf("%s (%s)", d.ID, d.State)) + } + } + if len(ended) == 0 { + return "" + } + sort.Strings(ended) + return "; every delivery it carries ended: " + strings.Join(ended, ", ") +} + +// lapsedOn are the deliveries, by id, whose word an open walk waiting for this owner's word takes from a stop that +// lapsed (novox/hq issue 753): each was stopped by its group for a member delivered since, so its own change waits for +// a later walk of its repository that contains its merge (issue 747), and this walk answers that merge — it names it, +// or is made at it — and has not left it out. Such a walk is the one that delivery waits for: it gives it its word, so +// a walk whose every other delivery ended is never left without one. A walk at the merge of a delivery stopped for +// itself, or by a person, gets nothing from it. Under the lock. +// +// No member is ordered before another (novox/hq ADR 0418 §2): each such delivery gives its word at once. +func (h *Holder) lapsedOn(w Walk) []string { + if w.Release != nil || w.IsBatch() || !w.Open() || !w.Seen().Waits { + return nil + } + var out []string + for _, d := range h.deliveries { + if h.lapsedAnswered(w, d) { + out = append(out, d.ID) + } + } + sort.Strings(out) + return out +} + +// lapsedAnswered is whether a delivery's stop lapsed and the walk answers its merge, not left out. +func (h *Holder) lapsedAnswered(w Walk, d *Delivery) bool { + if d.MergedAs == "" || !h.stopLapsed(d) { + return false + } + if _, ok := walkAnswers(w, d); !ok { + return false + } + _, left := w.leftOut(d.Repository, d.MergedAs) + return !left +} + +// lapsedSaid is why a walk whose word stops that lapsed give still waits: the members of their groups not yet +// delivered before them, what the last look held it for, or the controller's refusal of the word. Under the lock. +func (h *Holder) lapsedSaid(w Walk) string { + var parts []string + if why := h.waitsWhy[w.ID]; why != "" { + parts = append(parts, "not let go yet: "+why) + } + if err := h.deliverErr[w.ID]; err != "" { + parts = append(parts, "the controller refused its word: "+err) + } + for _, id := range h.lapsedOn(w) { + if d := h.deliveries[id]; d != nil && strings.Contains(d.LetGoWhy, lapsedWord(w.ID)) { + parts = append(parts, fmt.Sprintf("its word was given for %s at %s, and the controller has not started it", + id, d.LetGo.Local().Format("15:04:05"))) + } + } + return strings.Join(parts, "; ") +} + +// lapsedOwed is whether a walk's word is owed by stops that lapsed. +func (h *Holder) lapsedOwed(w Walk) bool { + return len(h.lapsedOn(w)) > 0 +} + +// lapsedTurns registers, for every open walk waiting for this owner's word, the turn of each delivery whose stop +// lapsed and whose merge it answers (lapsedOn, novox/hq issue 753). The word is then given as for any walk: once, +// when every live delivery it carries is at its turn too and it carries no stopped change still held back (words). +// Given once per walk: the walk's id is in the delivery's LetGoWhy once its word was given. Under the lock. +func (h *Holder) lapsedTurns() { + for id, s := range h.walks { + for _, did := range h.lapsedOn(s.walk) { + d := h.deliveries[did] + if strings.Contains(d.LetGoWhy, lapsedWord(id)) { + continue + } + if h.turning == nil { + h.turning = map[string]*turnWant{} + } + want := h.turning[id] + if want == nil { + want = &turnWant{} + h.turning[id] = want + } + if !containsString(want.ids, d.ID) { + want.ids = append(want.ids, d.ID) + want.whys = append(want.whys, fmt.Sprintf("%s: %s its merge %s, stopped by its group for %s, whose change "+ + "was delivered since; it waits for a later walk of %s that contains its merge, and this is that "+ + "walk (novox/hq issue 753)", d.ID, lapsedWord(id), shortOf(d.MergedAs, 8), d.stoppedFor(), d.Repository)) + } + } + } +} + +// lapsedWord is how the word a lapsed stop gives a walk names that walk. +func lapsedWord(walk string) string { return "the walk " + walk + " answers" } + +// words are the asks of the word of every walk whose turn came for every live delivery it carries: one ask per +// walk, however many deliveries it carries, and none while one of them is not at its turn (ADR 0276 decision 5). +// Under the lock. +func (h *Holder) words() []func() { + walks := make([]string, 0, len(h.turning)) + for walk := range h.turning { + walks = append(walks, walk) + } + sort.Strings(walks) + var asks []func() + h.waitsWhy = map[string]string{} + for _, walk := range walks { + want := h.turning[walk] + var waiting []string + for _, d := range h.deliveries { + if d.State.Final() || d.Walk == nil || d.Walk.ID != walk || containsString(want.ids, d.ID) { + continue + } + waiting = append(waiting, fmt.Sprintf("%s (%s)", d.ID, d.State)) + } + if len(waiting) > 0 { + sort.Strings(waiting) + h.waitsWhy[walk] = strings.Join(waiting, ", ") + " carried by it not at its turn" + h.Logf("[mesh-delivery] the walk %s is not let go yet: %s", walk, h.waitsWhy[walk]) + continue + } + ids, why := append([]string(nil), want.ids...), strings.Join(want.whys, "; ") + // **Never the word for a walk that carries a stopped change** (ADR 0299, review of #221): it is asked to + // leave the merge out first (leaveStopped). + if stopped, _ := h.carriesStopped(walk); len(stopped) > 0 { + h.waitsWhy[walk] = "it carries " + strings.Join(stopped, ", ") + ", whose delivery was stopped: it is " + + "asked to leave that merge out first" + if hold := h.coverHeld(walk); hold != "" { + h.waitsWhy[walk] = hold + } + h.Logf("[mesh-delivery] the walk %s is not let go: it carries %s, whose delivery was stopped", walk, + strings.Join(stopped, ", ")) + continue + } + if h.asking[walk] { + continue // its word is being asked + } + h.asking[walk] = true + walk := walk + asks = append(asks, func() { + err := h.Controller.Deliver(walk, why) + h.mu.Lock() + defer h.mu.Unlock() + delete(h.asking, walk) + if err != nil { + if h.deliverErr == nil { + h.deliverErr = map[string]string{} + } + h.deliverErr[walk] = err.Error() // said by `stalled` past the grace (review of novox/hq issue 753) + h.Logf("[mesh-delivery] the walk %s could not be let go yet: %v", walk, err) + h.again("a walk's word the controller refused or did not answer") + return + } + delete(h.deliverErr, walk) + for _, id := range ids { + d := h.deliveries[id] + if d == nil { + continue + } + d.LetGo, d.LetGoWhy = h.Now(), why + if d.Walk != nil && d.Walk.ID == walk { + d.Walk.Waits, d.Walk.GoBy = false, byOwner + } + h.settleAndKeep(d) + } + }) + } + h.turning = nil + return asks +} + +// endedMovingNothing is whether a delivery ended delivered with nothing for a walk to move: merged, its plan +// moving no module the mesh holds, and no walk followed. +func endedMovingNothing(d *Delivery) bool { + return d.State == Delivered && d.MergedAs != "" && d.Walk == nil && d.MovesNothing() +} + +// endedAsNothingToMove are the deliveries a walk answers that ended delivered as nothing for a walk to move (novox/hq +// issues 362, 529: the nzbgeek case); empty when it answers any other. Such a walk is held by nothing: said, never +// let go by this owner. Under the lock. +func (h *Holder) endedAsNothingToMove(w Walk) []string { + var out []string + for _, d := range h.deliveries { + _, answers := walkAnswers(w, d) + if !answers && (d.Walk == nil || d.Walk.ID != w.ID) { + continue + } + if !endedMovingNothing(d) { + return nil + } + out = append(out, d.ID) + } + sort.Strings(out) + return out +} + +// WalksWaiting is `stalled` asked with `walks` (novox/hq issue 529): for each walk waiting for this owner's word, +// what holds it, in the shape of a stalled line naming the walk — each held delivery it carries (State held, its +// reason and the verb that releases it), else why its word was not given at the last look (State published), else +// that no live delivery holds it (State unheld). A walk whose word is being given is not listed. The controller's +// `plans` says these instead of pointing at a hand start; the self-check never raises a condition for them. +func (h *Holder) WalksWaiting() []StalledLine { + h.mu.Lock() + defer h.mu.Unlock() + now := h.Now() + var out []StalledLine + for id, s := range h.walks { + w := s.walk + if w.Release != nil || w.IsBatch() || !w.Open() || !w.Seen().Waits || h.asking[id] { + continue + } + waited := now.Sub(w.Created).Round(time.Second).String() + var held []StalledLine + live := false + for _, d := range h.deliveries { + if d.State.Final() || d.Walk == nil || d.Walk.ID != id { + continue + } + live = true + if d.State == Held { + held = append(held, StalledLine{ID: d.ID, Number: d.Number, State: Held, Walk: id, For: waited, + Bound: "0s", H2: "none: held for a person — the operator's", Says: HeldLine(d)}) + } + } + switch { + case len(held) > 0: + out = append(out, held...) + case h.coverHeld(id) != "": + out = append(out, StalledLine{ID: IDOf(w.Repository, w.Commit), State: Published, Walk: id, For: waited, + Bound: coverAskBound.String(), H2: "none: it is asked to leave the merge out past the bound", + Says: "held back: " + h.coverHeld(id)}) + case h.waitsWhy[id] != "": + out = append(out, StalledLine{ID: IDOf(w.Repository, w.Commit), State: Published, Walk: id, For: waited, + Bound: "0s", H2: "none: it goes at its turn", Says: "not let go yet: " + h.waitsWhy[id]}) + case !live && h.lapsedOwed(w): + // Its word comes from the stopped deliveries whose merge it answers (novox/hq issue 753): said, never + // unheld, while the controller has not taken it, with what holds it. + lapsed := h.lapsedOn(w) + says := "its word is owed by the stopped deliveries whose merge it answers, each stopped for a member " + + "delivered since (novox/hq issue 753)" + if len(lapsed) > 0 { + says = "its word is given for " + strings.Join(lapsed, ", ") + ", stopped for a member delivered since, " + + "whose merge it answers (novox/hq issue 753)" + } + if why := h.lapsedSaid(w); why != "" { + says += "; " + why + } + out = append(out, StalledLine{ID: IDOf(w.Repository, w.Commit), State: Published, Walk: id, For: waited, + Bound: "0s", H2: "none: it goes at its turn", Says: says}) + case !live: + says := "no delivery that has not ended follows it or is answered by it, so nothing gives it its word: " + + "`mesh-delivery.show` for the deliveries of " + w.Repository + if ended := h.endedAsNothingToMove(w); len(ended) > 0 { + // The nzbgeek case (novox/hq issues 362, 529): its merges' deliveries ended as nothing for a walk to + // move, yet the walk moves modules — nothing judged those, so nothing here lets it go. + says = "held by nothing: its merges' deliveries ended as nothing-to-move (" + strings.Join(ended, ", ") + + "), yet the walk moves " + strings.Join(walkModules(w), ", ") + " — nothing gives it its word" + } + out = append(out, StalledLine{ID: IDOf(w.Repository, w.Commit), State: Unheld, Walk: id, For: waited, + Bound: "0s", H2: "none: no delivery holds the walk — the operator's", Says: says + h.endedSaid(w)}) + } + } + sort.Slice(out, func(i, j int) bool { + if out[i].Walk != out[j].Walk { + return out[i].Walk < out[j].Walk + } + return out[i].ID < out[j].ID + }) + return out +} + +// walkModules are the modules a walk's record names, sorted. +func walkModules(w Walk) []string { + out := make([]string, 0, len(w.Modules)) + for m := range w.Modules { + out = append(out, m) + } + sort.Strings(out) + return out +} diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/approval.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/approval.go new file mode 100644 index 00000000..71c3fc5b --- /dev/null +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/approval.go @@ -0,0 +1,703 @@ +package main + +import ( + "encoding/json" + "fmt" + "regexp" + "sort" + "strconv" + "strings" + "time" + + stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/asks" + sdk "git.novox.be/novox/mesh-sdk/go/delivery" +) + +// A merge into main waits for the operator's Approve (novox/hq ADR 0398 §3, issue 948, ADR 0421). +// +// This owner writes the ask into its bucket asks (novox/hq ADR 0437 §6) as soon as a pull request's checks pass: +// Approve or Decline, each option binding the act "merge these heads". The teller publishes it to the operator on the +// router and, on a warrant that holds the ask (its asker, the ask's digest, an option it offered, in time, and the act +// that option bound), writes the answer into its bucket answers. This owner reads the answer and acts on it only while +// the heads of the key it was written under are still its group's heads; it asks the router nothing. +// +// - Asked once per head. A group is asked once, for all its members' heads together, by the first member that reaches +// this point; the Approve covers every member at those heads. +// - A push after the ask ends it, said once on the pull request: the new head is asked on its own. +// - A Decline merges nothing and is said once on the pull request. +// - An ask that ended without a choice (expired, cancelled, refused by the router) is asked again when the router says so. +// - The ask, its state and the warrant's words are kept on the delivery's ReadyRun and shown by `show`. +// +// The terminal Approve (ADR 0398 §3) is issue 953; today the proving channel is the operator's phone. + +// Module is this module's name: the one the bus names its buckets and events under (novox/hq ADR 0437 §3). Seat is +// the seat it claims, whose name stays `mesh-delivery` (ADR 0437 §11). +const ( + Module = "mesh-delivery-gate" + Seat = "mesh-delivery" +) + +// The two answers, both at the level approve. +const ( + OptionApprove = "approve" + OptionDecline = "decline" +) + +// AskLasts is how long the operator has to answer: a day less a margin, under the router's bound for an approval. +const AskLasts = asks.ApproveLasts - 10*time.Minute + +// Operator is how this owner asks the operator: through the keys of its bucket asks and the teller's answers. kvOperator in +// the running module. +type Operator interface { + // Ask writes the ask under its key (sdk.MergeAskKey); it writes nothing when that key holds the same ask already. + Ask(key string, a sdk.Ask) error + // Cancel takes the ask at a key away, which the teller reads as the ask withdrawn. + Cancel(key string) error + // Where is the teller's answer to the ask at a key, or nil when there is none. + Where(key string) (*sdk.Answer, error) +} + +// askGroup is the name a delivery's ask and merge are kept under: its group's, or, for a lone pull request, a group of one +// under its branch name (novox/hq ADR 0437 §6). +func askGroup(d *Delivery) string { + switch { + case d.Group != "": + return d.Group + case d.Branch != "": + return d.Branch + } + return d.ID +} + +// headsString is the heads an act names, as an act's value: "=" for each, sorted, joined by commas. +func headsString(heads map[string]string) string { + var out []string + for id, sha := range heads { + out = append(out, id+"="+sha) + } + sort.Strings(out) + return strings.Join(out, ",") +} + +// mergeAct is what an answer binds: this ask, the answer, and the heads it covers. +func mergeAct(ask, option string, heads map[string]string) asks.Act { + return asks.Act{"verb": Seat + ".merge", "ask": ask, "option": option, "heads": headsString(heads)} +} + +var unshownPath = regexp.MustCompile(`[/\\@~$]|\d+\.\d+\.\d+|[A-Za-z0-9+/=_]{24,}|(?i)\b[a-z0-9-]+\.[a-z]{2,12}\b`) + +// phone is a line the operator's phone may be shown: what the router refuses in an ask's words (a path, an address, a +// host-like name, a long unbroken word) is left out, and the line is cut to n characters. +func phone(s string, n int) string { + s = strings.Join(strings.Fields(unshownPath.ReplaceAllString(s, "…")), " ") + if r := []rune(s); len(r) > n { + return string(r[:n-1]) + "…" + } + return s +} + +var issueNamed = regexp.MustCompile(`(?i)\bissues?\s*#?(\d+)`) + +// IssuesIn reads the issues a pull request's title and body name ("issue 948"), ascending, once each. +func IssuesIn(text string) []int { + seen := map[int]bool{} + var out []int + for _, m := range issueNamed.FindAllStringSubmatch(text, -1) { + if n, err := strconv.Atoi(m[1]); err == nil && !seen[n] { + seen[n] = true + out = append(out, n) + } + } + sort.Ints(out) + return out +} + +func issuesSaid(ds []*Delivery) string { + seen := map[int]bool{} + var out []string + for _, d := range ds { + for _, n := range d.Issues { + if !seen[n] { + seen[n] = true + out = append(out, "issue "+strconv.Itoa(n)) + } + } + } + if len(out) == 0 { + return "none named" + } + return strings.Join(out, ", ") +} + +func repoName(repository string) string { + _, name, ok := strings.Cut(repository, "/") + if !ok { + return repository + } + return name +} + +func pullSaid(d *Delivery) string { return fmt.Sprintf("%s#%d", repoName(d.Repository), d.Number) } + +// composeMergeAsk is the ask for a delivery, or for a group's members together. +func composeMergeAsk(id string, d *Delivery, group string, members []*Delivery, heads map[string]string, now time.Time) asks.Ask { + var a asks.Ask + a.ID, a.Who, a.Expires = id, asks.Operator, now.Add(AskLasts) + a.OnExpiry = "nothing is merged; the ask is made again" + a.About = "delivery." + phone(repoName(d.Repository), 40) + "." + strconv.Itoa(d.Number) + a.About = strings.ReplaceAll(a.About, " ", "-") + var reviewers, reaches, details []string + unreviewed := "" + for _, m := range members { + words, ok := reviewWords(m) + if !ok && unreviewed == "" { + unreviewed = pullSaid(m) + " is " + words + } + reviewers = append(reviewers, fmt.Sprintf("%s: %s", pullSaid(m), words)) + plan := "no delivery plan yet" + if m.Plan != nil && m.Plan.Summary != "" { + plan = m.Plan.Summary + } + reaches = append(reaches, plan) + details = append(details, fmt.Sprintf("%s %q at %s: issues: %s; review: %s; reaches: %s", + pullSaid(m), m.Title, shortOf(m.Commit, 12), issuesSaid([]*Delivery{m}), words, plan)) + } + if group == "" { + a.Headline = phone("Merge into main: "+pullSaid(d)+" "+d.Title, asks.HeadlineLength) + summary := fmt.Sprintf("%s %s passed its checks. Approve merges it into main at %s; Decline leaves it unmerged.", + pullSaid(d), d.Title, shortOf(d.Commit, 12)) + if unreviewed != "" { + summary = strings.ToUpper(unreviewed[:1]) + unreviewed[1:] + ". " + summary + } + a.Explanation = strings.Join([]string{ + phone(summary, asks.SummaryLength), + phone("Issues: "+issuesSaid(members), asks.PointLength), + phone("Review: "+reviewers[0], asks.PointLength), + phone("Reaches: "+reaches[0], asks.PointLength), + }, "\n") + } else { + var names []string + for _, m := range members { + names = append(names, pullSaid(m)) + } + a.Headline = phone("Merge into main: group "+group+" ("+strings.Join(names, ", ")+")", asks.HeadlineLength) + summary := fmt.Sprintf("The group %s, %d pull requests, passed its checks. Approve merges all of them into main in one pass, at the heads named; Decline leaves them unmerged.", + group, len(members)) + if unreviewed != "" { + summary = strings.ToUpper(unreviewed[:1]) + unreviewed[1:] + ". " + summary + } + a.Explanation = strings.Join([]string{ + phone(summary, asks.SummaryLength), + phone("Issues: "+issuesSaid(members), asks.PointLength), + phone("Review: "+strings.Join(reviewers, "; "), asks.PointLength), + phone("Reaches: "+strings.Join(reaches, "; "), asks.PointLength), + }, "\n") + } + a.Details = phone(strings.Join(details, "\n"), 1500) + if len(details) > 0 { // keep the lines apart: phone joined them + a.Details = strings.Join(func() []string { + var out []string + for _, l := range details { + out = append(out, phone(l, 400)) + } + return out + }(), "\n") + } + for _, o := range []struct{ id, label, does string }{ + {OptionApprove, "Approve", "the pull request is merged into main at the head named (a group: all its members, in one pass)"}, + {OptionDecline, "Decline", "nothing is merged"}, + } { + binds, _ := asks.ActDigest(mergeAct(id, o.id, heads)) + a.Options = append(a.Options, asks.Option{ID: o.id, Label: o.label, Does: o.does, Level: asks.Approve, Binds: binds}) + } + return a +} + +// askMembers are the deliveries one ask covers: the group's members not yet merged, by id, or the delivery. +func (h *Holder) askMembers(d *Delivery) []*Delivery { + g := h.groups[d.Group] + if d.Group == "" || g == nil { + return []*Delivery{d} + } + var out []*Delivery + seen := map[string]bool{} + ids := append([]string(nil), g.Members...) + sort.Strings(ids) + for _, id := range ids { + if o := h.deliveries[id]; o != nil && !seen[id] && o.MergedAs == "" && !o.State.Final() { + seen[id] = true + out = append(out, o) + } + } + if len(out) == 0 { + return []*Delivery{d} + } + return out +} + +// covering is the ready run holding an ask (pending or approved) that covers d at its head: its own, or a group +// sibling's, whose ask named d's head too. +func (h *Holder) covering(d *Delivery) *ReadyRun { + has := func(o *Delivery) *ReadyRun { + if o != nil && o.Ready != nil && (o.Ready.AskID != "" || o.Ready.Approved != nil) && o.Ready.Heads[d.ID] == d.Commit { + return o.Ready + } + return nil + } + if r := has(d); r != nil { + return r + } + if g := h.groups[d.Group]; d.Group != "" && g != nil { + for _, id := range g.Members { + if r := has(h.deliveries[id]); r != nil { + return r + } + } + } + return nil +} + +func headsEqual(a, b map[string]string) bool { + if len(a) != len(b) { + return false + } + for k, v := range a { + if b[k] != v { + return false + } + } + return true +} + +// headsHold is whether an ask's heads are still the heads of d's group: every member now at the head asked, and every +// head asked that is no member now a member already merged. +func (h *Holder) headsHold(d *Delivery, c *ReadyRun) bool { + now := map[string]bool{} + for _, m := range h.askMembers(d) { + now[m.ID] = true + if c.Heads[m.ID] != m.Commit { + return false + } + } + for id := range c.Heads { + if !now[id] { + if o := h.deliveries[id]; o == nil || o.MergedAs == "" { + return false + } + } + } + return true +} + +// endStaleAsks ends every ask whose heads are no longer its group's: a push to any member ends the approval (ADR 0398 +// §3), and says so once on each pull request it covered (ADR 0421 §6). An ask for a head a newer one replaced ends too. +// A Decline stands, and is not ended. Under the lock; the cancels run with the look's asks. +func (h *Holder) endStaleAsks() { + for _, d := range h.deliveries { + r := d.Ready + if r == nil || r.Merged != nil || d.MergedAs != "" || r.Declined != nil || (r.AskID == "" && r.Approved == nil) || len(r.Heads) == 0 { + continue + } + switch { + case d.State.Final(): + h.cancels = append(h.cancels, h.dropAsk(r.AskID, "its head "+shortOf(d.Commit, 12)+" is no longer the pull request's head, or the pull request closed")) + case !h.headsHold(d, r): + h.cancels = append(h.cancels, h.dropAsk(r.AskID, "a head it covers moved on: the heads it covers are no longer the group's heads")) + case r.Approved == nil && r.AskID != "" && !r.AskedAt.IsZero() && !h.Now().Before(r.AskedAt.Add(AskLasts)): + // The ask lapsed at the router with nobody's answer, and nothing in the gate's inputs says so: it is taken away and + // written again, which the teller asks afresh (ADR 0441 §1). + h.cancels = append(h.cancels, h.withdrawAsk(r.AskID, "nobody answered in "+AskLasts.String())) + } + } +} + +// approval is what a delivery that may merge waits on for the operator: empty once its head is approved, and the ask +// to make when none is made yet. Under the lock. +func (h *Holder) approval(d *Delivery, now time.Time) (string, func()) { + r := d.Ready + c := h.covering(d) + if c != nil { + if c.Approved != nil { + r.Approved, r.Warrant, r.AskID, r.Ask, r.Heads = c.Approved, c.Warrant, c.AskID, c.Ask, c.Heads + return "", nil + } + if !c.AskedAt.IsZero() { + h.lookAt(c.AskedAt.Add(AskLasts), "the operator's Approve, unanswered until the ask lapses") + } + return "the operator's Approve is asked (" + c.AskID + ")", nil + } + if r.Refused != "" { + return "the operator's answer was refused here: " + r.Refused, nil + } + if h.Operator == nil { + return "the operator cannot be asked from here", nil + } + members := h.askMembers(d) + group := "" + if len(members) > 1 { + group = d.Group + } + heads := map[string]string{} + var covered []sdk.Head + for _, m := range members { + heads[m.ID] = m.Commit + covered = append(covered, sdk.Head{Repository: m.Repository, Number: m.Number, Head: m.Commit}) + } + name := askGroup(d) + id, err := sdk.MergeAskKey(name, covered) + if err != nil { + return "the operator cannot be asked: " + err.Error(), nil + } + r.Asks++ + // Each asking at these heads has its own router id (sdk.AskingID): the router keeps an ask by its id and drops a second ask + // under it as a redelivery, so an ask written again at unchanged heads (a lapse, a status or the review record changing) is + // the next asking, with a new id (review round 1, G2; issue 1370). + asking := 1 + for _, m := range members { + if m.Ready != nil && m.Ready.AskedKey == id { + asking = max(asking, m.Ready.Asking+1) + } + } + ask := composeMergeAsk(sdk.AskingID(id, asking), d, group, members, heads, now) + taken := []*Delivery{} + var reads []memberRead + for _, m := range members { + h.runOf(m, now) + m.Ready.AskID, m.Ready.Ask, m.Ready.Heads, m.Ready.AskedAt, m.Ready.Asks = id, &ask, heads, now, max(m.Ready.Asks, r.Asks) + m.Ready.AskedKey, m.Ready.Asking = id, asking + h.keep(m) + taken = append(taken, m) + reads = append(reads, readOf(m, m.Commit)) + } + h.Logf("[mesh-delivery] %s passed its checks: the operator is asked to Approve its merge into main (%s, ask %d)", + d.ID, id, r.Asks) + gid := "" + if group != "" { + gid = d.Group + } + return "the operator's Approve is asked (" + id + ")", func() { + // A group is not asked while a member's required status is not a success at its head (issue 1275): the group + // says which, and is asked once every member may merge. + var holdMember, holdSays string + var readErr error + for _, m := range reads { + if gid == "" { + break + } + says, _, err := h.notASuccess(m) + if err != nil { + readErr = err + break + } + if says != "" { + holdMember, holdSays = m.id, says + break + } + } + undo := func() { + for _, m := range taken { + if m.Ready != nil && m.Ready.AskID == id { + m.Ready.AskID, m.Ready.Ask, m.Ready.Heads = "", nil, nil + h.keep(m) + } + } + } + // An earlier asking of this key may still stand in the bucket: not written again, it is taken away. + stale := func() { + h.mu.Lock() + standing := h.standing(id) + h.mu.Unlock() + if standing { + if err := h.Operator.Cancel(id); err != nil { + h.Logf("[mesh-delivery] the operator's ask %s could not be withdrawn (%v)", id, err) + return + } + h.mu.Lock() + h.markWritten(id, false) + h.mu.Unlock() + } + } + if readErr != nil || holdSays != "" { + stale() + h.mu.Lock() + defer h.mu.Unlock() + undo() + if g := h.groups[gid]; g != nil && holdSays != "" { + if g.Hold == nil || g.Hold.Kind != "unasked" || g.Hold.Says != holdSays || g.Hold.Member != holdMember { + h.Logf("[mesh-delivery] group %s is not asked for its Approve: %s waits on a required status: %s", gid, holdMember, holdSays) + g.Hold = &GroupHold{Kind: "unasked", Member: holdMember, Says: holdSays, Since: h.Now()} + h.keepGroup(g) + } + } + if readErr != nil { + h.Logf("[mesh-delivery] group %s: the required statuses cannot be read, so the operator is not asked yet: %v", gid, readErr) + } + h.again("a group's Approve held for a required status") + return + } + if g := func() *Group { h.mu.Lock(); defer h.mu.Unlock(); return h.groups[gid] }(); g != nil && g.Hold != nil && g.Hold.Kind == "unasked" { + h.mu.Lock() + g.Hold = nil + h.keepGroup(g) + h.mu.Unlock() + } + if err := h.Operator.Ask(id, sdk.Ask{Group: name, Heads: covered, Asking: asking, Ask: ask}); err == nil { + h.mu.Lock() + h.markWritten(id, true) + h.mu.Unlock() + } else { + h.mu.Lock() + defer h.mu.Unlock() + undo() + h.again("the operator's Approve, which could not be asked") + h.Logf("[mesh-delivery] the operator could not be asked about %s (%v); asked again in %s", d.ID, err, askAgainAfter) + } + } +} + +// dropAsk forgets a pending ask on every delivery it covers and takes it back from the router; under the lock. What it +// says is said once on each pull request it covered (ADR 0421 §6): the Approve is ended, and why. +func (h *Holder) dropAsk(id, why string) func() { + var said []func() + seen := map[string]bool{} + var covered []*Delivery + for _, d := range h.deliveries { + if d.Ready != nil && d.Ready.AskID == id { + covered = append(covered, d) + } + } + sort.Slice(covered, func(i, j int) bool { return covered[i].ID < covered[j].ID }) + for _, d := range covered { + d.Ready.AskID, d.Ready.Ask, d.Ready.Heads = "", nil, nil + if d.Ready.Merged == nil { + d.Ready.Approved, d.Ready.Warrant = nil, "" + } + h.keep(d) + if k := fmt.Sprintf("%s#%d", d.Repository, d.Number); !seen[k] && d.Number != 0 { + seen[k] = true + said = append(said, h.sayOn(d, "mesh-delivery: the operator's Approve of this pull request's merge into main is ended: "+why+ + ". Nothing is merged on it. The operator is asked again once the checks pass at the new head.")) + } + } + h.Logf("[mesh-delivery] the operator's ask %s is cancelled: %s", id, why) + return func() { + for _, s := range said { + s() + } + if h.Operator == nil { + return + } + if err := h.Operator.Cancel(id); err != nil { + h.Logf("[mesh-delivery] the operator's ask %s could not be cancelled (%v)", id, err) + return + } + h.mu.Lock() + h.markWritten(id, false) + h.mu.Unlock() + } +} + +// withdrawAsk takes back an ask nobody answered, on every delivery it covers, so the next look asks again with what is +// known now (issue 1079). Unlike dropAsk it ends no Approve and says nothing on a pull request: there was none. Under the +// lock; the router's cancel runs with the look's asks. +func (h *Holder) withdrawAsk(id, why string) func() { + for _, d := range h.deliveries { + if d.Ready != nil && d.Ready.AskID == id && d.Ready.Approved == nil { + d.Ready.AskID, d.Ready.Ask, d.Ready.Heads = "", nil, nil + h.keep(d) + } + } + // The key is not taken away: the next look writes it again with the next asking, if the group may be asked, and takes it away + // only if it may not (readyAsks, withdrawnKeys), so the key is never absent between the two. + h.withdrawn[id] = true + h.Logf("[mesh-delivery] the operator's ask %s is withdrawn to be asked again: %s", id, why) + return func() {} +} + +// withdrawnKeys are, after a look, the closures that take away the asks withdrawn and not written again by it. Under the lock. +func (h *Holder) withdrawnKeys() []func() { + var out []func() + for key := range h.withdrawn { + delete(h.withdrawn, key) + again := false + for _, d := range h.deliveries { + again = again || (d.Ready != nil && d.Ready.AskID == key) + } + if again || h.Operator == nil { + continue + } + key := key + out = append(out, func() { + if err := h.Operator.Cancel(key); err != nil { + h.Logf("[mesh-delivery] the operator's ask %s could not be withdrawn (%v)", key, err) + return + } + h.mu.Lock() + h.markWritten(key, false) + h.mu.Unlock() + }) + } + return out +} + +// pendingAsks are the asks waiting for an answer, by id; under the lock. +func (h *Holder) pendingAsks() []string { + seen := map[string]bool{} + var out []string + for _, d := range h.deliveries { + if r := d.Ready; r != nil && r.AskID != "" && r.Approved == nil && r.Declined == nil && !seen[r.AskID] { + seen[r.AskID] = true + out = append(out, r.AskID) + } + } + sort.Strings(out) + return out +} + +// hearAsks reads the teller's answer to every pending ask and applies each that is there. +func (h *Holder) hearAsks() { + if h.Operator == nil { + return + } + h.mu.Lock() + keys := h.pendingAsks() + h.mu.Unlock() + for _, key := range keys { + answer, err := h.Operator.Where(key) + if err != nil { + h.Logf("[mesh-delivery] the teller's answer to the ask %s cannot be read: %v", key, err) + continue + } + if answer == nil { + continue + } + h.mu.Lock() + said := h.answered(key, *answer) + h.mu.Unlock() + for _, s := range said { + s() + } + } +} + +// answered applies the teller's answer to the ask at a key to the deliveries it covers; under the lock. The teller has +// held the warrant to the ask (novox/hq ADR 0437 §6); this owner acts on the answer only if the key it was written under is +// still the key of the heads asked: an answer to an ask taken away, or to other heads, merges nothing (ADR 0441 §1). It +// returns what to say on the pull requests. +func (h *Holder) answered(key string, ans sdk.Answer) []func() { + var covered []*Delivery + for _, d := range h.deliveries { + if d.Ready != nil && d.Ready.AskID == key { + covered = append(covered, d) + } + } + if len(covered) == 0 { + h.Logf("[mesh-delivery] the answer to the ask %s names no ask that is asked now: nothing is merged on it", key) + return nil + } + sort.Slice(covered, func(i, j int) bool { return covered[i].ID < covered[j].ID }) + first, now := covered[0].Ready, h.Now() + refuse := func(why string) { + h.Logf("[mesh-delivery] REFUSED the answer to the ask %s: %s", key, why) + for _, d := range covered { + d.Ready.AskID, d.Ready.Ask, d.Ready.Heads = "", nil, nil + d.Ready.Refused = "was answered, and the answer does not hold here: " + clip(oneLine(why), 300) + h.keep(d) + } + } + var heads []sdk.Head + for id, sha := range first.Heads { + m := h.deliveries[id] + if m == nil { + refuse("the head " + id + " it covers is no delivery of this owner") + return nil + } + heads = append(heads, sdk.Head{Repository: m.Repository, Number: m.Number, Head: sha}) + } + if want, err := sdk.MergeAskKey(askGroup(covered[0]), heads); err != nil || want != key { + refuse(fmt.Sprintf("its key %s is not the key %s of the heads asked", key, want)) + return nil + } + // The answer is to the asking asked now: the teller names the router id it was given in the warrant's id (`@