Feature 57 (config and provisioning): modules declare config and provisions, their files rendered from templates; synced with main (issues of feature 57, 874)

This commit is contained in:
2026-10-16 07:37:43 +00:00
parent 1a2c917abf
commit 9652b2d5ce
771 changed files with 52063 additions and 1774 deletions
@@ -144,7 +144,9 @@ func (h *Holder) accounted(w Walk) bool {
return false
}
// held is whether a live delivery follows the walk, or a live delivery it answers follows a newer one.
// held is whether a live delivery follows the walk, or a live delivery it answers follows a newer one. A walk whose
// word only stops that lapsed give (lapsedOn, novox/hq issue 753) is not held: it stays under the orphans' timer, so
// past the grace without its word it is listed by `stalled`, saying why it was not given.
func (h *Holder) heldWalk(w Walk) bool {
for _, d := range h.deliveries {
if d.State.Final() {
@@ -292,12 +294,28 @@ func (h *Holder) orphanLines(now time.Time) []StalledLine {
continue
}
w := s.walk
if lapsed, behind := h.lapsedOf(w); len(lapsed) > 0 || len(behind) > 0 {
// Owed its word by stops that lapsed, and not given it past the grace (review of issue 753): said, why.
says := fmt.Sprintf("the walk %s of %s at %s waits for this owner's word, which the stopped deliveries whose "+
"merge it answers owe it (novox/hq issue 753), and it has not started", w.ID, w.Repository, shortOf(w.Commit, 8))
if why := h.lapsedSaid(w); why != "" {
says += ": " + why
}
out = append(out, StalledLine{ID: IDOf(w.Repository, w.Commit), State: Published, Walk: w.ID,
For: now.Sub(first).Round(time.Second).String(), Bound: matchGrace.String(),
H2: "none: its word is given at its turn — the operator's past the bound", Says: says})
continue
}
out = append(out, StalledLine{ID: IDOf(w.Repository, w.Commit), State: Unheld, Walk: w.ID,
For: now.Sub(first).Round(time.Second).String(), Bound: matchGrace.String(),
H2: "none: no delivery holds the walk, so nothing gives it its word — the operator's",
Says: fmt.Sprintf("the walk %s of %s at %s waits for this owner's word, and no delivery that has not ended "+
"follows it or is answered by it: `mesh-delivery.show` for the deliveries of %s, and `plans go %s --why …` "+
"starts it by hand", w.ID, w.Repository, shortOf(w.Commit, 8), w.Repository, w.ID)})
out[len(out)-1].Says += h.endedSaid(w)
if hold := h.coverHeld(id); hold != "" {
out[len(out)-1].Says += "; it is held back meanwhile: " + hold
}
}
return out
}
@@ -344,6 +362,7 @@ func (h *Holder) forgetWalks(now time.Time) {
}
case !w.Open() && now.Sub(s.at) > keptWalkFor:
delete(h.walks, id)
delete(h.deliverErr, id)
}
}
}
@@ -356,6 +375,168 @@ type turnWant struct {
whys []string
}
// endedSaid names the ended deliveries a waiting walk answers or that follow it, with their states, so a walk no live
// delivery holds says what it carries instead of only that nothing holds it (novox/hq issue 753); empty for none.
func (h *Holder) endedSaid(w Walk) string {
var ended []string
for _, d := range h.deliveries {
if !d.State.Final() {
continue
}
_, answers := walkAnswers(w, d)
if answers || d.Walk != nil && d.Walk.ID == w.ID {
ended = append(ended, fmt.Sprintf("%s (%s)", d.ID, d.State))
}
}
if len(ended) == 0 {
return ""
}
sort.Strings(ended)
return "; every delivery it carries ended: " + strings.Join(ended, ", ")
}
// lapsedOn are the deliveries, by id, whose word an open walk waiting for this owner's word takes from a stop that
// lapsed (novox/hq issue 753): each was stopped by its group for a member delivered since, so its own change waits for
// a later walk of its repository that contains its merge (issue 747), and this walk answers that merge — it names it,
// or is made at it — and has not left it out. Such a walk is the one that delivery waits for: it gives it its word, so
// a walk whose every other delivery ended is never left without one. A walk at the merge of a delivery stopped for
// itself, or by a person, gets nothing from it. Under the lock.
//
// In its group's order (review of issue 753): a member gives its word only once every member ordered before it reached
// the machines, or is itself a stop that lapsed whose merge this same walk answers — the walk then delivers both, in
// the group's order among its tiers, as members sharing one walk do. A member behind one still undelivered gives
// nothing; lapsedBehind names that one.
func (h *Holder) lapsedOn(w Walk) []string {
out, _ := h.lapsedOf(w)
return out
}
// lapsedOf is lapsedOn, and, by delivery, the member before it in its group that holds back the word of a stop that
// lapsed whose merge the walk answers. Under the lock.
func (h *Holder) lapsedOf(w Walk) ([]string, map[string]string) {
if w.Release != nil || w.IsBatch() || !w.Open() || !w.Seen().Waits {
return nil, nil
}
var out []string
behind := map[string]string{}
for _, d := range h.deliveries {
if !h.lapsedAnswered(w, d) {
continue
}
if before := h.lapsedBehind(w, d); before != "" {
behind[d.ID] = before
continue
}
out = append(out, d.ID)
}
sort.Strings(out)
return out, behind
}
// lapsedAnswered is whether a delivery's stop lapsed and the walk answers its merge, not left out.
func (h *Holder) lapsedAnswered(w Walk, d *Delivery) bool {
if d.MergedAs == "" || !h.stopLapsed(d) {
return false
}
if _, ok := walkAnswers(w, d); !ok {
return false
}
_, left := w.leftOut(d.Repository, d.MergedAs)
return !left
}
// lapsedBehind is the member ordered before a delivery in its group that has not reached the machines and that this
// walk does not deliver with it, with its state; empty for none.
func (h *Holder) lapsedBehind(w Walk, d *Delivery) string {
g := h.groups[d.Group]
if d.Group == "" || g == nil {
return ""
}
members := h.membersOf(g)
byID := map[string]*Delivery{}
for _, m := range members {
byID[m.ID] = m
}
for _, m := range h.inOrder(g, members) {
if m.ID == d.ID {
return ""
}
o := byID[m.ID]
if o == nil || memberReached(o) || h.lapsedAnswered(w, o) {
continue
}
return fmt.Sprintf("%s (%s)", o.ID, o.State)
}
return ""
}
// lapsedSaid is why a walk whose word stops that lapsed give still waits: the members of their groups not yet
// delivered before them, what the last tick held it for, or the controller's refusal of the word. Under the lock.
func (h *Holder) lapsedSaid(w Walk) string {
var parts []string
_, behind := h.lapsedOf(w)
ids := make([]string, 0, len(behind))
for id := range behind {
ids = append(ids, id)
}
sort.Strings(ids)
for _, id := range ids {
parts = append(parts, fmt.Sprintf("%s waits for %s, ordered before it in its group", id, behind[id]))
}
if why := h.waitsWhy[w.ID]; why != "" {
parts = append(parts, "not let go yet: "+why)
}
if err := h.deliverErr[w.ID]; err != "" {
parts = append(parts, "the controller refused its word: "+err)
}
lapsed, _ := h.lapsedOf(w)
for _, id := range lapsed {
if d := h.deliveries[id]; d != nil && strings.Contains(d.LetGoWhy, lapsedWord(w.ID)) {
parts = append(parts, fmt.Sprintf("its word was given for %s at %s, and the controller has not started it",
id, d.LetGo.Local().Format("15:04:05")))
}
}
return strings.Join(parts, "; ")
}
// lapsedOwed is whether a walk's word is owed by stops that lapsed: given now, or once a member before one is delivered.
func (h *Holder) lapsedOwed(w Walk) bool {
lapsed, behind := h.lapsedOf(w)
return len(lapsed) > 0 || len(behind) > 0
}
// lapsedTurns registers, for every open walk waiting for this owner's word, the turn of each delivery whose stop
// lapsed and whose merge it answers (lapsedOn, novox/hq issue 753). The word is then given as for any walk: once,
// when every live delivery it carries is at its turn too and it carries no stopped change still held back (words).
// Given once per walk: the walk's id is in the delivery's LetGoWhy once its word was given. Under the lock.
func (h *Holder) lapsedTurns() {
for id, s := range h.walks {
for _, did := range h.lapsedOn(s.walk) {
d := h.deliveries[did]
if strings.Contains(d.LetGoWhy, lapsedWord(id)) {
continue
}
if h.turning == nil {
h.turning = map[string]*turnWant{}
}
want := h.turning[id]
if want == nil {
want = &turnWant{}
h.turning[id] = want
}
if !containsString(want.ids, d.ID) {
want.ids = append(want.ids, d.ID)
want.whys = append(want.whys, fmt.Sprintf("%s: %s its merge %s, stopped by its group for %s, whose change "+
"was delivered since; it waits for a later walk of %s that contains its merge, and this is that "+
"walk (novox/hq issue 753)", d.ID, lapsedWord(id), shortOf(d.MergedAs, 8), d.stoppedFor(), d.Repository))
}
}
}
}
// lapsedWord is how the word a lapsed stop gives a walk names that walk.
func lapsedWord(walk string) string { return "the walk " + walk + " answers" }
// words are the asks of the word of every walk whose turn came for every live delivery it carries: one ask per
// walk, however many deliveries it carries, and none while one of them is not at its turn (ADR 0276 decision 5).
// Under the lock.
@@ -388,6 +569,9 @@ func (h *Holder) words() []func() {
if stopped, _ := h.carriesStopped(walk); len(stopped) > 0 {
h.waitsWhy[walk] = "it carries " + strings.Join(stopped, ", ") + ", whose delivery was stopped: it is " +
"asked to leave that merge out first"
if hold := h.coverHeld(walk); hold != "" {
h.waitsWhy[walk] = hold
}
h.Logf("[mesh-delivery] the walk %s is not let go: it carries %s, whose delivery was stopped", walk,
strings.Join(stopped, ", "))
continue
@@ -403,9 +587,14 @@ func (h *Holder) words() []func() {
defer h.mu.Unlock()
delete(h.asking, walk)
if err != nil {
if h.deliverErr == nil {
h.deliverErr = map[string]string{}
}
h.deliverErr[walk] = err.Error() // said by `stalled` past the grace (review of novox/hq issue 753)
h.Logf("[mesh-delivery] the walk %s could not be let go yet: %v", walk, err)
return
}
delete(h.deliverErr, walk)
for _, id := range ids {
d := h.deliveries[id]
if d == nil {
@@ -479,9 +668,28 @@ func (h *Holder) WalksWaiting() []StalledLine {
switch {
case len(held) > 0:
out = append(out, held...)
case h.coverHeld(id) != "":
out = append(out, StalledLine{ID: IDOf(w.Repository, w.Commit), State: Published, Walk: id, For: waited,
Bound: coverAskBound.String(), H2: "none: it is asked to leave the merge out past the bound",
Says: "held back: " + h.coverHeld(id)})
case h.waitsWhy[id] != "":
out = append(out, StalledLine{ID: IDOf(w.Repository, w.Commit), State: Published, Walk: id, For: waited,
Bound: "0s", H2: "none: it goes at its turn", Says: "not let go yet: " + h.waitsWhy[id]})
case !live && h.lapsedOwed(w):
// Its word comes from the stopped deliveries whose merge it answers (novox/hq issue 753): said, never
// unheld, while the controller has not taken it, with what holds it.
lapsed, _ := h.lapsedOf(w)
says := "its word is owed by the stopped deliveries whose merge it answers, each stopped for a member " +
"delivered since (novox/hq issue 753)"
if len(lapsed) > 0 {
says = "its word is given for " + strings.Join(lapsed, ", ") + ", stopped for a member delivered since, " +
"whose merge it answers (novox/hq issue 753)"
}
if why := h.lapsedSaid(w); why != "" {
says += "; " + why
}
out = append(out, StalledLine{ID: IDOf(w.Repository, w.Commit), State: Published, Walk: id, For: waited,
Bound: "0s", H2: "none: it goes at its turn", Says: says})
case !live:
says := "no delivery that has not ended follows it or is answered by it, so nothing gives it its word: " +
"`mesh-delivery.show` for the deliveries of " + w.Repository
@@ -492,7 +700,7 @@ func (h *Holder) WalksWaiting() []StalledLine {
"), yet the walk moves " + strings.Join(walkModules(w), ", ") + " — nothing gives it its word"
}
out = append(out, StalledLine{ID: IDOf(w.Repository, w.Commit), State: Unheld, Walk: id, For: waited,
Bound: "0s", H2: "none: no delivery holds the walk — the operator's", Says: says})
Bound: "0s", H2: "none: no delivery holds the walk — the operator's", Says: says + h.endedSaid(w)})
}
}
sort.Slice(out, func(i, j int) bool {
@@ -150,6 +150,9 @@ func statusOf(d *Delivery) (string, string) {
case Delivered:
return "success", "delivered"
case Failed:
if d.coveredStop() {
return "success", clip(d.coveredStopSaid(), 140) // its change reached the machines (novox/hq issue 747)
}
return "failure", clip("failed: "+why, 140)
case Superseded:
return "warning", clip("superseded: "+why, 140)
@@ -178,7 +181,7 @@ func ViewBody(d *Delivery, g *Group) string {
var b strings.Builder
fmt.Fprintf(&b, "%s\n**Delivery** `%s` — **%s** since %s\n\n", viewMarker, d.ID, d.State, d.Since.UTC().Format(time.RFC3339))
if d.coveredStop() {
fmt.Fprintf(&b, "**Covered** after its stop: a later merge that contains it was delivered: %s\n\n", d.CoveredBy)
fmt.Fprintf(&b, "**Covered**: %s\n\n", d.coveredStopSaid())
}
if d.HeldWhy != "" && d.State == Held {
fmt.Fprintf(&b, "Held for a person: %s — `mesh-delivery.release` with why lets it go on.\n\n", d.HeldWhy)
@@ -106,6 +106,8 @@ type fakeController struct {
moves map[string][]string
plans int
down bool
// refuseDeliver refuses every word, as a controller that will not take it does (review of novox/hq issue 753).
refuseDeliver bool
// onCheck runs as a check is asked, before it is answered: what the forge held at that moment.
onCheck func()
// planned is what Plan answers instead of "builds app", when set.
@@ -182,6 +184,9 @@ func (c *fakeController) Deliver(walk, why string) error {
if c.down {
return errors.New("down")
}
if c.refuseDeliver {
return errors.New("the controller refuses the word")
}
w, known := c.walks[walk]
if !known || w.Delivery == nil || w.Delivery.Go != nil {
return fmt.Errorf("%s waits for nobody", walk)
@@ -396,6 +401,9 @@ type fakeForge struct {
// open is what OpenPulls answers; openDown a forge that cannot say.
open []OpenPull
openDown bool
// merges, comments and unlabels are what was asked of the forge for ready marks (novox/hq issue 722).
merges, comments, unlabels []string
refuse error
}
func (f *fakeForge) OpenPulls() ([]OpenPull, error) {
@@ -649,3 +657,25 @@ func aWalk(id, repo, commit string, waits bool) Walk {
}
return w
}
// Merge, Comment and Unlabel record what was asked (novox/hq issue 722); Merge answers refuse when set.
func (f *fakeForge) Merge(owner, repo string, number int, head string) error {
f.mu.Lock()
defer f.mu.Unlock()
f.merges = append(f.merges, fmt.Sprintf("%s/%s#%d@%s", owner, repo, number, head))
return f.refuse
}
func (f *fakeForge) Comment(owner, repo string, number int, body string) error {
f.mu.Lock()
defer f.mu.Unlock()
f.comments = append(f.comments, fmt.Sprintf("%s/%s#%d: %s", owner, repo, number, body))
return nil
}
func (f *fakeForge) Unlabel(owner, repo string, number, label int) error {
f.mu.Lock()
defer f.mu.Unlock()
f.unlabels = append(f.unlabels, fmt.Sprintf("%s/%s#%d-%d", owner, repo, number, label))
return nil
}
+316 -26
View File
@@ -60,6 +60,8 @@ type Holder struct {
waitsWhy map[string]string
// asking are the walks whose word is being asked right now, outside the lock: not asked twice meanwhile.
asking map[string]bool
// deliverErr is, by walk, the controller's last refusal of its word, until it is given (review of issue 753).
deliverErr map[string]string
// leaving are, by walk, the merges a stop gathered to leave out of it, asked once (novox/hq ADR 0299).
leaving map[string][]string
// leaveAsked is when a walk was last asked to leave out the stopped merges it carries (leaveStopped).
@@ -67,8 +69,17 @@ type Holder struct {
// leaveRefused is, by walk, what the controller answered when it refused both to leave a stopped merge out and
// to stop the walk whole.
leaveRefused map[string]string
// coverAsked are the deliveries that failed or were stopped whose cover by a later finished walk waits for the
// forge's word on containment, worked out each pass (novox/hq issue 747): a walk carrying a merge stopped for one of
// them is held back meanwhile, never asked to leave it out before the answer.
coverAsked map[string]bool
// coverAskedSince is, by delivery, when the forge was first asked about its cover and has not answered: past
// coverAskBound the walk is no longer held back for it, and is asked to leave the merge out (review of issue 747).
coverAskedSince map[string]time.Time
// tried counts the owed effects Flush tried since this owner started, by how each ended (novox/hq issue 407).
tried owedTries
// marks are the open pull requests read with a ready mark, by owner/repo#number (novox/hq issue 722, ready.go).
marks map[string]*marked
}
// owedTries is how many owed effects were tried since this owner started, and when the last was: three numbers
@@ -120,6 +131,7 @@ func (h *Holder) init() {
h.askTried = map[string]time.Time{}
h.walkAsked = map[string]time.Time{}
h.unchecked = map[string]unchecked{}
h.marks = map[string]*marked{}
h.walks = map[string]seenWalk{}
h.orphans = map[string]time.Time{}
h.asking = map[string]bool{}
@@ -300,19 +312,25 @@ func (h *Holder) PullUpdated(p PullEvent) {
defer h.mu.Unlock()
id := IDOf(p.repository(), p.HeadSHA)
now := h.Now()
startedOver := ""
if d := h.deliveries[id]; d != nil {
// Announced again: nothing new — but for a delivery whose verdict is out of date, the announcement is its
// check asked again against the trunk's head (the merge tool's ask, novox/hq ADR 0323 §5), and its bound
// runs from now (§7). The base head it was announced with is kept, for a check this owner asks after it.
if p.BaseSHA != "" && d.BaseSHA != p.BaseSHA {
d.BaseSHA = p.BaseSHA
h.keep(d)
if startedOver = startedOverFor(d, p.Number); startedOver == "" {
// Announced again: nothing new — but for a delivery whose verdict is out of date, the announcement is its
// check asked again against the trunk's head (the merge tool's ask, novox/hq ADR 0323 §5), and its bound
// runs from now (§7). The base head it was announced with is kept, for a check this owner asks after it.
if p.BaseSHA != "" && d.BaseSHA != p.BaseSHA {
d.BaseSHA = p.BaseSHA
h.keep(d)
}
if d.State == Proposed && d.outOfDate() != "" && d.AskedAgain == nil {
d.AskedAgain = &now
h.keep(d)
}
return
}
if d.State == Proposed && d.outOfDate() != "" && d.AskedAgain == nil {
d.AskedAgain = &now
h.keep(d)
}
return
// A delivery stopped only because its pull request closed unmerged does not end its commit's delivery for
// another pull request with the same head (novox/hq issue 783): started over below, exactly as for a first
// announcement, under the same id — never a transition out of Stopped.
}
replaced := ""
for _, old := range h.deliveries {
@@ -326,11 +344,7 @@ func (h *Holder) PullUpdated(p PullEvent) {
}
}
}
d := &Delivery{ID: id, Repository: p.repository(), Commit: p.HeadSHA, Number: p.Number, Title: p.Title,
Base: p.Base, Branch: p.Head, HTMLURL: p.HTMLURL, CloneURL: p.CloneURL, Paths: p.Paths,
PathsTruncated: p.PathsTruncated, Removed: p.Removed, ModuleDirs: p.ModuleDirs, ModuleDirsSaid: p.ModuleDirsSaid,
After: AfterIn(p.Body), Created: now, BaseSHA: p.BaseSHA}
t, err := Apply(d, EvAnnounced, false, Facts{Now: now, Why: fmt.Sprintf("%s#%d's head announced", d.Repository, d.Number)})
d, t, err := h.freshAnnouncement(p, now, startedOver)
if err != nil {
h.refuse(err)
return
@@ -348,6 +362,34 @@ func (h *Holder) PullUpdated(p PullEvent) {
}
}
// startedOverFor is why a delivery already kept at a head's id is started over for a pull request's number rather
// than taken as announced again: only when it was stopped solely by an earlier, closed duplicate pull request with
// the same head — still Stopped, ClosedUnmerged, never merged, and under another number (novox/hq issue 783).
// Empty for the ordinary 'announced again' path, or for any other stop (a person's, a group's, its own close).
func startedOverFor(d *Delivery, number int) string {
if d != nil && d.State == Stopped && d.ClosedUnmerged && d.MergedAs == "" && d.Number != number {
return fmt.Sprintf("#%d with the same head closed unmerged", d.Number)
}
return ""
}
// freshAnnouncement makes a delivery for p's head exactly as a first announcement would: its own id, proposed.
// startedOver, when not empty, is folded into the transition's Why, naming the pull request this replaces
// (novox/hq issue 783); the delivery it replaces is swapped out by the caller keeping the new one under the same
// id (keep), never by a transition out of its final state.
func (h *Holder) freshAnnouncement(p PullEvent, now time.Time, startedOver string) (*Delivery, Transition, error) {
d := &Delivery{ID: IDOf(p.repository(), p.HeadSHA), Repository: p.repository(), Commit: p.HeadSHA, Number: p.Number,
Title: p.Title, Base: p.Base, Branch: p.Head, HTMLURL: p.HTMLURL, CloneURL: p.CloneURL, Paths: p.Paths,
PathsTruncated: p.PathsTruncated, Removed: p.Removed, ModuleDirs: p.ModuleDirs, ModuleDirsSaid: p.ModuleDirsSaid,
After: AfterIn(p.Body), Created: now, BaseSHA: p.BaseSHA}
why := fmt.Sprintf("%s#%d's head announced", d.Repository, d.Number)
if startedOver != "" {
why = fmt.Sprintf("%s; %s", why, startedOver)
}
t, err := Apply(d, EvAnnounced, false, Facts{Now: now, Why: why})
return d, t, err
}
// takePlaceOf puts a member's new head — a rebase, a push — in its group in the place of the head it superseded,
// when the group is closed: a member of it already reached the trunk, and a closed group takes no new member. The
// new head has its own check, and the group's composed check runs again for the new heads rather than holding the
@@ -479,7 +521,29 @@ func (h *Holder) PullMerged(p PullEvent) {
}
now := h.Now()
var ts []Transition
if d == nil {
if over := startedOverFor(d, p.Number); over != "" {
// A delivery stopped only by an earlier, closed duplicate pull request with the same head is not lost when
// its merge is heard without the open announcement either: it is started over for the merging pull request
// the same way an announcement would, before the merge is taken as usual (novox/hq issue 783). Heard this
// way — its own merge, with no open announcement of this pull request — it is made exactly as a first-heard
// merge in every field but its id: FromForge, and its check the forge's word on the head, so a long-past
// merge replayed from the bus's history still falls under the history rule (issue 313) and is not held for
// a person (review round 1).
fresh, t, err := h.freshAnnouncement(p, now, over)
if err != nil {
h.refuse(err)
return
}
fresh.FromForge = true
if s := p.HeadChecks["mesh/merge-gate"]; s != "" {
fresh.Check = &Verdict{At: now, Gate: verdictOfState(s), Summary: "the head's mesh/merge-gate, as the forge holds it",
Repo: verdictOfState(p.HeadChecks["mesh/repo-check"])}
}
d, ts = fresh, []Transition{t}
if d.Check != nil {
ts = append(ts, settle(d, Facts{Now: now})...)
}
} else if d == nil {
head := p.HeadSHA
if head == "" {
head = p.MergeCommit
@@ -996,6 +1060,12 @@ func (h *Holder) Tick() {
for _, ask := range h.leaveStopped() {
ask()
}
h.mu.Lock()
asks := h.readyAsks(h.Now())
h.mu.Unlock()
for _, ask := range asks {
ask()
}
}
// plan works out, under the lock, what to ask; each ask applies its own answer.
@@ -1103,6 +1173,48 @@ func (h *Holder) plan() []func() {
// A published delivery whose walk ended done before its word is read the same way (novox/hq issue 474): what
// that walk did not build, a later delivered merge may have; and when the cover test says none did, past the
// grace, it is stopped saying so — never left waiting for a start that cannot come.
// A delivery that failed or was stopped is read for a later finished walk of its repository that delivered its
// change after all: a walk that failed in an outage and was resumed, or the walk of a later merge that contains it
// (novox/hq issue 747). Read each tick, from what the walks and the kept deliveries say, so one stuck before this
// was built clears by itself after a restart.
h.coverAsked = map[string]bool{}
if h.coverAskedSince == nil {
h.coverAskedSince = map[string]time.Time{}
}
pending := map[string]bool{}
for _, d := range h.deliveries {
if d.State != Failed && d.State != Stopped || d.coveredStop() {
continue
}
by, ask := h.walkCover(d)
if len(ask) > 0 && by == "" {
pending[d.ID] = true
since, asked := h.coverAskedSince[d.ID]
if !asked {
since = now
h.coverAskedSince[d.ID] = now
}
// Held back only within the bound: a forge that keeps erroring never holds a walk for good.
h.coverAsked[d.ID] = now.Sub(since) < coverAskBound
}
if by != "" {
d.DeliveredBy = by
h.tellCovered(d, now)
continue
}
for _, in := range ask {
if a := h.askContains(d, in, now); a != nil {
asks = append(asks, a)
}
}
}
for id := range h.coverAskedSince {
if !pending[id] {
delete(h.coverAskedSince, id)
}
}
for _, d := range h.deliveries {
ended := d.endedUnstarted()
if d.coveredStop() && !d.CoverTold {
@@ -1186,6 +1298,8 @@ func (h *Holder) plan() []func() {
}
h.turn(d, "its turn: a delivery in no group")
}
// A walk that answers the merge of a delivery whose stop lapsed: that delivery's turn (novox/hq issue 753).
h.lapsedTurns()
// The word, once per walk, when every live delivery it carries is at its turn.
asks = append(asks, h.words()...)
@@ -1264,7 +1378,7 @@ func (h *Holder) laterOnItsWay(d *Delivery) string {
// stays final, and the bus's transition event is not said again.
func (h *Holder) tellCovered(d *Delivery, now time.Time) {
d.CoverTold = true
line := oneLine(fmt.Sprintf("%s delivery %s: stopped, %s", now.UTC().Format(time.RFC3339), d.ID, d.coveredStopSaid()))
line := oneLine(fmt.Sprintf("%s delivery %s: %s, %s", now.UTC().Format(time.RFC3339), d.ID, d.State, d.coveredStopSaid()))
d.Owed = append(d.Owed, Effect{Kind: EffectNote, Commit: d.Commit, Line: line, Since: now})
if d.MergedAs != "" && !sameCommit(d.MergedAs, d.Commit) {
d.Owed = append(d.Owed, Effect{Kind: EffectNote, Commit: d.MergedAs, Line: line, Since: now})
@@ -1274,7 +1388,7 @@ func (h *Holder) tellCovered(d *Delivery, now time.Time) {
state, desc := statusOf(d)
d.owe(Effect{Kind: EffectStatus, Commit: d.Commit, Context: "mesh/delivery", State: state, Line: desc, Since: now})
}
h.Logf("[mesh-delivery] %s: stopped, %s", d.ID, d.coveredStopSaid())
h.Logf("[mesh-delivery] %s: %s, %s", d.ID, d.State, d.coveredStopSaid())
h.keep(d)
}
@@ -1285,6 +1399,104 @@ func strandedUncovered(d *Delivery) bool {
return d.State == Stopped && d.Stranded != "" && d.CoveredBy == ""
}
// walkCover says which later finished walk delivered the change of a delivery that failed or was stopped, or
// nothing, and which walk commits the forge must still be asked about (novox/hq issue 747). A walk covers it when it
// is of its repository and trunk, done, made after its merge, its commit containing its merge — the merge itself, or
// a later one the forge says contains it — and it built every module the delivery's plan moves, each it did not
// build being one the delivery's own walk said unchanged (issue 611). This is the containment "one walk answers every
// merge it contains" uses (issue 362, ADR 0276), read after the fact: a failed walk resumed to done answers its own
// merge, and a later merge's walk answers every merge it contains. A walk that only said a module unchanged for its
// own merges says nothing of this one's, so it covers no module that way. Walks are read from the controller's records
// this owner holds and from every delivery's kept walk, so a walk long ended still counts after a restart.
func (h *Holder) walkCover(d *Delivery) (string, []string) {
if d.State != Failed && d.State != Stopped || d.MergedAs == "" || d.Plan == nil || len(d.Plan.Moved) == 0 {
return "", nil
}
var ask []string
for _, c := range h.doneWalksOf(d.Repository) {
w := c.walk
if !sameBranch(w.Branch, d.Base) || !w.Created.IsZero() && w.Created.Before(d.MergedAt.Add(-time.Minute)) {
continue
}
var missing []string
for _, m := range d.Plan.Moved {
unchanged := false
if d.Walk != nil {
_, unchanged = d.Walk.Unchanged[m]
}
if !containsString(w.Moved, m) && !unchanged {
missing = append(missing, m)
}
}
if len(missing) > 0 {
continue
}
if !sameCommit(c.commit, d.MergedAs) {
yes, known := d.ContainedIn[c.commit]
if !known && !containsString(ask, c.commit) {
ask = append(ask, c.commit)
}
if !yes {
continue
}
}
return fmt.Sprintf("%s (%s at %s into %s, done, contains its merge %s)", w.ID, d.Repository, shortOf(c.commit, 8),
orNone(d.Base), shortOf(d.MergedAs, 8)), nil
}
return "", ask
}
// doneWalk is a finished walk and the commit of its repository it carried.
type doneWalk struct {
walk *WalkSeen
commit string
}
// doneWalksOf are the finished walks of a repository this owner knows, oldest first: the controller's records it
// holds, and each delivery's kept walk with the commit of its repository that walk carried for it.
func (h *Holder) doneWalksOf(repository string) []doneWalk {
byID := map[string]doneWalk{}
for id, s := range h.walks {
w := s.walk
if w.Release != nil || w.IsBatch() || w.State != walkDone || !strings.EqualFold(w.Repository, repository) {
continue
}
byID[id] = doneWalk{walk: w.Seen(), commit: w.Commit}
}
for _, o := range h.deliveries {
if o.Walk == nil || o.Walk.State != walkDone || !strings.EqualFold(o.Repository, repository) {
continue
}
if _, known := byID[o.Walk.ID]; known {
continue
}
commit := o.AnsweredBy
if commit == "" && o.MergedAs != "" && sameCommit(o.Walk.Commit, o.MergedAs) {
commit = o.MergedAs
}
if commit == "" {
continue
}
byID[o.Walk.ID] = doneWalk{walk: o.Walk, commit: commit}
}
out := make([]doneWalk, 0, len(byID))
for _, c := range byID {
out = append(out, c)
}
sort.Slice(out, func(i, j int) bool {
if !out[i].walk.Created.Equal(out[j].walk.Created) {
return out[i].walk.Created.Before(out[j].walk.Created)
}
return out[i].walk.ID < out[j].walk.ID
})
return out
}
// stopLapsed is whether a member its group stopped for another is stopped for nothing any more (see stopLapsed).
func (h *Holder) stopLapsed(d *Delivery) bool {
return stopLapsed(d, func(id string) *Delivery { return h.deliveries[id] })
}
// unwalkedPast is whether a merged delivery has gone without a walk past the bound one opens within, read
// again after it: what holds it for a person, unless something already ends it.
func (h *Holder) unwalkedPast(d *Delivery, now time.Time) bool {
@@ -1727,7 +1939,8 @@ func (h *Holder) turnsOf(g *Group, members []*Delivery, waits map[string]carryWa
// A superseded member a later delivered walk carried counts as reached, as in GroupState: a group counts a
// covered member as delivered, so the member after it gets its turn (ADR 0270 decision 2, novox/hq
// issues 442 and 445).
if d.State.Reached() || d.State == Superseded && d.CoveredBy != "" {
// A stop or failure a later walk delivered since counts as reached too (novox/hq issues 480 and 747).
if memberReached(d) {
continue
}
if sharing != "" && (d.Walk == nil || d.Walk.ID != sharing) {
@@ -1752,7 +1965,7 @@ func (h *Holder) turnsOf(g *Group, members []*Delivery, waits map[string]carryWa
}
if w.why != "" {
for _, later := range ordered[i+1:] {
if a := h.stopInGroup(byID[later.ID], fmt.Sprintf("its group %s stopped: %s superseded, and %s", g.ID,
if a := h.stopInGroup(byID[later.ID], d.ID, fmt.Sprintf("its group %s stopped: %s superseded, and %s", g.ID,
d.ID, w.why)); a != nil {
asks = append(asks, a)
}
@@ -1779,9 +1992,13 @@ func (h *Holder) turnsOf(g *Group, members []*Delivery, waits map[string]carryWa
continue
}
return asks
case h.stopLapsed(d):
// Stopped for a member whose change reached the machines since: it stops nothing more. Its own change is
// delivered by a later walk of its repository that contains it, which then covers it (novox/hq issue 747).
continue
case d.State == Failed || d.State == Stopped:
for _, later := range ordered[i+1:] {
if a := h.stopInGroup(byID[later.ID], fmt.Sprintf("its group %s stopped: %s %s", g.ID, d.ID, d.State)); a != nil {
if a := h.stopInGroup(byID[later.ID], d.ID, fmt.Sprintf("its group %s stopped: %s %s", g.ID, d.ID, d.State)); a != nil {
asks = append(asks, a)
}
}
@@ -2019,7 +2236,7 @@ func mayCarry(c, d *Delivery, module string) bool {
// controller ends the walk and walks the other merges again without it, and without each later merge of its
// repository, which contains it. Every member a pass stops on one walk is gathered first, and the walk is asked
// once, by the first member's ask, after the pass: a member is never walked again because its stop came second.
func (h *Holder) stopInGroup(d *Delivery, why string) func() {
func (h *Holder) stopInGroup(d *Delivery, cause, why string) func() {
if d == nil || d.State.Final() {
return nil
}
@@ -2028,6 +2245,7 @@ func (h *Holder) stopInGroup(d *Delivery, why string) func() {
h.refuse(err)
return nil
}
d.StoppedFor = cause // the member it was stopped for: once that one is delivered, the stop lapses (novox/hq issue 747)
h.moved(d, []Transition{t})
// Its open walk; or, its walk stopped for another merge, the batch or walk that walks its merge again (review
// of #221): a merge walked again and then stopped by its group is left out there too.
@@ -2109,6 +2327,12 @@ func (h *Holder) carriesStopped(walk string) ([]string, string) {
if last := d.Transitions[len(d.Transitions)-1]; last.Event != EvStop || last.By == "" {
continue
}
// A stop whose change a later walk delivered since, or one its group made for a member delivered since,
// holds nothing back: the group stops a walk only for a member whose change is still undelivered (novox/hq
// issue 747).
if d.coveredStop() || h.stopLapsed(d) {
continue
}
// By the merges it names alone: its own commit may be another's merge it no longer answers.
if !slices.ContainsFunc(s.walk.Delivery.Merges, func(m WalkMerge) bool {
return strings.EqualFold(m.Repository, d.Repository) && sameCommit(m.Commit, d.MergedAs)
@@ -2126,6 +2350,51 @@ func (h *Holder) carriesStopped(walk string) ([]string, string) {
return out, why
}
// coverAwaited is whether a stopped merge a walk carries, given as `<repository>@<commit>`, waits for the forge's
// word on whether a later finished walk delivered it or the member its group stopped it for (novox/hq issue 747).
// It says since when the forge was first asked, the earliest of them. Held only within coverAskBound of the first
// ask: past it the forge's silence or error lets the walk be asked to leave the merge out. Under the lock.
func (h *Holder) coverAwaited(merges []string) (time.Time, bool) {
var since time.Time
awaited := false
for _, d := range h.deliveries {
if d.State != Stopped || d.MergedAs == "" || !containsString(merges, d.Repository+"@"+d.MergedAs) {
continue
}
for _, id := range []string{d.ID, d.stoppedFor()} {
if id == "" || !h.coverAsked[id] {
continue
}
if at := h.coverAskedSince[id]; !awaited || at.Before(since) {
since = at
}
awaited = true
}
}
return since, awaited
}
// coverHeld says why a walk is held back while the forge is asked whether a later finished walk delivered a stopped
// merge it carries (novox/hq issue 747), or nothing when it is not. Said by `waits`, `stalled` and `stalled walks`, so
// a forge that cannot answer is loud. Under the lock.
func (h *Holder) coverHeld(walk string) string {
without, _ := h.carriesStopped(walk)
if len(without) == 0 {
return ""
}
since, awaited := h.coverAwaited(without)
if !awaited {
return ""
}
return fmt.Sprintf("it carries %s, whose delivery was stopped, and the forge is asked since %s whether a later "+
"finished walk delivered it; it is asked to leave the merge out if no answer comes by %s (novox/hq issue 747)",
strings.Join(without, ", "), since.Local().Format("15:04:05"), since.Add(coverAskBound).Local().Format("15:04:05"))
}
// coverAskBound is how long a walk carrying a stopped merge is held back while the forge is asked whether a later
// finished walk delivered it (novox/hq issue 747); past it, the walk is asked to leave the merge out.
const coverAskBound = 10 * time.Minute
// lastWhyOf is the reason of a delivery's last transition.
func lastWhyOf(d *Delivery) string {
if len(d.Transitions) == 0 {
@@ -2161,6 +2430,9 @@ func (h *Holder) leaveStopped() []func() {
if without, _ := h.carriesStopped(id); len(without) > 0 {
said := fmt.Sprintf("its walk %s is not let go: it carries %s, whose delivery was stopped, and is asked to "+
"leave it out (novox/hq ADR 0299)", id, strings.Join(without, ", "))
if hold := h.coverHeld(id); hold != "" {
said = "its walk " + id + " is held back: " + hold
}
if r := h.leaveRefused[id]; r != "" {
said += "; the controller refused: " + r
}
@@ -2184,6 +2456,9 @@ func (h *Holder) leaveStopped() []func() {
delete(h.leaveRefused, id)
continue
}
if _, awaited := h.coverAwaited(without); awaited {
continue // held back, not stopped, until the forge says whether a later walk delivered it (issue 747)
}
// Once per record of it: the record that answers an ask is newer. Asked again after five minutes, should
// the ask have gone nowhere.
if a, asked := h.leaveAsked[id]; asked && seen.walk.Revision <= a.revision && now.Sub(a.at) < 5*time.Minute {
@@ -2305,18 +2580,28 @@ func againWords(a StopAnswer) string {
func GroupState(g *Group, members []*Delivery) (string, string) {
count := map[State]int{}
reached, ended := 0, 0
var uncovered []string
var uncovered, lapsed []string
byID := map[string]*Delivery{}
for _, d := range members {
byID[d.ID] = d
}
for _, d := range members {
count[d.State]++
if d.State.Final() {
ended++
}
switch {
case d.State.Reached(), d.State == Superseded && d.CoveredBy != "":
case memberReached(d):
// Reached, whatever its final state: a failure or stop a later walk delivered since is no failure of the
// group (novox/hq issue 747).
reached++
continue
case stopLapsed(d, func(id string) *Delivery { return byID[id] }):
lapsed = append(lapsed, d.ID)
continue
case d.State == Superseded:
uncovered = append(uncovered, d.ID)
}
count[d.State]++
}
n := len(members)
switch {
@@ -2326,6 +2611,11 @@ func GroupState(g *Group, members []*Delivery) (string, string) {
return "stopped", "a member was stopped"
case n > 0 && reached == n:
return "delivered", "every member is delivered"
case g.Closed && len(lapsed) > 0 && reached+len(lapsed) == n:
// Stopped for a member delivered since: its own change waits for a later walk of its repository, which then
// covers it (novox/hq issue 747).
return "delivering", fmt.Sprintf("%d of %d delivered; %s, stopped for a member delivered since, waits for a "+
"later walk of its repository", reached, n, strings.Join(lapsed, ", "))
case g.Closed && n > 0 && ended == n:
return "stopped", supersededWhy + strings.Join(uncovered, ", ")
case g.Closed && g.Check != nil && (g.Check.OutOfDate != "" || g.Check.Verdict == "" ||
@@ -1,6 +1,7 @@
package main
import (
"encoding/json"
"strings"
"testing"
"time"
@@ -573,3 +574,49 @@ func TestStalledAndCloseWorkFromTheTable(t *testing.T) {
t.Fatalf("H2 closed a held delivery: %v", err)
}
}
// novox/hq issue 867, ADR 0382 §1 (contract A): each member of a group in `groups` also says its pull
// request number, full head, base branch and whether it is open; `members` keeps its text form.
func TestGroupsNamesEachMembersPullRequestHeadBaseAndOpen(t *testing.T) {
w := newWorld(t)
headA := "a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1"
headB := "b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2"
pb := pr("novox/b", 2, headB, "feat/z")
pb.Base = "release/1"
w.h.PullUpdated(pr("novox/a", 1, headA, "feat/z"))
w.h.PullUpdated(pb)
lines := w.h.Groups(false)
if len(lines) != 1 || len(lines[0].Details) != 2 || len(lines[0].Members) != 2 {
t.Fatalf("groups: %+v", lines)
}
got := map[string]MemberLine{}
for _, m := range lines[0].Details {
got[m.Repository] = m
}
a, b := got["novox/a"], got["novox/b"]
if a.Number != 1 || a.Head != headA || a.Base != "main" || !a.Open || a.ID == "" {
t.Fatalf("member a: %+v", a)
}
if b.Number != 2 || b.Head != headB || b.Base != "release/1" || !b.Open {
t.Fatalf("member b: %+v", b)
}
if !strings.Contains(lines[0].Members[0], "(") {
t.Fatalf("members lost its text form: %v", lines[0].Members)
}
// Closed unmerged: still listed while the group is, but not open.
w.h.PullClosed(pr("novox/b", 2, headB, "feat/z"))
for _, l := range w.h.Groups(true) {
for _, m := range l.Details {
if m.Repository == "novox/b" && m.Open {
t.Fatalf("a closed pull request is open: %+v", m)
}
}
}
// JSON keys are the contract's.
raw, _ := json.Marshal(a)
for _, k := range []string{`"number":1`, `"head":"` + headA + `"`, `"base":"main"`, `"open":true`} {
if !strings.Contains(string(raw), k) {
t.Fatalf("%s lacks %s", raw, k)
}
}
}
@@ -0,0 +1,258 @@
package main
import (
"strings"
"testing"
"time"
)
// novox/hq issue 747: in the group fix/a-module-moves-only-when-its-program-changes, the controller's member #336
// (merged as 5d22d7df) failed at 22:32 during an outage of the forge's tools. The walk of the controller's next merge,
// #339 at 60cc9ec3, which contains 5d22d7df, failed with it and was resumed to done by a rebuild. mesh-delivery still
// held #336 failed, so its group stopped every later walk of the catalogue that carried the catalogue member's merge
// 2edef5a0: the catalogue could deliver nothing. A failed or stopped delivery is covered once a later finished walk of
// its repository that contains its merge built its modules, and a group stops a walk only for a member whose change is
// still undelivered.
//
// The shape is the live state of 2026-10-14 23:07, as mesh-delivery.show gave it, read back after a restart: the
// controller holds none of the walks of the night any more, only the walk of the catalogue's next merge.
const (
g747 = "fix/a-module-moves-only-when-its-program-changes"
ctl747, cat747 = "novox/mesh-controller", "novox/mesh-catalog"
head336, merge336 = "b34055f580d4e2a4", "5d22d7df86c73452"
head339, merge339 = "e4acab32d6a20ade", "60cc9ec363dfa7c6"
head301, merge301 = "d94764f00168471f", "2edef5a00ae05605"
merge307 = "0f76b6e2aaaaaaaa"
)
var (
at747 = time.Date(2026, 10, 14, 20, 21, 6, 0, time.UTC)
id336, id339, id301 = IDOf(ctl747, head336), IDOf(ctl747, head339), IDOf(cat747, head301)
unchangedRouteProxy747 = map[string]string{"route-proxy": "its build source holds none of the changed files"}
)
// night747 is the stuck state, with the forge saying, or not, that 60cc9ec3 contains 5d22d7df.
func night747(t *testing.T, contained bool) *world { return night747With(t, contained, nil) }
// night747With is night747 with #339's walk having built only the modules given, saying the others unchanged.
func night747With(t *testing.T, contained bool, built []string) *world {
t.Helper()
return night747Over(t, func(f *fakeForge) { f.contains[merge336+" in "+merge339] = contained }, built)
}
// night747Over is night747With with the forge set as given: one that is not told the pair cannot say.
func night747Over(t *testing.T, set func(*fakeForge), built []string) *world {
t.Helper()
at := at747
failedAt := at.Add(11 * time.Minute)
store := newMemStore()
moved339, unchanged339 := []string{"build-agent", "mesh-controller"}, unchangedRouteProxy747
if built != nil {
moved339, unchanged339 = built, map[string]string{}
for _, m := range []string{"build-agent", "mesh-controller", "route-proxy"} {
if !containsString(built, m) {
unchanged339[m] = "its build source holds none of the files #339 changed"
}
}
}
m336 := &Delivery{ID: id336, Repository: ctl747, Commit: head336, Number: 336, Base: "main", Branch: g747, Group: g747,
State: Failed, Since: failedAt, Created: at.Add(-time.Hour), MergedAs: merge336, MergedAt: at,
AnsweredBy: merge336, Check: &Verdict{Gate: "pass"},
Plan: planMoving(ctl747, "build-agent", "mesh-controller", "route-proxy"),
Walk: &WalkSeen{ID: "plan-336", Commit: merge336, State: walkFailed, Branch: "main", MovedSaid: true,
Note: "mesh-controller failed to build in tier 0", Created: at.Add(7 * time.Minute), Tiers: 2,
Unchanged: unchangedRouteProxy747},
Transitions: []Transition{{At: failedAt, From: Delivering, To: Failed, Event: EvFailed,
Why: "its walk failed, carrying its own commit"}}}
m339 := &Delivery{ID: id339, Repository: ctl747, Commit: head339, Number: 339, Base: "main", Branch: "fix/726",
State: Failed, Since: failedAt, Created: at.Add(-10 * time.Minute), MergedAs: merge339, MergedAt: at.Add(7 * time.Minute),
AnsweredBy: merge339, Check: &Verdict{Gate: "pass"}, Plan: planMoving(ctl747, "mesh-controller"),
// Its walk failed with #336's, and a rebuild resumed it to done: the record it keeps says so.
Walk: &WalkSeen{ID: "plan-339", Commit: merge339, State: walkDone, Branch: "main", MovedSaid: true,
Moved: moved339, Unchanged: unchanged339,
Created: at.Add(11 * time.Minute), Tiers: 2, Tier: 2},
Transitions: []Transition{{At: failedAt, From: Delivering, To: Failed, Event: EvFailed,
Why: "its walk failed, carrying its own commit"}}}
m301 := &Delivery{ID: id301, Repository: cat747, Commit: head301, Number: 301, Base: "main", Branch: g747, Group: g747,
State: Stopped, Since: failedAt, Created: at.Add(-2 * time.Hour), MergedAs: merge301, MergedAt: at.Add(7 * time.Second),
Check: &Verdict{Gate: "pass"}, Plan: planMoving(cat747, "build-agent", "route-proxy"),
Transitions: []Transition{{At: failedAt, From: Ready, To: Stopped, Event: EvStop, By: byOwner,
Why: "its group " + g747 + " stopped: " + id336 + " failed"}}}
for _, d := range []*Delivery{m336, m339, m301} {
if err := store.PutDelivery(d); err != nil {
t.Fatal(err)
}
}
g := &Group{ID: g747, Members: []string{id301, id336}, Closed: true, Order: []string{id336, id301}, Said: "failed",
OrderOf: commitsOf([]*Delivery{m301, m336}), Check: &GroupCheck{Asked: "check-1", At: at.Add(-time.Minute),
Verdict: "pass", Summary: "both compose", Commits: commitsOf([]*Delivery{m301, m336})}}
if err := store.PutGroup(g); err != nil {
t.Fatal(err)
}
ctlr := newFakeController()
// The catalogue's next merge: its walk carries 2edef5a0, merged before it and never walked, and waits for its word.
ctlr.put(Walk{ID: "plan-cat", Repository: cat747, Branch: "main", Commit: merge307, Created: at.Add(46 * time.Minute),
State: walkBuilding, Tiers: [][]string{{"gitea"}}, Modules: map[string]*WalkModule{"gitea": {}},
Delivery: &WalkDelivery{Awaits: "mesh-delivery", Merges: []WalkMerge{
{Repository: cat747, Commit: merge301, Carried: merge307}, {Repository: cat747, Commit: merge307}}}})
ctlr.mergedAt[merge301], ctlr.mergedAt[merge307] = m301.MergedAt, at.Add(46*time.Minute)
forge := newFakeForge()
set(forge)
w := newWorldOver(t, store, ctlr, forge, at.Add(47*time.Minute))
if err := w.h.Reconcile(); err != nil {
t.Fatal(err)
}
w.settleAll()
return w
}
func groupState747(w *world) (string, string) {
g := w.h.groups[g747]
return GroupState(g, w.h.membersOf(g))
}
// **A failed member is covered by a later finished walk that contains it**: #339's walk, done at 60cc9ec3, which the
// forge says contains 5d22d7df, built build-agent and mesh-controller, and #336's own walk said route-proxy unchanged.
// #336 stays failed, reads covered by that walk on its pull request, and its group no longer reads failed. #339, whose
// own walk was resumed to done, is covered by it too.
func TestAFailedMemberIsCoveredByALaterDoneWalkThatContainsIt(t *testing.T) {
w := night747(t, true)
d := w.delivery(id336)
if state, desc := statusOf(d); d.State != Failed || state != "success" || !strings.Contains(desc, "covered after its failure") ||
!strings.Contains(desc, "plan-339") {
t.Fatalf("#336's status reads %s %q", state, desc)
}
if s := statusOn(w, ctl747, head336); !strings.HasPrefix(s, "success") {
t.Fatalf("the forge was told %q of #336", s)
}
if state, desc := statusOf(w.delivery(id339)); state != "success" || !strings.Contains(desc, "plan-339") {
t.Fatalf("#339 reads %s %q; want it covered by its own walk, resumed to done", state, desc)
}
if state, why := groupState747(w); state == "failed" || state == "stopped" {
t.Fatalf("the group reads %s: %s", state, why)
}
// Told once, and kept: a restart tells nothing again.
notes := len(w.forge.notesOn(merge336))
again := newWorldOver(t, w.store, w.ctl, w.forge, w.now.Add(time.Minute))
again.settleAll()
if state, _ := statusOf(again.delivery(id336)); len(w.forge.notesOn(merge336)) != notes || state != "success" {
t.Fatalf("a restart told it again or forgot it: %d notes, was %d", len(w.forge.notesOn(merge336)), notes)
}
}
// **The group then no longer stops the other repository's walk**: the catalogue member was stopped for #336 alone,
// so the walk of the catalogue's next merge, which carries its merge, is neither stopped nor held back, and the
// group waits for that walk to deliver the catalogue member's change.
func TestACoveredMemberNoLongerStopsTheOtherRepositorysWalk(t *testing.T) {
w := night747(t, true)
if len(w.ctl.withouts) > 0 || containsString(w.ctl.stops, "plan-cat") {
t.Fatalf("the catalogue's walk was stopped: without %v, stops %v", w.ctl.withouts, w.ctl.stops)
}
if stopped, _ := w.h.carriesStopped("plan-cat"); len(stopped) > 0 {
t.Fatalf("the catalogue's walk is held back for %v", stopped)
}
if state, why := groupState747(w); state != "delivering" || !strings.Contains(why, id301) {
t.Fatalf("the group reads %s: %s; want delivering, the catalogue member waiting for a later walk", state, why)
}
// Once a later walk of the catalogue that contains its merge built its modules, it is covered, and the group is
// delivered.
w.forge.contains[merge301+" in "+merge307] = true
r := w.ctl.walk("plan-cat")
sent := w.now
r.Modules = map[string]*WalkModule{}
for _, m := range []string{"build-agent", "route-proxy", "gitea"} {
r.Modules[m] = &WalkModule{State: "built", SentAt: &sent}
}
r.State = walkDone
w.h.WalkMoved(w.ctl.put(r))
w.later(time.Minute)
w.settleAll()
if state, desc := statusOf(w.delivery(id301)); state != "success" || !strings.Contains(desc, "plan-cat") {
t.Fatalf("the catalogue member reads %s %q", state, desc)
}
if state, why := groupState747(w); state != "delivered" {
t.Fatalf("the group reads %s: %s", state, why)
}
}
// **A failed member not contained still stops it**: the forge says 60cc9ec3 does not contain 5d22d7df, so #336's change
// never reached the machines; it stays failed and uncovered, its group failed, and the catalogue's walk that carries
// the stopped member's merge is asked to leave it out.
func TestAFailedMemberNotContainedStillStopsTheOtherRepositorysWalk(t *testing.T) {
w := night747(t, false)
d := w.delivery(id336)
if state, desc := statusOf(d); d.State != Failed || state != "failure" {
t.Fatalf("#336 is %s and reads %s %q", d.State, state, desc)
}
if state, _ := groupState747(w); state != "failed" {
t.Fatalf("the group reads %s", state)
}
if len(w.ctl.withouts) == 0 || !strings.HasPrefix(w.ctl.withouts[0], "plan-cat without "+cat747+"@"+merge301) {
t.Fatalf("the catalogue's walk was not asked to leave the stopped merge out: %v", w.ctl.withouts)
}
}
// **A later walk that only said a module unchanged covers nothing of an earlier merge that moves it**: #339's walk
// built mesh-controller alone and said build-agent unchanged for #339's own files, which says nothing of #336's; only
// #336's own walk says that of its change (route-proxy). #336 stays failed and its group still stops the catalogue.
func TestALaterWalksUnchangedCoversNoModuleOfAnEarlierMerge(t *testing.T) {
w := night747With(t, true, []string{"mesh-controller"})
if state, desc := statusOf(w.delivery(id336)); state != "failure" {
t.Fatalf("#336 reads %s %q, though no walk built build-agent for it", state, desc)
}
if state, _ := groupState747(w); state != "failed" {
t.Fatalf("the group reads %s", state)
}
if len(w.ctl.withouts) == 0 {
t.Fatal("the catalogue's walk was not asked to leave the stopped merge out")
}
}
// **A forge that cannot say holds the walk only within the bound** (review of issue 747): while the forge errors on
// whether 60cc9ec3 contains 5d22d7df, the catalogue's walk is held back, loudly, neither stopped nor asked to leave the
// merge out, and #336 still reads failed. Past the bound the forge's silence holds nothing back any more: the walk
// is asked to leave the stopped merge out, as with no later walk at all.
func TestAForgeThatCannotSayHoldsTheWalkOnlyWithinTheBound(t *testing.T) {
for _, c := range []struct {
name string
set func(*fakeForge)
}{
{"erroring", func(f *fakeForge) { f.containsDown = true }},
{"not answering for the pair", func(*fakeForge) {}},
} {
t.Run(c.name, func(t *testing.T) {
w := night747Over(t, c.set, nil)
if len(w.ctl.withouts) > 0 || len(w.ctl.stops) > 0 {
t.Fatalf("within the bound the walk was not held: without %v, stops %v", w.ctl.withouts, w.ctl.stops)
}
if state, desc := statusOf(w.delivery(id336)); state != "failure" {
t.Fatalf("#336 reads %s %q while the forge cannot say", state, desc)
}
said := ""
for _, l := range w.h.WalksWaiting() {
if l.Walk == "plan-cat" {
said += l.Says
}
}
w.later(3 * time.Minute) // past the grace an unheld walk is given, so `stalled` lists it
w.settleAll()
for _, l := range w.h.Stalled() {
if l.Walk == "plan-cat" {
said += " | " + l.Says
}
}
if strings.Count(said, "the forge is asked since") != 2 {
t.Fatalf("the hold is not said by `stalled walks` and `stalled`: %q", said)
}
if len(w.ctl.withouts) > 0 || len(w.ctl.stops) > 0 {
t.Fatalf("within the bound the walk was not held: without %v, stops %v", w.ctl.withouts, w.ctl.stops)
}
w.later(coverAskBound + time.Minute)
w.settleAll()
if len(w.ctl.withouts) == 0 || !strings.HasPrefix(w.ctl.withouts[0], "plan-cat without "+cat747+"@"+merge301) {
t.Fatalf("past the bound the walk was not asked to leave the stopped merge out: %v", w.ctl.withouts)
}
})
}
}
@@ -0,0 +1,292 @@
package main
import (
"strings"
"testing"
"time"
)
// novox/hq issue 753: once issue 747's fix was live (2026-10-14 23:35), the catalogue's walk plan-1792012015539062661 at
// 0f76b6e2 (#307's merge; build-agent, gitea, route-proxy to all four machines) was resumed and said "published; its
// walk waits for mesh-delivery's word: no delivery that has not ended follows it or is answered by it". Every delivery
// it carried had ended: #301 stopped by its group for the controller's #336, delivered since, and waiting for "a later
// walk of its repository that contains its merge"; #304 stopped; #307 delivered. The only walk that contained #301's
// merge waited for #301's word: a deadlock, released by a hand `plans go`. A delivery whose stop lapsed gives its word
// to the walk that answers its merge, and a walk with no living delivery behind it says what it carries.
//
// The shape is the live state as `mesh-delivery.show` and `mesh-controller.plans` gave it, read back after a restart.
const (
merge304, head304, head307 = "4c0ffee0bbbbbbbb", "a304a304a304a304", "b307b307b307b307"
plan753 = "plan-1792012015539062661"
)
var id304, id307 = IDOf(cat747, head304), IDOf(cat747, head307)
// night753 is the stuck walk: #336 delivered (its change reached the machines), #301 stopped by the given act, #304
// stopped by its group, #307 delivered, and the catalogue's walk at #307's merge naming all three merges, waiting.
func night753(t *testing.T, stop Transition) *world { return night753With(t, stop) }
// night753With is night753 with more deliveries kept: one in #301's group is ordered after #336 and before #301.
func night753With(t *testing.T, stop Transition, more ...*Delivery) *world {
t.Helper()
return night753Over(t, stop, nil, nil, more...)
}
// night753Over is night753With with more merges named by the walk and the controller set as given before the first
// tick.
func night753Over(t *testing.T, stop Transition, merges []WalkMerge, set func(*fakeController), more ...*Delivery) *world {
t.Helper()
at := at747
store := newMemStore()
m336 := &Delivery{ID: id336, Repository: ctl747, Commit: head336, Number: 336, Base: "main", Branch: g747, Group: g747,
State: Delivered, Since: at.Add(70 * time.Minute), Created: at.Add(-time.Hour), MergedAs: merge336, MergedAt: at,
AnsweredBy: merge336, Check: &Verdict{Gate: "pass"}, Plan: planMoving(ctl747, "build-agent", "mesh-controller"),
Walk: &WalkSeen{ID: "plan-336", Commit: merge336, State: walkDone, Branch: "main", MovedSaid: true,
Moved: []string{"build-agent", "mesh-controller"}, Created: at.Add(7 * time.Minute), Tiers: 2, Tier: 2}}
m301 := &Delivery{ID: id301, Repository: cat747, Commit: head301, Number: 301, Base: "main", Branch: g747, Group: g747,
State: Stopped, Since: stop.At, Created: at.Add(-2 * time.Hour), MergedAs: merge301, MergedAt: at.Add(7 * time.Second),
Check: &Verdict{Gate: "pass"}, Plan: planMoving(cat747, "build-agent", "route-proxy"), Transitions: []Transition{stop}}
if stop.By == byOwner {
m301.StoppedFor = id336
}
m304 := &Delivery{ID: id304, Repository: cat747, Commit: head304, Number: 304, Base: "main", Branch: "fix/forge-events",
State: Stopped, Since: at.Add(80 * time.Minute), Created: at.Add(-time.Hour), MergedAs: merge304,
MergedAt: at.Add(30 * time.Minute), Check: &Verdict{Gate: "pass"},
Plan: planMoving(cat747, "build-agent", "gitea", "route-proxy"),
// Stopped with its walk, which the walk of #307 took over, and covered since by the walk of #306 (issue 747).
DeliveredBy: "plan-1792011830935225963 (novox/mesh-catalog at 0361c44a into main, done, contains its merge 4c0ffee0)",
CoverTold: true,
Walk: &WalkSeen{ID: "plan-1792009301770286799", Commit: merge304, State: walkSuperseded, Branch: "main",
TakenOverBy: plan753, Waits: true, WaitedFor: "mesh-delivery", Created: at.Add(31 * time.Minute)},
Transitions: []Transition{{At: at.Add(80 * time.Minute), From: Published, To: Stopped, Event: EvStop,
Why: "its walk plan-1792009301770286799 was stopped by mesh-delivery"}}}
m307 := &Delivery{ID: id307, Repository: cat747, Commit: head307, Number: 307, Base: "main", Branch: "fix/ts-check",
State: Delivered, Since: at.Add(90 * time.Minute), Created: at.Add(-time.Hour), MergedAs: merge307,
MergedAt: at.Add(46 * time.Minute), Check: &Verdict{Gate: "pass"}, Plan: planMoving(cat747, "build-agent")}
for _, d := range append([]*Delivery{m336, m301, m304, m307}, more...) {
if err := store.PutDelivery(d); err != nil {
t.Fatal(err)
}
}
inGroup, order := []*Delivery{m301, m336}, []string{id336}
for _, d := range more {
if d.Group == g747 {
inGroup, order = append(inGroup, d), append(order, d.ID)
}
}
order = append(order, id301)
var ids []string
for _, d := range inGroup {
ids = append(ids, d.ID)
}
g := &Group{ID: g747, Members: ids, Closed: true, Order: order, Said: "delivering",
OrderOf: commitsOf(inGroup), Check: &GroupCheck{Asked: "check-1", At: at.Add(-time.Minute),
Verdict: "pass", Summary: "all compose", Commits: commitsOf(inGroup)}}
if err := store.PutGroup(g); err != nil {
t.Fatal(err)
}
ctlr := newFakeController()
modules := map[string]*WalkModule{"build-agent": {}, "gitea": {}, "route-proxy": {}}
ctlr.put(Walk{ID: plan753, Repository: cat747, Branch: "main", Commit: merge307, Created: at.Add(46 * time.Minute),
State: walkBuilding, Tiers: [][]string{{"build-agent", "gitea", "route-proxy"}}, Modules: modules,
Delivery: &WalkDelivery{Awaits: "mesh-delivery", Merges: []WalkMerge{
{Repository: cat747, Commit: merge301, Carried: merge307}, {Repository: cat747, Commit: merge304, Carried: merge307},
{Repository: cat747, Commit: merge307}}}})
if len(merges) > 0 {
r := ctlr.walk(plan753)
r.Delivery.Merges = append(merges, r.Delivery.Merges...)
ctlr.put(r)
}
if set != nil {
set(ctlr)
}
ctlr.mergedAt[merge301], ctlr.mergedAt[merge304], ctlr.mergedAt[merge307] = m301.MergedAt, m304.MergedAt, m307.MergedAt
w := newWorldOver(t, store, ctlr, newFakeForge(), at.Add(100*time.Minute))
if err := w.h.Reconcile(); err != nil {
t.Fatal(err)
}
w.settleAll()
return w
}
// groupStop753 is #301's stop as its group made it: for #336, whose change was delivered since.
func groupStop753() Transition {
return Transition{At: at747.Add(11 * time.Minute), From: Ready, To: Stopped, Event: EvStop, By: byOwner,
Why: "its group " + g747 + " stopped: " + id336 + " failed"}
}
func saidOf(lines []StalledLine, walk string) (string, State) {
said, state := "", State("")
for _, l := range lines {
if l.Walk == walk {
said, state = said+l.Says, l.State
}
}
return said, state
}
// **A stopped delivery gives its word to the later walk it waits for**: #301, stopped for #336, delivered since, waits for
// a later walk of the catalogue that contains its merge; plan-1792012015539062661 names that merge and waits for this
// owner's word. #301 gives it, once, saying why, and `stalled walks` never calls the walk unheld.
func TestAStoppedDeliveryGivesItsWordToTheWalkItWaitsFor(t *testing.T) {
w := night753(t, groupStop753())
if n := strings.Count(strings.Join(w.ctl.delivers, " "), plan753); n != 1 {
t.Fatalf("the walk was let go %d times (%v); want once", n, w.ctl.delivers)
}
d := w.delivery(id301)
if d.State != Stopped || !strings.Contains(d.LetGoWhy, lapsedWord(plan753)) || !strings.Contains(d.LetGoWhy, "issue 753") {
t.Fatalf("#301 is %s, its word given for %q", d.State, d.LetGoWhy)
}
if said, state := saidOf(w.h.WalksWaiting(), plan753); state == Unheld {
t.Fatalf("the walk reads unheld: %q", said)
}
if len(w.ctl.withouts) > 0 || len(w.ctl.stops) > 0 {
t.Fatalf("the walk was stopped: without %v, stops %v", w.ctl.withouts, w.ctl.stops)
}
// Once that walk built build-agent and route-proxy and ended done, #301 is covered by it (issue 747).
w.forge.contains[merge301+" in "+merge307] = true
r := w.ctl.walk(plan753)
sent := w.now
for _, m := range []string{"build-agent", "gitea", "route-proxy"} {
r.Modules[m] = &WalkModule{State: "built", SentAt: &sent}
}
r.State = walkDone
w.h.WalkMoved(w.ctl.put(r))
w.later(time.Minute)
w.settleAll()
if state, desc := statusOf(w.delivery(id301)); state != "success" || !strings.Contains(desc, plan753) {
t.Fatalf("#301 reads %s %q", state, desc)
}
}
// **A walk still holding a live delivery not at its turn waits for it**: the stopped delivery's word is one turn among
// the walk's, never a way past a live delivery it carries.
func TestALapsedStopDoesNotLetGoPastALiveDelivery(t *testing.T) {
at := at747
held := &Delivery{ID: IDOf(cat747, "c308c308c308c308"), Repository: cat747, Commit: "c308c308c308c308", Number: 308,
Base: "main", Branch: "fix/held", State: Held, HeldWhy: "a person's", Since: at, Created: at.Add(-time.Hour),
MergedAs: "c308aaaaaaaaaaaa", MergedAt: at.Add(40 * time.Minute), Check: &Verdict{Gate: "pass"},
Plan: planMoving(cat747, "gitea"), Walk: &WalkSeen{ID: plan753, Commit: merge307, State: walkBuilding, Branch: "main",
Waits: true, Created: at.Add(46 * time.Minute)}}
w := night753With(t, groupStop753(), held)
if containsString(w.ctl.delivers, plan753) {
t.Fatalf("the walk was let go past a held delivery it carries: %v", w.ctl.delivers)
}
if said, state := saidOf(w.h.WalksWaiting(), plan753); state != Held {
t.Fatalf("`stalled walks` reads %s %q; want the held delivery", state, said)
}
}
// **A stop by a person gives no word**: #301 stopped by hand holds its change back; the walk is asked to leave it out,
// never let go for it.
func TestAPersonsStopGivesNoWord(t *testing.T) {
w := night753(t, Transition{At: at747.Add(11 * time.Minute), From: Ready, To: Stopped, Event: EvStop, By: "jochen",
Why: "stopped by hand"})
if containsString(w.ctl.delivers, plan753) {
t.Fatalf("the walk was let go for a person's stop: %v", w.ctl.delivers)
}
if len(w.ctl.withouts) == 0 && len(w.ctl.stops) == 0 {
t.Fatal("the walk carrying a person's stop was neither left without it nor stopped")
}
}
// **A walk no living delivery holds says what it carries** (the issue's third expectation): with nothing to give it
// its word, `stalled walks` and, past the grace, `stalled` name every ended delivery it carries and its state, never
// only that nothing holds it.
func TestAWalkWithNoLivingDeliverySaysWhatItCarries(t *testing.T) {
w := night753(t, groupStop753())
// The walk read again as waiting, naming only #307's merge, which ended delivered: no delivery whose stop lapsed
// gives it its word, and no live one holds it.
r := w.ctl.walk(plan753)
r.Delivery.Go = nil
r.Delivery.Merges = []WalkMerge{{Repository: cat747, Commit: merge307}}
w.h.WalkMoved(w.ctl.put(r))
said, state := saidOf(w.h.WalksWaiting(), plan753)
if state != Unheld || !strings.Contains(said, id307+" (delivered)") {
t.Fatalf("`stalled walks` reads %s %q; want unheld, naming %s (delivered)", state, said, id307)
}
w.settleAll()
w.later(matchGrace + time.Minute)
w.settleAll()
said, _ = saidOf(w.h.Stalled(), plan753)
if !strings.Contains(said, "every delivery it carries ended: ") || !strings.Contains(said, id307+" (delivered)") {
t.Fatalf("`stalled` reads %q", said)
}
}
// m299 is a member of #301's group ordered before it, stopped for #336 as #301 was, its merge on another walk.
func m299() *Delivery {
at := at747
return &Delivery{ID: IDOf(cat747, "e299e299e299e299"), Repository: cat747, Commit: "e299e299e299e299", Number: 299,
Base: "main", Branch: g747, Group: g747, State: Stopped, StoppedFor: id336, Since: at.Add(11 * time.Minute),
Created: at.Add(-3 * time.Hour), MergedAs: "e299aaaaaaaaaaaa", MergedAt: at.Add(5 * time.Second),
Check: &Verdict{Gate: "pass"}, Plan: planMoving(cat747, "gitea"),
Transitions: []Transition{{At: at.Add(11 * time.Minute), From: Ready, To: Stopped, Event: EvStop, By: byOwner,
Why: "its group " + g747 + " stopped: " + id336 + " failed"}}}
}
// **A stopped member gives its word only in its group's order** (review of issue 753): #299, ordered before #301 and
// stopped for #336 too, is not delivered and this walk does not carry its merge, so #301 gives no word — it would land
// before #299 — and `stalled walks` names #299. When the walk names #299's merge too, it delivers both, in order, and
// is let go.
func TestAStoppedMemberGivesItsWordOnlyInItsGroupsOrder(t *testing.T) {
w := night753With(t, groupStop753(), m299())
if containsString(w.ctl.delivers, plan753) {
t.Fatalf("the walk was let go for #301 before #299, ordered before it: %v", w.ctl.delivers)
}
said, state := saidOf(w.h.WalksWaiting(), plan753)
if state == Unheld || !strings.Contains(said, m299().ID+" (stopped)") {
t.Fatalf("`stalled walks` reads %s %q; want #299 named", state, said)
}
w = night753Over(t, groupStop753(), []WalkMerge{{Repository: cat747, Commit: "e299aaaaaaaaaaaa", Carried: merge307}}, nil,
m299())
if !containsString(w.ctl.delivers, plan753) {
t.Fatalf("the walk carrying both members was not let go: %v", w.ctl.delivers)
}
}
// **A word the controller refuses is never silent** (review of issue 753): the controller keeps refusing it, and past
// the grace `stalled` lists the walk, with the refusal.
func TestARefusedWordIsListedByStalled(t *testing.T) {
w := night753Over(t, groupStop753(), nil, func(c *fakeController) { c.refuseDeliver = true })
if containsString(w.ctl.delivers, plan753) {
t.Fatalf("the walk was let go: %v", w.ctl.delivers)
}
w.later(matchGrace + time.Minute)
w.settleAll()
said, state := saidOf(w.h.Stalled(), plan753)
if state != Published || !strings.Contains(said, "the controller refused its word: the controller refuses the word") {
t.Fatalf("`stalled` reads %s %q", state, said)
}
// Once the controller takes it, the walk is let go.
w.ctl.refuseDeliver = false
w.settleAll()
if !containsString(w.ctl.delivers, plan753) {
t.Fatalf("the walk was not let go once the controller took the word: %v", w.ctl.delivers)
}
}
// **A walk held back for another stopped change is never silent** (review of issue 753): #304 was stopped by a person,
// and the controller refuses to leave its merge out or to stop the walk, so the word #301 owes is never sent; past the
// grace `stalled` lists the walk, saying why.
func TestAWalkHeldForAPersonsStopIsListedByStalled(t *testing.T) {
at := at747
m305 := &Delivery{ID: IDOf(cat747, "f305f305f305f305"), Repository: cat747, Commit: "f305f305f305f305", Number: 305,
Base: "main", Branch: "fix/other", State: Stopped, Since: at.Add(80 * time.Minute), Created: at.Add(-time.Hour),
MergedAs: "f305aaaaaaaaaaaa", MergedAt: at.Add(35 * time.Minute), Check: &Verdict{Gate: "pass"},
Plan: planMoving(cat747, "gitea"), Transitions: []Transition{{At: at.Add(80 * time.Minute), From: Ready,
To: Stopped, Event: EvStop, By: "jochen", Why: "stopped by hand"}}}
w := night753Over(t, groupStop753(), []WalkMerge{{Repository: cat747, Commit: m305.MergedAs, Carried: merge307}},
func(c *fakeController) { c.refuseWithout, c.refuseStop = true, true }, m305)
if containsString(w.ctl.delivers, plan753) {
t.Fatalf("the walk carrying a person's stop was let go: %v", w.ctl.delivers)
}
w.later(matchGrace + time.Minute)
w.settleAll()
said, _ := saidOf(w.h.Stalled(), plan753)
if !strings.Contains(said, "owe it (novox/hq issue 753)") || !strings.Contains(said, "f305aaaaaaaaaaaa") ||
!strings.Contains(said, "whose delivery was stopped") {
t.Fatalf("`stalled` reads %q", said)
}
}
@@ -0,0 +1,151 @@
package main
import (
"strings"
"testing"
"time"
)
// novox/hq issue 783: mesh-catalog commit b5417c28 was pushed while #305 (from branch adr-359-mesh-mail-733) was
// open, so this owner recorded its delivery as #305's. #305 then closed unmerged (its base branch had been
// deleted), and #315, opened from the same branch and head, merged it as 0914a33f. The delivery stayed stopped
// ("the pull request closed unmerged"), though it recorded merged_as 0914a33f: a rule did not hold — a delivery
// keyed by commit follows the pull request that merges that commit, and a closed duplicate does not end it — and
// the walk that carried the merge waited 64 minutes for this owner's word until a person's `plans go` released it.
//
// A delivery stopped only because its pull request closed unmerged, with no merge recorded, does not end its
// commit's delivery for another pull request with the same head: it is started over, under the same id, for the
// pull request that announces or merges it next.
const (
head783, merge783 = "d305d305d305d305", "d315d315d315d315"
head783b, merge783b = "d306d306d306d306", "d316d316d316d316"
head783c = "d307d307d307d307"
head783d, merge783d = "d308d308d308d308", "d318d318d318d318"
catalog783 = "novox/mesh-catalog"
branch783 = "adr-359-mesh-mail-733"
)
// TestADeliveryStoppedByAClosedDuplicateFollowsThePullRequestThatMergesItsCommit is the issue's own shape: #305
// announced, closed unmerged (stopped); #315 announced from the same head, checked and readied; #315 merges it,
// and the walk that answers that merge is let go — the delivery follows #315, never stuck on #305's stop.
func TestADeliveryStoppedByAClosedDuplicateFollowsThePullRequestThatMergesItsCommit(t *testing.T) {
w := newWorld(t)
id := IDOf(catalog783, head783)
w.h.PullUpdated(pr(catalog783, 305, head783, branch783, "src/a.go"))
w.h.PullClosed(pr(catalog783, 305, head783, branch783))
if w.state(id) != Stopped {
t.Fatalf("#305 closed unmerged left it %s; want stopped", w.state(id))
}
w.h.PullUpdated(pr(catalog783, 315, head783, branch783, "src/a.go"))
if d := w.delivery(id); d.State == Stopped || d.Number != 315 {
t.Fatalf("#315, the same head, left it %s, Number %d; want started over for #315, not stopped", d.State, d.Number)
}
if d := w.delivery(id); !strings.Contains(d.Transitions[len(d.Transitions)-1].Why, "#305 with the same head closed unmerged") {
t.Fatalf("the start-over's transition names what it replaces: %q", d.Transitions[len(d.Transitions)-1].Why)
}
w.h.Checked(verdict(catalog783, 315, head783, "pass"))
if d := w.delivery(id); d.State != Ready {
t.Fatalf("a passing verdict for #315 left it %s", d.State)
}
walk := w.ctl.put(aWalk("plan-783", catalog783, merge783, true))
w.h.WalkMoved(walk)
w.h.PullMerged(merged(catalog783, 315, head783, merge783))
if d := w.delivery(id); d.State != Published || d.Number != 315 {
t.Fatalf("#315's merge left it %s, Number %d; want published under #315", d.State, d.Number)
}
w.settleAll()
if len(w.ctl.delivers) != 1 || w.ctl.delivers[0] != "plan-783" {
t.Fatalf("the walk that answers #315's merge was not let go: %v", w.ctl.delivers)
}
if d := w.delivery(id); d.State != Delivering {
t.Fatalf("let go, it is %s; want delivering", d.State)
}
}
// TestAMergeWithoutAnAnnouncementStartsOverAStoppedDuplicateToo is decision 4: the merging pull request is never
// announced on its own (heard only as `pull.merged`) — the start-over happens in PullMerged itself, not only in
// PullUpdated, so a merge heard without the open announcement is not lost either.
func TestAMergeWithoutAnAnnouncementStartsOverAStoppedDuplicateToo(t *testing.T) {
w := newWorld(t)
id := IDOf(catalog783, head783b)
w.h.PullUpdated(pr(catalog783, 306, head783b, branch783+"-b", "src/b.go"))
w.h.PullClosed(pr(catalog783, 306, head783b, branch783+"-b"))
if w.state(id) != Stopped {
t.Fatalf("#306 closed unmerged left it %s; want stopped", w.state(id))
}
walk := w.ctl.put(aWalk("plan-783b", catalog783, merge783b, true))
w.h.WalkMoved(walk)
m := merged(catalog783, 316, head783b, merge783b)
m.HeadChecks = map[string]string{"mesh/merge-gate": "success", "mesh/repo-check": "success"}
w.h.PullMerged(m)
if d := w.delivery(id); d.State == Stopped || d.Number != 316 {
t.Fatalf("#316's merge, heard with no announcement of its own, left it %s, Number %d; want started over for #316",
d.State, d.Number)
}
w.settleAll()
if len(w.ctl.delivers) != 1 || w.ctl.delivers[0] != "plan-783b" {
t.Fatalf("the walk that answers #316's merge was not let go: %v", w.ctl.delivers)
}
}
// TestAStoppedDeliveryAnnouncedAgainByItsOwnPullRequestStaysStopped guards decision 3's "any other stopped
// delivery ... is untouched": the same pull request announced again after its own close is the ordinary
// 'announced again' path, never a start-over, and stays stopped.
func TestAStoppedDeliveryAnnouncedAgainByItsOwnPullRequestStaysStopped(t *testing.T) {
w := newWorld(t)
id := IDOf(catalog783, head783c)
w.h.PullUpdated(pr(catalog783, 307, head783c, branch783+"-c", "src/c.go"))
w.h.PullClosed(pr(catalog783, 307, head783c, branch783+"-c"))
if w.state(id) != Stopped {
t.Fatalf("#307 closed unmerged left it %s; want stopped", w.state(id))
}
w.h.PullUpdated(pr(catalog783, 307, head783c, branch783+"-c", "src/c.go"))
if d := w.delivery(id); d.State != Stopped || d.Number != 307 {
t.Fatalf("#307 announced again by itself left it %s, Number %d; want it to stay stopped", d.State, d.Number)
}
}
// TestAMergeHeardLongAfterStartingOverAStoppedDuplicateIsHistoryLikeAnyOtherReplayedMerge is review round 1's
// decision: a delivery started over from a merge heard without its open announcement (as in decision 4, above)
// takes FromForge exactly as a first-heard merge does, so one replayed from the bus's history long after it
// happened still falls under the history rule (novox/hq issue 313) and is never held for a person.
func TestAMergeHeardLongAfterStartingOverAStoppedDuplicateIsHistoryLikeAnyOtherReplayedMerge(t *testing.T) {
w := newWorld(t)
id := IDOf(catalog783, head783d)
w.h.PullUpdated(pr(catalog783, 308, head783d, branch783+"-d", "src/d.go"))
w.h.PullClosed(pr(catalog783, 308, head783d, branch783+"-d"))
if w.state(id) != Stopped {
t.Fatalf("#308 closed unmerged left it %s; want stopped", w.state(id))
}
m := merged(catalog783, 318, head783d, merge783d)
m.MergedAt = w.now.Add(-historyLate - time.Minute).UTC().Format(time.RFC3339)
w.h.PullMerged(m)
d := w.delivery(id)
if d.State == Stopped || d.Number != 318 {
t.Fatalf("#318's merge, heard long after it happened with no announcement of its own, left it %s, Number %d; "+
"want started over for #318", d.State, d.Number)
}
if !d.FromForge {
t.Fatal("a merge heard without its own announcement is made from the forge's word: FromForge is false")
}
if d.History == "" {
t.Fatalf("a merge heard %s after it happened is history, never held for a person; History is empty", historyLate+time.Minute)
}
if d.State == Held {
t.Fatalf("history is never held for a person; state is %s, HeldWhy %q", d.State, d.HeldWhy)
}
}
@@ -76,8 +76,9 @@ func run(h *Holder, l *listening) {
logf("[mesh-delivery] the controller's walks cannot be read: %v", err)
}
}
// Every open pull request, from the forge itself, every five minutes (novox/hq issue 347).
if now.Sub(looked) >= 5*time.Minute {
// Every open pull request, from the forge itself, every minute (novox/hq issues 347, 722): what says a
// head nobody checks, and the ready marks a merge is asked on.
if now.Sub(looked) >= time.Minute {
looked = now
h.SeeOpenPulls()
}
@@ -65,7 +65,9 @@ func TestItClaimsTheDeliverySeatAndFollowsWhatItHandles(t *testing.T) {
want = append(want, "seat:"+ControllerSeat+"."+v)
}
want = append(want, "gitea.gitea_note_append", "gitea.gitea_delivery_view", "gitea.gitea_commit_status",
"gitea.gitea_commit_statuses", "gitea.gitea_contains", "gitea.gitea_open_pulls")
"gitea.gitea_commit_statuses", "gitea.gitea_contains", "gitea.gitea_open_pulls",
// The merge on a ready mark (novox/hq issue 722): the merge tool, a comment, a label taken off.
"gitea.gitea_merge_pull_request", "gitea.gitea_add_comment", "gitea.gitea_api")
if !reflect.DeepEqual(m.Invokes, want) {
t.Fatalf("invokes %v, wanted %v", m.Invokes, want)
}
@@ -83,6 +83,14 @@ type Delivery struct {
// CoverTold is whether the forge was told that such a stop was covered since: its status, its view and its note
// (novox/hq issue 480). Kept, so a restart tells it once, and a stop covered before this was built is told too.
CoverTold bool `json:"cover_told,omitempty"`
// DeliveredBy says, for a delivery that failed or was stopped, the later finished walk of its repository whose
// commit contains its merge and which built every module it moves — each it did not build, its own walk said
// unchanged: its change reached the machines after all (novox/hq issue 747). Its state stays final, as a covered
// stop's does (issue 480), and it reads covered: its group counts it as delivered.
DeliveredBy string `json:"delivered_by,omitempty"`
// StoppedFor is, for a member its group stopped, the member whose end stopped it (novox/hq issue 747): once that
// member's change reached the machines, the stop has lapsed, and holds no walk of its repository back.
StoppedFor string `json:"stopped_for,omitempty"`
// TurnAwaits says, for a group member whose turn waits for a later merge to carry a superseded member before it,
// which merges and until when; TurnAwaitsUntil is that moment, which bounds the wait (ADR 0276 decision 9,
// novox/hq issue 445). `show` shows both, and `stalled` says them past that moment.
@@ -110,6 +118,9 @@ type Delivery struct {
LetGo time.Time `json:"let_go"`
LetGoWhy string `json:"let_go_why,omitempty"`
// Ready is what this owner did with its pull request's ready mark (novox/hq issue 722, ready.go).
Ready *ReadyRun `json:"ready,omitempty"`
Transitions []Transition `json:"transitions,omitempty"`
// Owed are the forge's and the bus's effects of its transitions not yet done: said only after the
// delivery is kept, retried until done.
@@ -300,18 +311,77 @@ func (d *Delivery) FromForgesWord() bool {
// shownOnItsPullRequest is whether its view and status are kept on its pull request: not for history, whose
// pull request the forge may no longer hold — or holds under its number another one.
// coveredStop is whether a delivery stopped because its walk ended done before its word without building its modules
// (novox/hq issue 474) was covered since: a later merge into its trunk that contains it was delivered, carrying every
// module it moves to every machine running it (ADR 0270's cover test, kept as CoveredBy). Its state stays stopped —
// final, as a covered superseded member stays superseded (issue 442) — and it reads covered, by that merge (novox/hq
// issue 480). A stop for any other reason never reads covered.
// coveredStop is whether a delivery that ended stopped or failed was covered since, and reads covered. A stop
// because its walk ended done before its word without building its modules (novox/hq issue 474) is covered when a
// later merge into its trunk that contains it was delivered, carrying every module it moves to every machine running
// it (ADR 0270's cover test, kept as CoveredBy; issue 480). Any stop or failure is covered when a later finished walk
// of its repository whose commit contains its merge built its modules (DeliveredBy, novox/hq issue 747). Its state
// stays final, as a covered superseded member stays superseded (issue 442), and it reads covered, by that merge or
// walk. A person's stop with a cover found only while it was held keeps its error.
func (d *Delivery) coveredStop() bool {
return d.State == Stopped && d.Stranded != "" && d.CoveredBy != ""
switch d.State {
case Stopped:
return d.Stranded != "" && d.CoveredBy != "" || d.DeliveredBy != ""
case Failed:
return d.DeliveredBy != ""
}
return false
}
// coveredStopSaid is how a covered stop reads: on the forge's status, in its view and note, and in `show`.
// coveredStopSaid is how a covered stop or failure reads: on the forge's status, in its view and note, and in `show`.
func (d *Delivery) coveredStopSaid() string {
return "covered after its stop: a later merge that contains it was delivered: " + d.CoveredBy
ended := "its stop"
if d.State == Failed {
ended = "its failure"
}
if d.Stranded != "" && d.CoveredBy != "" {
return "covered after " + ended + ": a later merge that contains it was delivered: " + d.CoveredBy
}
return "covered after " + ended + ": a later walk that contains it was delivered: " + d.DeliveredBy
}
// memberReached is whether a group counts a member as delivered: its change reached the machines, or had nothing to
// reach them with — delivered, history or covered, a superseded member a later delivered walk carried (issue 442),
// or a stop or failure covered since (issues 480 and 747).
func memberReached(d *Delivery) bool {
return d.State.Reached() || d.State == Superseded && d.CoveredBy != "" || d.coveredStop()
}
// stoppedFor is the member whose end stopped this one in its group: kept since novox/hq issue 747, and read from the
// stop's reason for a stop made before ("its group <id> stopped: <member> …").
func (d *Delivery) stoppedFor() string {
if d.StoppedFor != "" || d.State != Stopped || d.Group == "" {
return d.StoppedFor
}
prefix := "its group " + d.Group + " stopped: "
for i := len(d.Transitions) - 1; i >= 0; i-- {
t := d.Transitions[i]
if t.Event != EvStop {
continue
}
if t.By != byOwner || !strings.HasPrefix(t.Why, prefix) {
return ""
}
member, _, _ := strings.Cut(strings.TrimPrefix(t.Why, prefix), " ")
return strings.TrimRight(member, ",;:")
}
return ""
}
// stopLapsed is whether a member its group stopped for another member is stopped for nothing any more: that member's
// change reached the machines since — delivered, or its failure covered by a later walk (novox/hq issue 747). Its own
// change is still to deliver, by a later walk of its repository that contains it; its stop holds none back. find
// gives a delivery by its id.
func stopLapsed(d *Delivery, find func(string) *Delivery) bool {
if d.State != Stopped || d.coveredStop() {
return false
}
by := d.stoppedFor()
if by == "" || by == d.ID {
return false
}
o := find(by)
return o != nil && memberReached(o)
}
func (d *Delivery) shownOnItsPullRequest() bool {
@@ -91,6 +91,13 @@ type Forge interface {
// OpenPulls is every open pull request, with what its base requires and what its head has (novox/hq issue
// 347). Reads only.
OpenPulls() ([]OpenPull, error)
// Merge asks the forge's merge tool to merge a pull request at the head named (gitea_merge_pull_request, novox/hq
// issue 722): the one path every merge takes. Its refusal is the error, in its words.
Merge(owner, repo string, number int, head string) error
// Comment adds a comment to a pull request.
Comment(owner, repo string, number int, body string) error
// Unlabel takes a label off a pull request, by the label's id.
Unlabel(owner, repo string, number, label int) error
}
// ForgeChecks is what the forge says of a commit's checks (the gitea module's gitea_commit_statuses).
@@ -392,6 +399,19 @@ func (f toolForge) Statuses(owner, repo string, number int, commit, base string)
return &c, nil
}
func (f toolForge) Merge(owner, repo string, number int, head string) error {
return f.tool("gitea_merge_pull_request", map[string]any{"owner": owner, "repo": repo, "number": number, "head": head})
}
func (f toolForge) Comment(owner, repo string, number int, body string) error {
return f.tool("gitea_add_comment", map[string]any{"owner": owner, "repo": repo, "number": number, "body": body})
}
func (f toolForge) Unlabel(owner, repo string, number, label int) error {
return f.tool("gitea_api", map[string]any{"method": "DELETE",
"path": fmt.Sprintf("/repos/%s/%s/issues/%d/labels/%d", owner, repo, number, label)})
}
func (f toolForge) OpenPulls() ([]OpenPull, error) {
raw, err := f.ask("gitea.gitea_open_pulls", map[string]any{})
if err != nil {
@@ -0,0 +1,323 @@
package main
import (
"fmt"
"strings"
"time"
)
// Merge on ready (novox/hq issue 722).
//
// The operator, 2026-10-14: "the reviewer task should merge the request or mark the MR as ready to merge in any case".
// Before, a reviewed pull request with green checks waited until a session called the forge's merge tool, and again
// after every check the tool asked because the trunk had moved. Now the independent reviewer marks the pull request
// ready at the head it approved — the forge label `mesh/ready` and a comment "ready at <full head sha>" (approving
// through the forge is the operator's) — and this owner merges it, through the forge's own merge tool
// (gitea_merge_pull_request), the one path every merge takes:
//
// - only at the head the mark names, and only while that is the pull request's head: a push after the mark clears
// it — the label taken off, and said on the pull request — and the new head needs a mark of its own;
// - only when its delivery is ready here: its check passed against its trunk's head. A verdict out of date is judged
// again by the merge tool, which asks the check against the trunk's head itself (novox/hq ADR 0323 §5); the merge
// is asked again once a minute, and lands on the passing verdict;
// - a group member only at its turn in the group's order, and only once the group's composed check passed for its
// members' heads as they are;
// - a check that failed merges nothing: it is said once on the pull request, and as a stalled line, which the
// controller raises as the condition `delivery.<id>.stalled`. So is a mark the merge tool refused for an hour.
//
// What this owner does with a mark is kept on the delivery (Ready), said in its log, and the merge's transitions name
// it as their author.
// ReadyMark is a pull request's ready mark as the forge reads it (the gitea module's gitea_open_pulls): the head the
// newest "ready at" comment names, who wrote it and when, and the label's id. Head is empty when the label is on and
// no comment names a head.
type ReadyMark struct {
Head string `json:"head"`
By string `json:"by,omitempty"`
At string `json:"at,omitempty"`
LabelID int `json:"label_id"`
}
// ReadyLabel is the label that marks a pull request ready.
const ReadyLabel = "mesh/ready"
// readyTryEvery is how often a marked head's merge is asked while the merge tool refuses it: a check it asked takes
// minutes, and a refusal costs the forge a few reads.
const readyTryEvery = time.Minute
// readyRefusedFor is how long the merge tool may refuse a marked head before it is said as stalled: longer than a
// check takes (the controller's checkTimeout is 45 minutes), so a check on its way is never said.
const readyRefusedFor = time.Hour
// ReadyRun is what this owner did with a delivery's ready mark: kept on the delivery and shown whole by `show`.
type ReadyRun struct {
// Head, By and MarkedAt are the mark: the head it names, who marked it, when.
Head string `json:"head"`
By string `json:"by,omitempty"`
MarkedAt string `json:"marked_at,omitempty"`
// Seen is when this owner first read the mark.
Seen time.Time `json:"seen"`
// Waits is what the merge waits on, when it waits: a verdict, the group's turn or its composed check.
Waits string `json:"waits,omitempty"`
// Tries is how many times the merge was asked; LastTry when, and Answer what the merge tool last answered.
Tries int `json:"tries,omitempty"`
LastTry time.Time `json:"last_try"`
Answer string `json:"answer,omitempty"`
// RefusedSince is when the merge tool began refusing it without a break.
RefusedSince *time.Time `json:"refused_since,omitempty"`
// Merged is when the merge tool merged it.
Merged *time.Time `json:"merged,omitempty"`
// Failed is why it is not merged and will not be on this head: its check failed. Said once on the pull request.
Failed string `json:"failed,omitempty"`
}
// mergedBy is who a merge this owner made on a ready mark is said to be by, in its transitions.
func (r *ReadyRun) mergedBy() string {
if r == nil || r.Merged == nil {
return ""
}
by := byOwner + ", on the ready mark at " + shortOf(r.Head, 12)
if r.By != "" {
by += " by " + r.By
}
return by
}
// marked is a pull request read with a ready mark.
type marked struct {
pull OpenPull
// cleared is when the mark was cleared, the label taken off: not cleared again within readyTryEvery.
cleared time.Time
}
// pullKey is a pull request as a key: owner/repo#number.
func pullKey(p OpenPull) string {
return strings.ToLower(p.Owner+"/"+p.Repo) + fmt.Sprintf("#%d", p.Number)
}
// seeMarks keeps the open pull requests that carry a ready mark; under the lock. A pull request no longer open or
// no longer marked drops out.
func (h *Holder) seeMarks(open []OpenPull) {
seen := map[string]*marked{}
for _, p := range open {
if p.Ready == nil || p.HeadSHA == "" {
continue
}
k := pullKey(p)
m := &marked{pull: p}
if was := h.marks[k]; was != nil && was.pull.Ready != nil && was.pull.Ready.Head == p.Ready.Head &&
was.pull.HeadSHA == p.HeadSHA {
m.cleared = was.cleared
}
seen[k] = m
}
h.marks = seen
}
// sameFull is whether a mark names a head: the whole sha, or — read from an older forge module — at least twelve
// characters of it.
func sameFull(mark, head string) bool {
mark, head = strings.ToLower(mark), strings.ToLower(head)
return len(mark) >= 12 && strings.HasPrefix(head, mark)
}
// readyAsks works out, under the lock, what each ready mark needs: a mark cleared, a merge asked, a failure said.
// The asking happens outside the lock.
func (h *Holder) readyAsks(now time.Time) []func() {
var asks []func()
for k, m := range h.marks {
p, mark := m.pull, *m.pull.Ready
repository := strings.ToLower(p.Owner + "/" + p.Repo)
if !sameFull(mark.Head, p.HeadSHA) {
if !m.cleared.IsZero() && now.Sub(m.cleared) < readyTryEvery {
continue
}
m.cleared = now
asks = append(asks, h.clearMark(k, p, mark))
continue
}
d := h.deliveries[IDOf(repository, p.HeadSHA)]
if d == nil || d.State.Final() || d.MergedAs != "" {
continue // not announced yet, or past its merge
}
if d.Ready == nil || d.Ready.Head != mark.Head {
d.Ready = &ReadyRun{Head: mark.Head, By: mark.By, MarkedAt: mark.At, Seen: now}
h.Logf("[mesh-delivery] %s is marked ready at %s by %s: merged once its check passes against its trunk's head",
d.ID, shortOf(mark.Head, 12), orNone(mark.By))
h.keep(d)
}
r := d.Ready
if r.Merged != nil {
continue
}
if d.State == Rejected {
if r.Failed == "" {
r.Failed = "its check failed"
if d.Check != nil {
r.Failed += ": " + oneLine(d.Check.Summary)
if j := d.Check.Judged(); j != "" {
r.Failed += " (" + j + ")"
}
}
r.Waits = ""
h.Logf("[mesh-delivery] %s is marked ready and not merged: %s", d.ID, r.Failed)
h.keep(d)
asks = append(asks, h.sayOn(d, fmt.Sprintf("mesh-delivery: **not merged.** This pull request is marked "+
"ready at %s, and %s. A new head, marked ready once reviewed, or a recheck that passes, merges it.",
mark.Head, r.Failed)))
}
continue
}
r.Failed = ""
waits := h.readyWaits(d)
if waits != r.Waits {
r.Waits = waits
if waits != "" {
h.Logf("[mesh-delivery] %s is marked ready and waits: %s", d.ID, waits)
}
h.keep(d)
}
if waits != "" || (!r.LastTry.IsZero() && now.Sub(r.LastTry) < readyTryEvery) {
continue
}
r.Tries++
r.LastTry = now
h.keep(d)
asks = append(asks, h.askMerge(d.ID, p, mark.Head))
}
return asks
}
// readyWaits is what a marked delivery's merge waits on; empty when it is asked now. A ready delivery is asked; so is
// a proposed one whose verdict went out of date and whose check nobody asked again — the merge tool asks it.
func (h *Holder) readyWaits(d *Delivery) string {
switch {
case d.State == Ready:
case d.State == Proposed && d.outOfDate() != "" && d.AskedAgain == nil:
case d.State == Proposed && d.AskedAgain != nil:
return "its check against the trunk's head is asked and has not answered"
default:
return "its check has not passed yet (" + string(d.State) + ")"
}
if d.Group == "" {
return ""
}
return h.groupTurnToMerge(d)
}
// groupTurnToMerge is why a group member may not merge yet; empty at its turn: the group's composed check passed for
// its members' heads as they are and is not out of date, and every member before it in the group's order merged.
func (h *Holder) groupTurnToMerge(d *Delivery) string {
g := h.groups[d.Group]
if g == nil {
return ""
}
members := h.membersOf(g)
if state, why := GroupState(g, members); state != "ready" && state != "delivering" {
return fmt.Sprintf("its group %s is %s: %s", g.ID, state, why)
}
if c := g.Check; c == nil || (c.Verdict != "pass" && c.Verdict != "warning") || c.OutOfDate != "" ||
!sameMembers(c.Commits, commitsOf(members)) {
return "its group " + g.ID + "'s composed check has not passed for its members' heads"
}
if !containsString(g.Order, d.ID) {
return "its group " + g.ID + "'s order is not worked out for it yet"
}
for _, id := range g.Order {
if id == d.ID {
return ""
}
if o := h.deliveries[id]; o != nil && o.MergedAs == "" && !o.State.Final() {
return "its group " + g.ID + "'s order merges " + id + " first"
}
}
return ""
}
// askMerge asks the forge's merge tool to merge a marked head, and keeps its answer.
func (h *Holder) askMerge(id string, p OpenPull, head string) func() {
return func() {
err := h.Forge.Merge(p.Owner, p.Repo, p.Number, head)
h.mu.Lock()
defer h.mu.Unlock()
d := h.deliveries[id]
if d == nil || d.Ready == nil {
return
}
r, now := d.Ready, h.Now()
if err == nil {
r.Merged, r.Answer, r.RefusedSince = &now, "merged", nil
h.Logf("[mesh-delivery] %s merged on its ready mark at %s, its check passed against its trunk's head", id,
shortOf(head, 12))
h.keep(d)
return
}
answer := oneLine(err.Error())
if r.RefusedSince == nil {
r.RefusedSince = &now
}
if answer != r.Answer {
h.Logf("[mesh-delivery] %s is marked ready, and the merge tool did not merge it (asked again in %s): %s", id,
readyTryEvery, answer)
}
r.Answer = answer
h.keep(d)
}
}
// clearMark takes a mark off a pull request whose head moved after it, and says so on it.
func (h *Holder) clearMark(k string, p OpenPull, mark ReadyMark) func() {
return func() {
err := h.Forge.Unlabel(p.Owner, p.Repo, p.Number, mark.LabelID)
if err == nil {
err = h.Forge.Comment(p.Owner, p.Repo, p.Number, fmt.Sprintf("mesh-delivery: the ready mark at %s is cleared: "+
"the head is now %s, pushed after it. Mark the new head ready once it is reviewed.", mark.Head, p.HeadSHA))
}
if err != nil {
h.Logf("[mesh-delivery] %s's ready mark at %s names an older head than %s, and could not be cleared yet: %v",
k, shortOf(mark.Head, 12), shortOf(p.HeadSHA, 12), err)
return
}
h.Logf("[mesh-delivery] %s's ready mark at %s cleared: its head is now %s", k, shortOf(mark.Head, 12),
shortOf(p.HeadSHA, 12))
h.mu.Lock()
delete(h.marks, k)
h.mu.Unlock()
}
}
// sayOn comments on a delivery's pull request.
func (h *Holder) sayOn(d *Delivery, body string) func() {
owner, repo, number, id := d.Owner(), d.Repo(), d.Number, d.ID
return func() {
if err := h.Forge.Comment(owner, repo, number, body); err != nil {
h.Logf("[mesh-delivery] %s: could not say on its pull request: %v", id, err)
}
}
}
// readyLines are the marked heads that will not merge: a check that failed at once, a merge refused past its bound;
// as stalled lines, which the controller raises as conditions. Under the lock.
func (h *Holder) readyLines(now time.Time) []StalledLine {
var out []StalledLine
for _, d := range h.deliveries {
r := d.Ready
if r == nil || r.Merged != nil || d.MergedAs != "" || d.State.Final() {
continue
}
switch {
case r.Failed != "" && d.State == Rejected:
out = append(out, StalledLine{ID: d.ID, Number: d.Number, State: d.State, For: now.Sub(r.Seen).Round(time.Second).String(),
Bound: "0s", H2: "none: a new head marked ready, or a recheck that passes — the operator's",
Says: fmt.Sprintf("%s#%d is marked ready at %s and not merged: %s", d.Repository, d.Number,
shortOf(r.Head, 12), r.Failed)})
case r.RefusedSince != nil && now.Sub(*r.RefusedSince) >= readyRefusedFor:
out = append(out, StalledLine{ID: d.ID, Number: d.Number, State: d.State,
For: now.Sub(*r.RefusedSince).Round(time.Second).String(), Bound: readyRefusedFor.String(),
H2: "none: the merge is asked again each minute — the operator's",
Says: fmt.Sprintf("%s#%d is marked ready at %s, and the merge tool has refused it since %s: %s", d.Repository,
d.Number, shortOf(r.Head, 12), r.RefusedSince.UTC().Format(time.RFC3339), r.Answer)})
}
}
return out
}
@@ -0,0 +1,199 @@
package main
import (
"errors"
"strings"
"testing"
"time"
)
// **A pull request marked ready is merged by this owner** (novox/hq issue 722): before, a reviewed pull request with
// green checks waited until a session called the merge tool, and again after every check the tool asked because the
// trunk had moved. Each test below fails on a holder without the ready mark: nothing merged it.
const (
readyHead = "1111111111111111111111111111111111111111"
pushed = "2222222222222222222222222222222222222222"
)
// markedPull is an open pull request as gitea_open_pulls reads it, labelled mesh/ready at the head named.
func markedPull(repo string, number int, head, markedAt string) OpenPull {
owner, name, _ := strings.Cut(repo, "/")
return OpenPull{Owner: owner, Repo: name, Number: number, Base: "main", HeadSHA: head, Labels: []string{ReadyLabel},
Ready: &ReadyMark{Head: markedAt, By: "reviewer", At: "2026-10-14T19:00:00Z", LabelID: 31}}
}
func judgedAt(repo string, number int, sha, gate, trunk string) CheckedEvent {
c := verdict(repo, number, sha, gate)
c.Trunk = trunk
return c
}
func TestAReadyPullRequestMarkedAtItsHeadIsMergedThroughTheMergeTool(t *testing.T) {
w := newWorld(t)
id := IDOf("novox/app", readyHead)
w.h.PullUpdated(pr("novox/app", 7, readyHead, "feat/x", "src/a.go"))
w.h.Checked(judgedAt("novox/app", 7, readyHead, "pass", "t1t1t1t1t1t1"))
w.settleAll()
if len(w.forge.merges) != 0 {
t.Fatalf("an unmarked pull request was merged: %v", w.forge.merges)
}
w.forge.open = []OpenPull{markedPull("novox/app", 7, readyHead, readyHead)}
w.h.SeeOpenPulls()
w.settleAll()
if len(w.forge.merges) != 1 || w.forge.merges[0] != "novox/app#7@"+readyHead {
t.Fatalf("a ready, marked pull request was merged %v; wanted once, at the marked head", w.forge.merges)
}
d := w.delivery(id)
if d.Ready == nil || d.Ready.Merged == nil || d.Ready.By != "reviewer" || d.Ready.Tries != 1 {
t.Fatalf("what was done with the mark is not kept: %+v", d.Ready)
}
// Its merge's transition names who merged it.
w.h.PullMerged(merged("novox/app", 7, readyHead, "m7m7m7m7m7m7"))
w.h.WalkMoved(w.ctl.put(aWalk("plan-7", "novox/app", "m7m7m7m7m7m7", true)))
w.settleAll()
d = w.delivery(id)
var last Transition
for _, tr := range d.Transitions {
if tr.Event == EvMerged {
last = tr
}
}
if !strings.Contains(last.By, "ready mark at "+readyHead[:12]+" by reviewer") {
t.Fatalf("the merge's transition does not name the mark: %+v", last)
}
if len(w.forge.merges) != 1 {
t.Fatalf("merged again: %v", w.forge.merges)
}
}
func TestAPushAfterTheMarkClearsItAndNothingIsMerged(t *testing.T) {
w := newWorld(t)
w.h.PullUpdated(pr("novox/app", 7, readyHead, "feat/x", "src/a.go"))
w.h.PullUpdated(pr("novox/app", 7, pushed, "feat/x", "src/a.go"))
w.h.Checked(judgedAt("novox/app", 7, pushed, "pass", "t1t1t1t1t1t1"))
w.forge.open = []OpenPull{markedPull("novox/app", 7, pushed, readyHead)}
w.h.SeeOpenPulls()
w.settleAll()
if len(w.forge.merges) != 0 {
t.Fatalf("a head pushed after the mark was merged: %v", w.forge.merges)
}
if len(w.forge.unlabels) != 1 || w.forge.unlabels[0] != "novox/app#7-31" {
t.Fatalf("the stale mark's label was not taken off: %v", w.forge.unlabels)
}
if len(w.forge.comments) != 1 || !strings.Contains(w.forge.comments[0], "cleared") ||
!strings.Contains(w.forge.comments[0], pushed) {
t.Fatalf("the cleared mark is not said on the pull request: %v", w.forge.comments)
}
}
func TestAMovedTrunkIsJudgedAgainByTheMergeToolAndMergedOnItsVerdict(t *testing.T) {
w := newWorld(t)
id := IDOf("novox/app", readyHead)
w.h.PullUpdated(pr("novox/app", 7, readyHead, "feat/x", "src/a.go"))
w.h.Checked(judgedAt("novox/app", 7, readyHead, "pass", "t1t1t1t1t1t1"))
// Another merge moves the trunk: its verdict is out of date, and it goes back to proposed.
w.h.PullMerged(merged("novox/app", 9, "9999999999999999", "t2t2t2t2t2t2"))
if w.state(id) != Proposed {
t.Fatalf("out of date, it is %s", w.state(id))
}
// The merge tool asks the check against the trunk's head and refuses: nothing merged.
w.forge.refuse = errors.New("the forge refused gitea_merge_pull_request: The check of head merged into trunk was asked")
w.forge.open = []OpenPull{markedPull("novox/app", 7, readyHead, readyHead)}
w.h.SeeOpenPulls()
w.settleAll()
if len(w.forge.merges) != 1 {
t.Fatalf("the merge tool was not asked once to judge it again: %v", w.forge.merges)
}
// The tool's ask is the head announced again; it waits for that verdict, asking nothing.
w.h.PullUpdated(pr("novox/app", 7, readyHead, "feat/x", "src/a.go"))
w.later(5 * time.Minute)
w.settleAll()
if len(w.forge.merges) != 1 || !strings.Contains(w.delivery(id).Ready.Waits, "asked") {
t.Fatalf("asked again while its check was on its way: %v (%+v)", w.forge.merges, w.delivery(id).Ready)
}
// It passes against the trunk's head: merged, without anyone asking.
w.forge.refuse = nil
w.h.Checked(judgedAt("novox/app", 7, readyHead, "pass", "t2t2t2t2t2t2"))
w.settleAll()
if w.state(id) != Ready || len(w.forge.merges) != 2 || w.delivery(id).Ready.Merged == nil {
t.Fatalf("on its passing verdict it was not merged: %s %v %+v", w.state(id), w.forge.merges, w.delivery(id).Ready)
}
}
func TestAMarkedHeadWhoseCheckFailedIsNotMergedAndIsSaid(t *testing.T) {
w := newWorld(t)
id := IDOf("novox/app", readyHead)
w.h.PullUpdated(pr("novox/app", 7, readyHead, "feat/x", "src/a.go"))
w.h.Checked(judgedAt("novox/app", 7, readyHead, "fail", "t1t1t1t1t1t1"))
w.forge.open = []OpenPull{markedPull("novox/app", 7, readyHead, readyHead)}
w.h.SeeOpenPulls()
w.settleAll()
w.later(10 * time.Minute)
w.h.SeeOpenPulls()
w.settleAll()
if len(w.forge.merges) != 0 {
t.Fatalf("a head whose check failed was merged: %v", w.forge.merges)
}
if len(w.forge.comments) != 1 || !strings.Contains(w.forge.comments[0], "not merged") {
t.Fatalf("the failure is not said once on the pull request: %v", w.forge.comments)
}
lines := w.h.Stalled()
if len(lines) != 1 || lines[0].ID != id || lines[0].State != Rejected || !strings.Contains(lines[0].Says, "marked ready") {
t.Fatalf("the failure is not a stalled line, which the controller raises as a condition: %+v", lines)
}
}
func TestAMarkRefusedForAnHourIsSaid(t *testing.T) {
w := newWorld(t)
w.h.PullUpdated(pr("novox/app", 7, readyHead, "feat/x", "src/a.go"))
w.h.Checked(judgedAt("novox/app", 7, readyHead, "pass", "t1t1t1t1t1t1"))
w.forge.refuse = errors.New("the forge refused gitea_merge_pull_request: mesh/repo-check is pending")
w.forge.open = []OpenPull{markedPull("novox/app", 7, readyHead, readyHead)}
w.h.SeeOpenPulls()
w.settleAll()
w.later(30 * time.Second)
w.settleAll()
if len(w.forge.merges) != 1 {
t.Fatalf("asked again within the minute: %v", w.forge.merges)
}
w.later(readyRefusedFor)
w.settleAll()
if len(w.forge.merges) != 2 {
t.Fatalf("not asked again after a minute: %v", w.forge.merges)
}
if lines := w.h.Stalled(); len(lines) != 1 || !strings.Contains(lines[0].Says, "refused it since") {
t.Fatalf("a mark refused for an hour is not said: %+v", lines)
}
}
func TestAGroupMergesItsMarkedMembersInOrderOnlyOnceComposed(t *testing.T) {
w := newWorld(t)
const ctlHead, catHead = "c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1", "d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2"
ctl, cat := IDOf("novox/mesh-controller", ctlHead), IDOf("novox/mesh-catalog", catHead)
w.ctl.order = func(ms []Member) Order { return Order{Order: []string{ctl, cat}} }
w.h.PullUpdated(pr("novox/mesh-controller", 1, ctlHead, "feat/x", "internal/a.go"))
w.h.PullUpdated(pr("novox/mesh-catalog", 2, catHead, "feat/x", "modules/app/module.json"))
w.h.Checked(judgedAt("novox/mesh-controller", 1, ctlHead, "pass", "t1t1t1t1t1t1"))
w.h.Checked(judgedAt("novox/mesh-catalog", 2, catHead, "pass", "t3t3t3t3t3t3"))
w.forge.open = []OpenPull{markedPull("novox/mesh-controller", 1, ctlHead, ctlHead),
markedPull("novox/mesh-catalog", 2, catHead, catHead)}
w.h.SeeOpenPulls()
w.settleAll()
if len(w.forge.merges) != 0 {
t.Fatalf("a member merged before its group's composed check passed: %v", w.forge.merges)
}
w.h.Checked(CheckedEvent{Group: "feat/x", ID: "check-1", Verdict: "pass", Summary: "both compose"})
w.settleAll()
if len(w.forge.merges) != 1 || w.forge.merges[0] != "novox/mesh-controller#1@"+ctlHead {
t.Fatalf("composed, the first member in the group's order was not merged alone: %v", w.forge.merges)
}
if waits := w.delivery(cat).Ready.Waits; !strings.Contains(waits, "merges "+ctl+" first") {
t.Fatalf("the second member does not say it waits its turn: %q", waits)
}
w.h.PullMerged(merged("novox/mesh-controller", 1, ctlHead, "m1m1m1m1m1m1"))
w.settleAll()
if len(w.forge.merges) != 2 || w.forge.merges[1] != "novox/mesh-catalog#2@"+catHead {
t.Fatalf("after the first member merged, the next was not: %v", w.forge.merges)
}
}
@@ -3,6 +3,7 @@ package main
import (
"strings"
"testing"
"time"
)
// A recheck puts the merge check's statuses back to pending before it asks the check: the forge keeps the
@@ -95,3 +96,80 @@ func TestARecheckTheForgeCannotTakeIsRefused(t *testing.T) {
t.Fatalf("refused, it moved to %s", s)
}
}
// novox/hq issue 867, ADR 0323 §6: "the composed check runs again when a member is rechecked". A group whose composed
// check failed stayed rejected after a member was rechecked and passed, because the kept verdict was for the same heads
// and nothing marked it out of date.
func TestAMembersRecheckAsksTheGroupsComposedCheckAgain(t *testing.T) {
w := newWorld(t)
w.h.PullUpdated(pr("novox/a", 1, "a1a1a1a1a1a1a1", "feat/z"))
w.h.PullUpdated(pr("novox/b", 2, "b2b2b2b2b2b2b2", "feat/z"))
w.h.Checked(verdict("novox/a", 1, "a1a1a1a1a1a1a1", "pass"))
w.h.Checked(verdict("novox/b", 2, "b2b2b2b2b2b2b2", "pass"))
w.settleAll()
groupAsks := func() (n int) {
for _, c := range w.ctl.checks {
if strings.HasPrefix(c, "feat/z:") {
n++
}
}
return
}
if groupAsks() != 1 {
t.Fatalf("the group was asked %v", w.ctl.checks)
}
w.h.Checked(CheckedEvent{Group: "feat/z", ID: "check-1", Verdict: "fail", Summary: "they do not compose"})
w.settleAll()
if state, _ := GroupState(w.h.groups["feat/z"], w.h.membersOf(w.h.groups["feat/z"])); state != "rejected" {
t.Fatalf("the group is %s", state)
}
if _, err := w.h.Recheck(IDOf("novox/b", "b2b2b2b2b2b2b2"), "the judge was fixed", "a person"); err != nil {
t.Fatal(err)
}
// The member's own check passes again, at the same head, a while later.
w.later(5 * time.Minute)
w.h.Checked(verdict("novox/b", 2, "b2b2b2b2b2b2b2", "pass"))
w.settleAll()
if groupAsks() != 2 {
t.Fatalf("a member's recheck did not ask the group again: %v", w.ctl.checks)
}
if g := w.h.groups["feat/z"]; g.Check == nil || g.Check.Verdict != "" || g.Check.Asked == "check-1" {
t.Fatalf("the old verdict still stands: %+v", g.Check)
}
}
// A recheck made while the group's composed check is still running is not lost: the old run's answer lands, and the
// group is still asked again once the member is ready.
func TestARecheckWhileTheComposedCheckRunsAsksTheGroupAgain(t *testing.T) {
w := newWorld(t)
w.h.PullUpdated(pr("novox/a", 1, "a1a1a1a1a1a1a1", "feat/z"))
w.h.PullUpdated(pr("novox/b", 2, "b2b2b2b2b2b2b2", "feat/z"))
w.h.Checked(verdict("novox/a", 1, "a1a1a1a1a1a1a1", "pass"))
w.h.Checked(verdict("novox/b", 2, "b2b2b2b2b2b2b2", "pass"))
w.settleAll() // the composed check check-1 is asked and running: no verdict yet
if _, err := w.h.Recheck(IDOf("novox/b", "b2b2b2b2b2b2b2"), "the judge was fixed", "a person"); err != nil {
t.Fatal(err)
}
// The old run answers.
w.h.Checked(CheckedEvent{Group: "feat/z", ID: "check-1", Verdict: "fail", Summary: "they do not compose"})
w.later(5 * time.Minute)
w.h.Checked(verdict("novox/b", 2, "b2b2b2b2b2b2b2", "pass"))
w.settleAll()
n := 0
for _, c := range w.ctl.checks {
if strings.HasPrefix(c, "feat/z:") {
n++
}
}
if n != 2 {
t.Fatalf("the group was not asked again after a recheck made while its check ran: %v", w.ctl.checks)
}
if g := w.h.groups["feat/z"]; g.Check == nil || g.Check.Verdict != "" || g.Check.OutOfDate != "" {
t.Fatalf("the old run's answer stands: %+v", g.Check)
}
// The old run answering again is history.
w.h.Checked(CheckedEvent{Group: "feat/z", ID: "check-1", Verdict: "fail", Summary: "late"})
if g := w.h.groups["feat/z"]; g.Check.Verdict != "" {
t.Fatalf("a late answer of the old run was taken: %+v", g.Check)
}
}
@@ -498,7 +498,12 @@ func Apply(d *Delivery, ev Event, act bool, f Facts) (Transition, error) {
if why == "" {
why = r.Guard
}
t := Transition{At: f.Now, From: d.State, To: r.To, Event: ev, Why: why, By: f.By}
by := f.By
if by == "" && (ev == EvMerged || ev == EvDone) && d.MergedAs != "" {
// A merge this owner made on a ready mark names it as its author (novox/hq issue 722).
by = d.Ready.mergedBy()
}
t := Transition{At: f.Now, From: d.State, To: r.To, Event: ev, Why: why, By: by}
d.State, d.Since = r.To, f.Now
d.Transitions = append(d.Transitions, t)
if len(d.Transitions) > keptTransitions {
@@ -72,6 +72,9 @@ type OpenPull struct {
// than that says none: the head is then read by Statuses alone, as before, and a pending check is not told
// from an answered one.
Checks map[string]HeadCheck `json:"checks,omitempty"`
// Labels are its labels by name, and Ready its ready mark, when it carries one (novox/hq issue 722).
Labels []string `json:"labels,omitempty"`
Ready *ReadyMark `json:"ready,omitempty"`
}
// HeadCheck is one status on a head as the forge keeps it: pending, success, failure, error or warning.
@@ -201,6 +204,7 @@ func (h *Holder) SeeOpenPulls() {
seen[id] = u
}
h.unchecked = seen
h.seeMarks(open)
}
// started is whether the head's check started, rather than was never asked: it has any merge check, or a delivery
@@ -72,6 +72,10 @@ func (h *Holder) waitsFor(d *Delivery) string {
return "its walk: " + clip(d.Walk.Note, 120)
}
case Stopped:
if h.stopLapsed(d) {
return "a later walk of its repository that contains its merge: its group stopped it for " + d.stoppedFor() +
", whose change was delivered since (novox/hq issue 747)"
}
if strandedUncovered(d) {
return "a later delivered merge to carry " + strings.Join(d.notMovedBy(), ", ") + ": it is stopped, and " +
"listed by stalled until one does (novox/hq issue 480)"
@@ -150,6 +154,21 @@ type GroupLine struct {
Contradictions []Contradiction `json:"contradictions,omitempty"`
Check string `json:"composed,omitempty"`
Members []string `json:"members"`
// Details are the members again, one object each, with what a reader needs to find the pull request's
// siblings: its number, full head, base branch and whether it is open (novox/hq ADR 0382 §1). Additive:
// `members` keeps its text form for older readers.
Details []MemberLine `json:"member_details,omitempty"`
}
// MemberLine is one member of a group, as `groups` lists it beside its text line.
type MemberLine struct {
ID string `json:"id"`
Repository string `json:"repository"`
State State `json:"state"`
Number int `json:"number"`
Head string `json:"head"`
Base string `json:"base"`
Open bool `json:"open"`
}
// Groups is the `groups` verb.
@@ -170,6 +189,9 @@ func (h *Holder) Groups(all bool) []GroupLine {
Contradictions: g.Contradictions}
for _, d := range members {
l.Members = append(l.Members, fmt.Sprintf("%s (%s)", d.ID, d.State))
l.Details = append(l.Details, MemberLine{ID: d.ID, Repository: d.Repository, State: d.State,
Number: d.Number, Head: d.Commit, Base: d.Base,
Open: !d.State.Final() && d.MergedAs == "" && !d.ClosedUnmerged})
}
if g.Check != nil {
l.Check = g.Check.Verdict
@@ -249,6 +271,7 @@ func (h *Holder) Stalled() []StalledLine {
}
out = append(out, h.withChecks(h.uncheckedLines(now))...)
out = append(out, h.orphanLines(now)...)
out = append(out, h.readyLines(now)...)
sort.Slice(out, func(i, j int) bool { return out[i].ID < out[j].ID })
return out
}
@@ -287,6 +310,16 @@ func (h *Holder) Recheck(id, why, by string) (string, error) {
return "", fmt.Errorf("%v (its merge check says pending; the forge's next announcement asks it)", err)
}
d.Check = nil
// The composed check of its group goes out of date (novox/hq ADR 0323 §6, issue 867): it was judged for these very
// heads (running or answered: a run's answer is taken by its id and must not stand for the recheck), so without this the kept verdict (a failure above all) stands and the group is never asked again. It is
// asked again once the member is judged again, when every member is ready.
if g := h.groups[d.Group]; g != nil && g.Check != nil {
now := h.Now()
g.Check.OutOfDate = fmt.Sprintf("composed check out of date: %s was rechecked (%s)", d.ID, why)
g.Check.OutOfDateAt = now
g.Updated = now
h.keepGroup(g)
}
h.moved(d, []Transition{t})
m := MemberOf(d)
h.mu.Unlock()
+4 -1
View File
@@ -47,7 +47,10 @@
"gitea.gitea_commit_status",
"gitea.gitea_commit_statuses",
"gitea.gitea_contains",
"gitea.gitea_open_pulls"
"gitea.gitea_open_pulls",
"gitea.gitea_merge_pull_request",
"gitea.gitea_add_comment",
"gitea.gitea_api"
],
"state": [
"deliveries",