Hand a delivery build out before pull-request checks (hq issue 558)

A merged change's walk waited minutes on its build behind checks in a
first-come queue. The queue now hands deliveries out first, a check that
has waited ten minutes before deliveries asked after it, and says each
ask's kind; the order is kept in the work queue itself.

cmd/mesh-controller/queue.go

@@ -81,6 +81,39 @@ func listQueue(ctx context.Context, args []string, w io.Writer) error {
 	return nil
 }
 
+// keepingTheQueueInOrder puts the build queue in order every watchEvery while this controller acts
+// (novox/hq issue 558). An ask puts it in order as it joins; this is for the clock: a check comes of age
+// with nothing asked.
+func keepingTheQueueInOrder(ctx context.Context, open *stores, js *broker.JetStream) {
+	tick := time.NewTicker(watchEvery)
+	defer tick.Stop()
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case <-tick.C:
+		}
+		if !link.Holding() || js == nil {
+			continue
+		}
+		entries, err := open.inventory.Catalogued(ctx)
+		if err != nil {
+			fmt.Printf("the build queue is not put in order: the catalogue cannot be read: %v\n", err)
+			continue
+		}
+		seat := buildSeatAmong(entries)
+		ordering, cancel := context.WithTimeout(ctx, watchEvery)
+		moved, err := link.PutInOrder(ordering, js, seat)
+		cancel()
+		if len(moved) > 0 {
+			fmt.Printf("the %s queue put in order: %s moved behind what goes first\n", seat, strings.Join(moved, ", "))
+		}
+		if err != nil {
+			fmt.Printf("the %s queue could not be put in order: %v\n", seat, err)
+		}
+	}
+}
+
 // queueAnswer is the queue as data (novox/hq issue 455): what a caller reads to say how far back an ask
 // waits — "waiting in the build queue behind 2" — which the table alone could not tell it. Text is the
 // table, written from the same read: the verb answers it as its output and the rest as its answer, so the
@@ -100,7 +133,8 @@ type queueAnswer struct {
 type queuedAnswer struct {
 	ID    string `json:"id"`
 	State string `json:"state"`
-	// Kind is "check" for a pull request's merge check, "build" for anything else.
+	// Kind is "delivery" for a build a walk waits for, "check" for a pull request's check and any ask
+	// whose kind cannot be told (link.KindOf, novox/hq issue 558).
 	Kind       string `json:"kind"`
 	Repository string `json:"repository"`
 	Path       string `json:"path,omitempty"`
@@ -129,10 +163,7 @@ func queueAnswerOf(q link.Queue, now time.Time) queueAnswer {
 	out := queueAnswer{Seat: q.Seat, Waiting: len(waiting), InFlight: len(running), Dead: len(dead),
 		MaxDeliver: q.MaxDeliver, Asks: []queuedAnswer{}, Text: queueText(q, now)}
 	one := func(a link.QueuedAsk, position int) queuedAnswer {
-		kind := "build"
-		if a.Request.Check != nil {
-			kind = "check"
-		}
+		kind := link.KindOf(a.Request)
 		since := a.AskedAt
 		if a.State == link.AskInFlight && a.On != "" && !a.Started.IsZero() {
 			since = a.Started
@@ -189,19 +220,20 @@ func queueText(q link.Queue, now time.Time) string {
 			case a.Was != "":
 				where = fmt.Sprintf("handed back by %s, to be handed out again", a.Was)
 			}
-			fmt.Fprintf(&b, "  %-26s %s — %s, %s (seq %d)\n", a.ID, what(a), where, ago(a.AskedAt), a.Seq)
+			fmt.Fprintf(&b, "  %-26s %-8s %s — %s, %s (seq %d)\n", a.ID, link.KindOf(a.Request), what(a), where,
+				ago(a.AskedAt), a.Seq)
 		}
 	}
 	if len(waiting) > 0 {
-		fmt.Fprintln(&b, "\nwaiting:")
+		fmt.Fprintln(&b, "\nwaiting, in the order they are handed out:")
 		for _, a := range waiting {
-			fmt.Fprintf(&b, "  %-26s %s — %s (seq %d)\n", a.ID, what(a), ago(a.AskedAt), a.Seq)
+			fmt.Fprintf(&b, "  %-26s %-8s %s — %s (seq %d)\n", a.ID, link.KindOf(a.Request), what(a), ago(a.AskedAt), a.Seq)
 		}
 	}
 	if len(dead) > 0 {
 		fmt.Fprintf(&b, "\ndead — handed out as often as the worker allows (%d) and never settled, still held:\n", q.MaxDeliver)
 		for _, a := range dead {
-			fmt.Fprintf(&b, "  %-26s %s — %s (seq %d)\n", a.ID, what(a), ago(a.AskedAt), a.Seq)
+			fmt.Fprintf(&b, "  %-26s %-8s %s — %s (seq %d)\n", a.ID, link.KindOf(a.Request), what(a), ago(a.AskedAt), a.Seq)
 		}
 	}
 	switch {

cmd/mesh-controller/queue_answer_test.go

@@ -24,7 +24,7 @@ func aQueueOfEachKind(now time.Time) link.Queue {
 			State: link.AskInFlight, On: "ace", Started: started,
 			Request: link.BuildRequest{ID: "build-running", Check: &link.CheckRequest{Owner: "novox", Repo: "mesh-host"}}},
 		{Seq: 3, ID: "build-first", Repository: "ssh://forge/novox/mesh-catalog", Ref: "main", AskedAt: asked,
-			State: link.AskWaiting},
+			State: link.AskWaiting, Request: link.BuildRequest{ID: "build-first", Repository: "ssh://forge/novox/mesh-catalog"}},
 		{Seq: 4, ID: "build-second", Repository: "ssh://forge/novox/mesh-catalog", Path: "modules/gitea", Ref: aCommit,
 			AskedAt: now.Add(-time.Minute), State: link.AskWaiting,
 			Request: link.BuildRequest{ID: "build-second", Check: &link.CheckRequest{Owner: "novox", Repo: "mesh-catalog"}}},
@@ -51,7 +51,7 @@ func TestTheQueueVerbAnswersEachAskAsData(t *testing.T) {
 		t.Errorf("in flight: %+v", running)
 	}
 	first, second := byID["build-first"], byID["build-second"]
-	if first.Position != 1 || first.Kind != "build" || first.Commit != "" || first.Ref != "main" || first.Machine != "" ||
+	if first.Position != 1 || first.Kind != "delivery" || first.Commit != "" || first.Ref != "main" || first.Machine != "" ||
 		!first.Since.Equal(now.Add(-5*time.Minute)) {
 		t.Errorf("first waiting: %+v", first)
 	}
@@ -62,6 +62,13 @@ func TestTheQueueVerbAnswersEachAskAsData(t *testing.T) {
 	if dead := byID["build-dead"]; dead.Position != 0 || dead.State != link.AskDead {
 		t.Errorf("dead: %+v", dead)
 	}
+	// And the table says each ask's kind too (novox/hq issue 558).
+	for _, line := range []string{"build-running              check    ", "build-first                delivery ",
+		"build-second               check    ", "build-dead                 check    "} {
+		if !strings.Contains(got.Text, line) {
+			t.Errorf("the table does not say %q:\n%s", line, got.Text)
+		}
+	}
 	// In the table's order: in flight, then the waiting by position, then the dead.
 	var order []string
 	for _, a := range got.Asks {

cmd/mesh-controller/watchdogs.go

@@ -739,6 +739,9 @@ func watchTheMesh(ctx context.Context, open *stores, server *link.Server, bus li
 	// and the answer chosen is performed on its warrant.
 	startAsking(watching, open, server, bus.Conn, keeper)
 	go forgettingOldHeals(watching, open.inventory)
+	// And the build queue's order (novox/hq issue 558): a check that has waited its while goes ahead of the
+	// deliveries asked after it even when nothing new is asked.
+	go keepingTheQueueInOrder(watching, open, server.JetStream())
 	fmt.Printf("watching the mesh: %d signal(s) every %s, %d probe(s) every %s; what is wrong is kept in %s "+
 		"and said as %s events\n", len(watchedRows()), watchEvery, len(runnableProbes()), doctorEvery,
 		broker.ConditionsBucket, conditions.Seat)