Show the backlog of build asks per language and kind: waiting, held by a node, and the oldest waiting ask's age (issue 1887 A8a)

modules/mesh-delivery-view/cmd/mesh-delivery-view/integration_test.go

@@ -602,3 +602,97 @@ func TestProcessWholeWritesTheAnswerToAnAskNeedingNoProof(t *testing.T) { // 129
 	}
 	wait(func(s view.Snapshot) bool { return len(s.Asks) == 1 && s.Asks[0].Key == mkey }, "the answered ask never left the list")
 }
+
+// TestProcessWholeShowsTheBacklogOfBuildAsks starts the program whole against a bus holding the go seat's work-queue stream
+// with its three shared workers, three check asks waiting and one held by a node: the socket's backlog row names three
+// waiting, one held, and the oldest's age (1887/A8a AC; compiled in the unit run, run by the merge check).
+func TestProcessWholeShowsTheBacklogOfBuildAsks(t *testing.T) { // 1887/A8a AC
+	srv, err := server.NewServer(&server.Options{Host: "127.0.0.1", Port: -1, JetStream: true, StoreDir: t.TempDir(), NoLog: true, NoSigs: true})
+	if err != nil {
+		t.Fatal(err)
+	}
+	go srv.Start()
+	if !srv.ReadyForConnections(10 * time.Second) {
+		t.Fatal("the bus did not come up")
+	}
+	t.Cleanup(srv.Shutdown)
+	nc, err := nats.Connect(srv.ClientURL())
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Cleanup(nc.Close)
+	js, err := jetstream.New(nc)
+	if err != nil {
+		t.Fatal(err)
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
+	defer cancel()
+	stream, _ := delivery.AskStream(delivery.AskGo)
+	st, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: stream, Subjects: []string{"mesh.seat.node-build-go.accept.>"}, Retention: jetstream.WorkQueuePolicy})
+	if err != nil {
+		t.Fatal(err)
+	}
+	var checkWorker jetstream.Consumer
+	for _, kind := range delivery.AskKinds {
+		name, _ := delivery.AskWorker(delivery.AskGo, kind)
+		subject, _ := delivery.AskSubject(delivery.AskGo, kind)
+		c, err := st.CreateOrUpdateConsumer(ctx, jetstream.ConsumerConfig{Durable: name, FilterSubject: subject, AckPolicy: jetstream.AckExplicitPolicy})
+		if err != nil {
+			t.Fatal(err)
+		}
+		if kind == delivery.AskCheck {
+			checkWorker = c
+		}
+	}
+	subject, _ := delivery.AskSubject(delivery.AskGo, delivery.AskCheck)
+	oldest := time.Now().UTC().Add(-2 * time.Minute)
+	for i, id := range []string{"m0000001aaaaaa", "m0000002bbbbbb", "m0000003cccccc", "m0000004dddddd"} {
+		body, err := delivery.EncodeStepAsk(delivery.StepAsk{ID: id, Kind: delivery.AskCheck, Language: delivery.AskGo, Asker: "mesh-delivery-checker", At: oldest.Add(time.Duration(i) * time.Second),
+			Check: &delivery.CheckJob{Fetch: map[string]any{"repository": "novox/x", "commit": "0123456789abcdef0123456789abcdef01234567"}, Levels: []string{delivery.LevelUnit}}})
+		if err != nil {
+			t.Fatal(err)
+		}
+		if _, err := js.Publish(ctx, subject, body); err != nil {
+			t.Fatal(err)
+		}
+	}
+	// A node pulls one and holds it unacknowledged.
+	held, err := checkWorker.Fetch(1)
+	if err != nil {
+		t.Fatal(err)
+	}
+	for range held.Messages() {
+	}
+
+	dir, err := os.MkdirTemp(os.Getenv("TMPDIR"), "i")
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Cleanup(func() { os.RemoveAll(dir) })
+	broker := filepath.Join(dir, "broker")
+	cred, _ := json.Marshal(credential{URL: srv.ClientURL(), Node: "n", Module: "mesh-delivery-view", User: "mesh-delivery-view", Password: "x"})
+	if err := os.WriteFile(broker, cred, 0o600); err != nil {
+		t.Fatal(err)
+	}
+	socket := filepath.Join(dir, "run", "cli.sock")
+	env := map[string]string{"MESH_BROKER_FILE": broker, "MESH_DELIVERY_SOCKET": socket, "MESH_DELIVERY_WEB": "127.0.0.1:0"}
+	runCtx, stop := context.WithCancel(ctx)
+	done := make(chan error, 1)
+	go func() { done <- run(runCtx, func(k string) string { return env[k] }, io.Discard) }()
+	t.Cleanup(func() { stop(); <-done })
+
+	var last view.Snapshot
+	for deadline := time.Now().Add(20 * time.Second); time.Now().Before(deadline); time.Sleep(100 * time.Millisecond) {
+		s, err := view.Ask(socket)
+		if err != nil {
+			continue
+		}
+		last = s
+		for _, b := range s.Backlog {
+			if b.Language == "go" && b.Kind == "check" && b.Waiting == 3 && b.Held == 1 && b.OldestSecs >= 118 {
+				return
+			}
+		}
+	}
+	t.Fatalf("the backlog never showed three waiting asks with the oldest's age; last: %+v", last.Backlog)
+}

modules/mesh-delivery-view/cmd/mesh-delivery-view/main.go

@@ -138,6 +138,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
 		return err
 	}
 	model.SetWriter(kvWriter{ctx: ctx, js: js})
+	model.SetBacklog(view.BacklogOver(js, 5*time.Second))
 	for _, bucket := range []string{delivery.HaveBucket, view.AssignmentsBucket, view.BuildsBucket, delivery.PresenceBucket, delivery.StuckBucket, delivery.CandidatesBucket, delivery.ModulesBucket, delivery.AsksBucket, delivery.AnswersBucket, delivery.BuilderBuildsBucket, delivery.AssignerDifferencesBucket, delivery.AssignerAsksBucket, delivery.ViewAnswersBucket} {
 		go follow(ctx, js, bucket, model, logger)
 	}