Checker: an ask the build seat gives up on ends its try as error and asks a check again; a try no node takes ends at the step bound; checker_try_end ends one by hand (issue 734)

A check ask handed out five times during a bus reload was given up into DEAD_LETTERS and no build node wrote a step for it,
so try-1793017340003993788 stayed running for ever, held one of the four try places, and checker_try of its head kept
answering its id. The checker now hears the copies DEAD_LETTERS keeps of the build seats' given-up asks (an ordered
consumer from its start; the grant is mesh-sdk/mesh-controller DeadLetterReads, branch fix/734-lost-ask) and ends the
step it waits for: a try ends as error saying why, a check is asked again under its claim, up to three times, then fails
as not heard. A try whose ask no node takes within the step bound ends the same way (a check's backlog stays no fault,
ADR 0525 §9). checker_try_end ends a try by hand, freeing its place and stopping its judging.

modules/mesh-delivery-checker/cmd/mesh-delivery-checker-tools/main.go

@@ -76,6 +76,13 @@ func (r runsOf) CommitOf(module string) string {
 type stateStore struct {
 	check.StateRead
 	create func(key string, value []byte) error
+	entry  func(key string) (json.RawMessage, uint64, error)
+	update func(key string, value []byte, revision uint64) error
+}
+
+func (s stateStore) Entry(key string) (json.RawMessage, uint64, error) { return s.entry(key) }
+func (s stateStore) Update(key string, value []byte, revision uint64) error {
+	return s.update(key, value, revision)
 }
 
 func (s stateStore) Keys() ([]string, error)                 { return s.StateRead.Keys() }
@@ -85,11 +92,12 @@ func (s stateStore) Create(key string, value []byte) error   { return s.create(k
 // Trying is what the try's tools work on: the tries, and the gate's trunk heads.
 type Trying struct {
 	Tries  check.TryStore
+	Ender  check.TryEnder
 	Trunks func() (map[string]string, error)
 	Now    func() time.Time
 }
 
-// TryTools are the try's two tools (ADR 0501 §5): ask one, read one. mesh-controller's try calls them.
+// TryTools are the try's tools (ADR 0501 §5): ask one, read one, and end one by hand (issue 734). mesh-controller's try calls them.
 func TryTools(tr Trying) []stdio.Tool {
 	str := func(args map[string]any, k string) string { s, _ := args[k].(string); return s }
 	return []stdio.Tool{
@@ -134,6 +142,28 @@ func TryTools(tr Trying) []stdio.Tool {
 				}
 				return check.ReadTry(tr.Tries, str(args, "id"))
 			}},
+		{Name: check.TryEndTool,
+			Description: "End a try that waits or runs, as error, by hand: say why (kept in what it says). Its place among the 4 is free at once, " +
+				"the checker stops waiting for its asks, and the next " + check.TryTool + " of its head asks anew with a new id. For a try whose ask " +
+				"was lost (no step runs for it in checker_status); the checker ends one by itself when the build seat gives its ask up, or no node " +
+				"takes it within the step bound. A try that ended already is refused.",
+			Input: map[string]any{"type": "object", "required": []string{"id", "why"}, "properties": map[string]any{
+				"id":  map[string]any{"type": "string", "description": "the try's id, try-<digits>"},
+				"why": map[string]any{"type": "string", "description": "why it is ended, in words"}}},
+			Run: func(args map[string]any) (any, error) {
+				if tr.Ender == nil {
+					return nil, fmt.Errorf("the tries state cannot be written here")
+				}
+				now := time.Now
+				if tr.Now != nil {
+					now = tr.Now
+				}
+				t, err := check.EndTry(tr.Ender, str(args, "id"), str(args, "why"), now())
+				if err != nil {
+					return nil, err
+				}
+				return map[string]any{"id": t.ID, "state": t.State, "said": t.Said}, nil
+			}},
 	}
 }
 
@@ -228,13 +258,21 @@ func main() {
 		return s.Heads, err
 	}
 	r.Trunks = trunks
-	trying := Trying{
-		Tries: stateStore{StateRead: read(tries), create: func(key string, value []byte) error {
-			_, err := tries.Create(key, json.RawMessage(value))
-			return err
-		}},
-		Trunks: trunks,
-	}
+	store := stateStore{StateRead: read(tries), create: func(key string, value []byte) error {
+		_, err := tries.Create(key, json.RawMessage(value))
+		return err
+	}, entry: func(key string) (json.RawMessage, uint64, error) {
+		reads.Add(1)
+		e, err := tries.Get(key)
+		if err != nil || e == nil {
+			return nil, 0, err
+		}
+		return e.Value, e.Revision, nil
+	}, update: func(key string, value []byte, revision uint64) error {
+		_, err := tries.Update(key, json.RawMessage(value), revision)
+		return err
+	}}
+	trying := Trying{Tries: store, Ender: store, Trunks: trunks}
 	if err := stdio.Serve("", logged(append(Tools(r), TryTools(trying)...))); err != nil {
 		fmt.Fprintf(os.Stderr, "[mesh-delivery-checker-tools] %v\n", err)
 		os.Exit(1)

modules/mesh-delivery-checker/cmd/mesh-delivery-checker-tools/main_test.go

@@ -53,13 +53,19 @@ func (m *memStore) Create(key string, value []byte) error {
 	return nil
 }
 
+func (m *memStore) Entry(key string) (json.RawMessage, uint64, error) { return m.vals[key], 1, nil }
+func (m *memStore) Update(key string, value []byte, revision uint64) error {
+	m.vals[key] = value
+	return nil
+}
+
 func TestTheTryToolsAskAndReadATry(t *testing.T) { // ADR 0501 §5, the agreed interface with mesh-controller's try
 	store := &memStore{vals: map[string]json.RawMessage{}}
-	tools := TryTools(Trying{Tries: store, Now: func() time.Time { return time.Unix(1, 5) },
+	tools := TryTools(Trying{Tries: store, Ender: store, Now: func() time.Time { return time.Unix(1, 5) },
 		Trunks: func() (map[string]string, error) {
 			return map[string]string{"novox/mesh-delivery": strings.Repeat("a", 40)}, nil
 		}})
-	if len(tools) != 2 || tools[0].Name != "checker_try" || tools[1].Name != "checker_try_read" {
+	if len(tools) != 3 || tools[0].Name != "checker_try" || tools[1].Name != "checker_try_read" || tools[2].Name != "checker_try_end" {
 		t.Fatalf("tools = %+v", tools)
 	}
 	out, err := tools[0].Run(map[string]any{"repository": "novox/mesh-delivery", "head": strings.Repeat("1", 40), "trunk": "main"})
@@ -74,6 +80,14 @@ func TestTheTryToolsAskAndReadATry(t *testing.T) { // ADR 0501 §5, the agreed i
 	if err != nil || !strings.Contains(string(raw.(json.RawMessage)), `"state":"queued"`) {
 		t.Errorf("read = %s, %v", raw, err)
 	}
+	// Issue 734: a try whose ask was lost is ended by hand with why, and its head is asked anew.
+	ended, err := tools[2].Run(map[string]any{"id": id, "why": "its ask was lost"})
+	if err != nil || ended.(map[string]any)["state"] != "error" {
+		t.Errorf("end = %v, %v", ended, err)
+	}
+	if _, err := tools[2].Run(map[string]any{"id": id, "why": "again"}); err == nil {
+		t.Error("an ended try was ended again")
+	}
 	if _, err := tools[0].Run(map[string]any{"repository": "novox/hq", "head": strings.Repeat("1", 40)}); err == nil ||
 		err.Error() != "no level of novox/hq is checked by the checker" {
 		t.Errorf("hq: %v", err)

modules/mesh-delivery-checker/cmd/mesh-delivery-checker/integration_test.go

@@ -79,6 +79,8 @@ type fakeBuildModules struct {
 	js       jetstream.JetStream
 	ctx      context.Context
 	conflict atomic.Bool
+	// hold is a commit whose fetch never answers: its ask stays taken, as one lost in a bus reload (issue 734).
+	hold     atomic.Value
 	mu       sync.Mutex
 	calls    map[string]int // by `<module>.<tool>`
 	testTool *testing.T
@@ -98,6 +100,9 @@ func (f *fakeBuildModules) result(t *testing.T, tool string, args map[string]any
 	var v any
 	switch {
 	case strings.HasSuffix(tool, "_fetch"):
+		if h, _ := f.hold.Load().(string); h != "" && args["commit"] == h {
+			<-f.ctx.Done()
+		}
 		name := "go-fetch-clean"
 		if f.conflict.Load() {
 			name = "go-fetch-conflict"
@@ -298,6 +303,10 @@ func startBed(t *testing.T) *bed {
 		}
 		b.tools[module] = kv
 	}
+	// What the controller keeps of the messages a consumer gave up on (issue 734).
+	if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: check.DeadLettersStream, Subjects: []string{"mesh.events.dead.>"}}); err != nil {
+		t.Fatal(err)
+	}
 	// The checker's stream of events, which the indexer and the forge bridge hear.
 	if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: "CHECKS", Subjects: []string{"mesh.stream.mesh-delivery-checker.event.>"}}); err != nil {
 		t.Fatal(err)
@@ -716,6 +725,49 @@ func TestATryIsJudgedAndItsEndWrittenWithNoStatusOwed(t *testing.T) {
 			t.Errorf("a try wrote %s: %v", name, keys)
 		}
 	}
+	// Issue 734: a try whose ask the build seat gives up on (the controller keeps its copy in DEAD_LETTERS with the ask's
+	// message id) ends as error saying so, and its head asked again is a new try.
+	b.fake.hold.Store(head2)
+	lost, err := check.AskTry(store, map[string]string{repo: trunk}, check.TryAsk{Repository: repo, Head: head2}, time.Now())
+	if err != nil {
+		t.Fatal(err)
+	}
+	steps, err := b.js.KeyValue(b.ctx, delivery.StepsBucket("mesh-build-go"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	askID := ""
+	for deadline := time.Now().Add(20 * time.Second); askID == "" && time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) {
+		keys, _ := steps.Keys(b.ctx)
+		for _, k := range keys {
+			if e, err := steps.Get(b.ctx, k); err == nil && strings.Contains(string(e.Value()), `"running"`) {
+				askID = strings.TrimPrefix(k, "n1.")
+			}
+		}
+	}
+	if askID == "" {
+		t.Fatal("the held try's ask was never taken")
+	}
+	dead := &nats.Msg{Subject: "mesh.events.dead.SEAT_NODE_BUILD_GO.SEAT_NODE_BUILD_GO_CHECK_worker", Header: nats.Header{}}
+	dead.Header.Set(check.DeadHeaderMsgID, askID)
+	dead.Header.Set(check.DeadHeaderDeliveries, "5")
+	dead.Header.Set(check.DeadHeaderStream, "SEAT_NODE_BUILD_GO")
+	dead.Header.Set(check.DeadHeaderGaveUp, time.Now().UTC().Format(time.RFC3339Nano))
+	if _, err := b.js.PublishMsg(b.ctx, dead); err != nil {
+		t.Fatal(err)
+	}
+	for deadline := time.Now().Add(20 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) {
+		raw, _ := store.Get(lost)
+		if last, err = check.DecodeTry(raw); err == nil && last.Over() {
+			break
+		}
+	}
+	if last.State != check.TryError || !strings.Contains(last.Said, check.LostWords+" after 5 deliveries") {
+		t.Fatalf("the try whose ask was given up = %+v, want error saying so", last)
+	}
+	if again, err := check.AskTry(store, map[string]string{repo: trunk}, check.TryAsk{Repository: repo, Head: head2}, time.Now()); err != nil || again == lost {
+		t.Errorf("its head asked again answered %q, %v, want a new try", again, err)
+	}
 }
 
 // TestTheCheckerSaysOwedReadableOnTheBus: started, the checker says on its own states subject what it keeps, owed among

modules/mesh-delivery-checker/cmd/mesh-delivery-checker/main.go

@@ -198,6 +198,10 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
 	// A check is one ask per pull request and build module on the build seats' check stream, pulled by a build node with a
 	// free slot (novox/hq ADR 0525 §1, §3): the checker chooses no node and starts no step.
 	asks := buildrun.JetStreamAsks{JS: js}
+	// The asks the build seats gave up on (issue 734): heard from DEAD_LETTERS, each ends the step its asker waits for.
+	dead := &check.DeadLetters{}
+	defer hearDeadLetters(ctx, js, dead, logger.Printf)()
+	waiter := check.LostAskWaiter{Inner: asks, Dead: dead}
 	ch := &check.Checker{
 		Verdicts: kvStore{buckets[delivery.CheckerVerdictsBucket]},
 		Owed:     kvStore{buckets[delivery.CheckerOwedBucket]},
@@ -231,7 +235,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
 		return cw, nil
 	}}
 	// The tries (ADR 0501 §5) are judged by the same judge, held by the same deliveries.
-	tryer := &check.Tryer{Tries: kvStore{buckets[check.TriesBucket]}, Bound: checkBound, Logf: logger.Printf}
+	tryer := &check.Tryer{Tries: kvStore{buckets[check.TriesBucket]}, Bound: checkBound, NotTaken: stepBound, Logf: logger.Printf}
 	announce := func(ctx context.Context, event, key string) {
 		body, _ := json.Marshal(map[string]string{"key": key})
 		if _, err := js.Publish(ctx, eventSubject(event), body); err != nil {
@@ -241,7 +245,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
 	repos := &repoModules{manifests: kvStore{buckets[delivery.ManifestsBucket]}, logf: logger.Printf}
 	ch.Runs = repos // the node-engine a check reads beside at its running commit (ADR 0548 §6)
 	ch.Judge = newJudge(judgeDeps{
-		asks: buildrun.Asker{Publisher: asks, Waiter: asks, Bound: stepBound}, tools: tools, trunks: trunks,
+		asks: buildrun.Asker{Publisher: asks, Waiter: waiter, Bound: stepBound}, tools: tools, trunks: trunks,
 		kept: kvStore{buckets[delivery.CheckerKeptBucket]}, previews: kvStore{buckets[delivery.PreviewsBucket]},
 		repos: repos, builds: builds, forge: forgeReads{call: caller.Call}, logf: logger.Printf,
 		announce: announce, compositions: compositions,
@@ -397,3 +401,43 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
 		return err
 	}
 }
+
+// hearDeadLetters follows, for each build seat's stream, the copies DEAD_LETTERS keeps of the asks its workers gave up on
+// (issue 734): an ordered consumer from this start, which resumes where it was after a reconnect, so a bus reload loses none. Its
+// grant is a user's of a seat pulled by kind (mesh-sdk buscompose DeadLetterReads); where the bus refuses it, the checker
+// says so and goes on: a try still ends when no node took its ask within the step bound, and a check waits as before.
+func hearDeadLetters(ctx context.Context, js jetstream.JetStream, dead *check.DeadLetters, logf func(string, ...any)) func() {
+	var stops []func()
+	// From this start on: a step this process waits for was asked since, and a reset of the consumer before it heard any
+	// copy starts here again rather than at the reset.
+	since := time.Now()
+	for _, stream := range check.BuildSeatStreams {
+		filter := check.DeadLetterFilter(stream)
+		notHeard := func(err error) {
+			logf("the asks %s gives up on are not heard (%s of %s), so a check whose ask it gives up waits until the checker starts again: %v",
+				stream, filter, check.DeadLettersStream, err)
+		}
+		cons, err := js.OrderedConsumer(ctx, check.DeadLettersStream, jetstream.OrderedConsumerConfig{
+			FilterSubjects: []string{filter}, DeliverPolicy: jetstream.DeliverByStartTimePolicy, OptStartTime: &since})
+		if err != nil {
+			notHeard(err)
+			continue
+		}
+		cc, err := cons.Consume(func(m jetstream.Msg) {
+			if g, ok := check.GivenUpOf(m.Headers()); ok {
+				logf("the ask %s on %s was given up after %d deliveries at %s: the step that waits for it ends", g.AskID, g.Stream, g.Deliveries, g.At.Format(time.RFC3339))
+				dead.Heard(g)
+			}
+		}, jetstream.ConsumeErrHandler(func(_ jetstream.ConsumeContext, err error) { logf("hearing %s: %v", check.DeadLettersStream, err) }))
+		if err != nil {
+			notHeard(err)
+			continue
+		}
+		stops = append(stops, cc.Stop)
+	}
+	return func() {
+		for _, stop := range stops {
+			stop()
+		}
+	}
+}