Checker, builder and grants part: hear DEAD_LETTERS again until heard, say when they do not, and compose the same grant (issue 734, review round 1)

mesh-grants moves to the same mesh-sdk (3203358c), so its shadow user list grants the checker and the builder the
DEAD_LETTERS reads the controller's list will, and the take-over is not blocked by a difference. The checker and the
builder hear DEAD_LETTERS through buildrun.DeadLetterHearing: in the background, again at every reconnect and every ask
waited for while deaf, and keep whether they hear it in their new state hearing, which checker_status and builder_stuck
say as dead_letters. checker_try says that a try ended by the not-taken bound leaves its ask in the stream.

Not done as asked: a module health tool. The catalogue refuses a tool check on a module with no runtime of its own and no
other check beside it (ADR 0227 rule 8, ADR 0366 §1), so the deaf state is said in the status tools and the log.

modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/main.go

@@ -15,6 +15,7 @@ import (
 
 	stdio "git.novox.be/novox/mesh-sdk/go"
 	"git.novox.be/novox/mesh-sdk/go/delivery"
+	"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
 
 	"mesh-delivery-builder/internal/build"
 )
@@ -31,7 +32,8 @@ func Tools(r build.StuckReader, askAgain func(key string, value []byte) error, q
 			Description: "Every build key that waits (building or queued), oldest first: its module, fingerprint, step words and since when, and each ask it " +
 				"waits on by language with its id, when it was put, its asking (1 for the first; the builder asks an ask again when it can no longer be " +
 				"answered, at most 3 times) and what its build module's steps hold of it (none, or its state on its node since its start). The " +
-				"seat workers' delivery counts are not readable by the builder. With module, that module's keys only. Read-only.",
+				"seat workers' delivery counts are not readable by the builder. dead_letters says whether the builder hears the asks the build seats give up " +
+				"on (issue 734), and why not: heard, a given-up ask is asked again at once; deaf, at its check bound. With module, that module's keys only. Read-only.",
 			Input: map[string]any{"type": "object", "properties": map[string]any{
 				"module": map[string]any{"type": "string", "description": "only this module's keys"}}},
 			Run: func(args map[string]any) (any, error) { return r.Stuck(str(args, "module")) }},
@@ -166,7 +168,8 @@ func main() {
 		steps[bm] = read(stdio.State(bm + ".steps"))
 	}
 	reasks := stdio.State(build.ReAsksState)
-	r := build.StuckReader{Logf: func(f string, a ...any) { fmt.Fprintf(os.Stderr, "[mesh-delivery-builder-tools] "+f+"\n", a...) }, Builds: read(stdio.State("builds")), ReAsks: read(reasks), Steps: steps}
+	r := build.StuckReader{Logf: func(f string, a ...any) { fmt.Fprintf(os.Stderr, "[mesh-delivery-builder-tools] "+f+"\n", a...) }, Builds: read(stdio.State("builds")), ReAsks: read(reasks), Steps: steps,
+		Hearing: read(stdio.State(buildrun.HearingState))}
 	put := func(key string, value []byte) error {
 		_, err := reasks.Put(key, json.RawMessage(value))
 		return err

modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go

@@ -151,8 +151,14 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
 	asks := buildrun.JetStreamAsks{JS: js}
 	// The asks the build seats gave up on (issue 734): heard from DEAD_LETTERS, each ends the wait for its step at once,
 	// and the lane asks its part again (Builder.waitOnce) instead of waiting for the check bound.
+	// Heard in the background, and again at a reconnect or an ask while it is not; whether it is heard is kept in the
+	// state hearing, which builder_stuck says (dead_letters).
 	dead := &buildrun.DeadLetters{}
-	defer buildrun.HearDeadLetters(ctx, js, delivery.AskLanguages, dead, logger.Printf)()
+	hearingKV := process.OptionalBucket(ctx, js, delivery.BuilderModule+"_"+buildrun.HearingState, "builder_stuck cannot say whether DEAD_LETTERS is heard", logger.Printf)
+	hearing := buildrun.NewDeadLetterHearing(ctx, buildrun.JetStreamDeadLetters{JS: js}, delivery.AskLanguages, dead, logger.Printf,
+		buildrun.SayHearing(ctx, hearingKV, logger.Printf))
+	defer hearing.Stop()
+	conn.SetReconnectHandler(func(*nats.Conn) { hearing.Again() })
 	// The builder asks the controller, as its own account, whether the artifact store still holds a done key's blobs
 	// (the seat verb held, novox/hq issue 1937) before it leaves the key done.
 	account, err := process.ReadAccount(path)
@@ -166,7 +172,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
 	defer calls.Close()
 	b := &build.Builder{
 		Claimer: build.Claimer{Store: kvStore{builds}, By: cred.Node + "." + cred.Module},
-		Asks:    buildrun.Asker{Publisher: asks, Waiter: buildrun.LostAskWaiter{Inner: asks, Dead: dead}, Bound: bound},
+		Asks:    buildrun.Asker{Publisher: asks, Waiter: buildrun.LostAskWaiter{Inner: asks, Dead: dead, Hearing: hearing}, Bound: bound},
 		Held:    heldOnTheBus{js},
 		// An ask that can no longer be answered is asked again (issue 1945), read at the step bound: a context's bound.
 		Gone:       heldOnTheBus{js},

modules/mesh-delivery-builder/go.mod

@@ -3,7 +3,7 @@ module mesh-delivery-builder
 go 1.26.0
 
 require (
-	git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026130046-d96b9fd66069
+	git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026131841-3203358c4cb5
 	github.com/nats-io/nats-server/v2 v2.11.17
 	github.com/nats-io/nats.go v1.54.0
 )

modules/mesh-delivery-builder/go.sum

@@ -1,5 +1,5 @@
-git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026130046-d96b9fd66069 h1:dPwZwGbW8LuT4p2baXuiyVNOWU6XO46kbw46VU5TAUQ=
-git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026130046-d96b9fd66069/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
+git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026131841-3203358c4cb5 h1:sBSS509nZDRczmKb/IcHoLtFKJR36eZdQoTMK9tScsU=
+git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026131841-3203358c4cb5/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
 github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
 github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
 github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=

modules/mesh-delivery-builder/internal/build/run_test.go

@@ -849,12 +849,17 @@ func TestABuildAskWaitingInItsStreamIsNotBounded(t *testing.T) {
 		running, _ := json.Marshal(delivery.Step{V: delivery.Version, ID: "mvo53jcm70b5f8", Tool: "delivery ask", Node: "g14", State: delivery.StepRunning, Started: at})
 		_, reask, _ := NewReAsk("m", fp1, "stuck", "shanks.lead", at)
 		r := StuckReader{Builds: state(map[string]string{key: string(building), done: string(finished)}), ReAsks: state(map[string]string{key: string(reask)}),
-			Steps: map[string]StateRead{buildrun.GoBuildModule: state(map[string]string{}), ImageBuildModule: state(map[string]string{"g14.mvo53jcm70b5f8": string(running)})},
-			Now:   func() time.Time { return at.Add(time.Hour) }}
+			Steps:   map[string]StateRead{buildrun.GoBuildModule: state(map[string]string{}), ImageBuildModule: state(map[string]string{"g14.mvo53jcm70b5f8": string(running)})},
+			Hearing: state(map[string]string{buildrun.HearingKey: `{"heard":false,"streams":["SEAT_NODE_BUILD_GO"],"why":"DEAD_LETTERS is not heard: SEAT_NODE_BUILD_GO: permissions violation","at":"2026-10-26T13:00:00Z"}`}),
+			Now:     func() time.Time { return at.Add(time.Hour) }}
 		out, err := r.Stuck("")
 		if err != nil {
 			t.Fatal(err)
 		}
+		// Issue 734, review round 1: a builder that does not hear DEAD_LETTERS says so, with why.
+		if dl := out["dead_letters"].(map[string]any); dl["heard"] != false || !strings.Contains(dl["why"].(string), "permissions violation") {
+			t.Errorf("dead_letters = %v, want not heard, saying why", dl)
+		}
 		stuck := out["stuck"].([]Stuck)
 		if len(stuck) != 1 || stuck[0].Key != key || stuck[0].Age != "1h0m0s" || stuck[0].NoneStep != 1 || stuck[0].ReAsk == nil || stuck[0].ReAsk.Why != "stuck" {
 			t.Fatalf("stuck = %+v, want the building key alone, an hour old, one ask with no step, with its ask again", stuck)

modules/mesh-delivery-builder/internal/build/stuck.go

@@ -106,7 +106,9 @@ type StuckReader struct {
 	ReAsks StateRead
 	// Steps are the build modules' steps states, by build module.
 	Steps map[string]StateRead
-	Now   func() time.Time
+	// Hearing is the builder's state hearing: whether it hears DEAD_LETTERS (issue 734); without it none is said.
+	Hearing StateRead
+	Now     func() time.Time
 	// Logf, when set, gets the one line a call logs: the values it read and the time it took.
 	Logf func(format string, args ...any)
 }
@@ -149,6 +151,20 @@ func parallel(n int, read func(i int) error) error {
 }
 
 // Stuck is every key building or queued with its asks, oldest first; with module, that module's only.
+// deadLetters says whether the builder hears the asks the build seats give up on (issue 734), and why not: deaf, a lane
+// whose ask is given up waits for its check bound (buildrun.HearingHealth over the kept hearing).
+func (r StuckReader) deadLetters() map[string]any {
+	if r.Hearing.Get == nil {
+		return map[string]any{"heard": false, "why": "the builder's state hearing is not readable here"}
+	}
+	raw, _, err := r.Hearing.Get(buildrun.HearingKey)
+	if err != nil {
+		return map[string]any{"heard": false, "why": "the builder's state hearing cannot be read: " + err.Error()}
+	}
+	a := buildrun.HearingHealth(raw)
+	return map[string]any{"heard": a.Healthy, "why": a.Why}
+}
+
 func (r StuckReader) Stuck(module string) (map[string]any, error) {
 	started := time.Now()
 	reads := 0
@@ -268,6 +284,7 @@ func (r StuckReader) Stuck(module string) (map[string]any, error) {
 	})
 	answer := map[string]any{"stuck": out, "count": len(out),
 		"delivery_counts": "not readable by the builder: the build seats' workers' information is granted to the delivery view alone (its backlog); an ask given up on is told by the builder at its check bound and asked again"}
+	answer["dead_letters"] = r.deadLetters()
 	if cut {
 		answer["cut"], answer["note"] = true, fmt.Sprintf("read %d of %d build keys in key order and stopped; give a module to read its keys alone", MaxReads, total)
 	}

modules/mesh-delivery-builder/manifest_test.go

@@ -87,7 +87,7 @@ func TestManifestDeclarations(t *testing.T) { // 1607/AC1
 	if m.Module != "mesh-delivery-builder" {
 		t.Errorf("module = %q", m.Module)
 	}
-	if want := []state{{Name: "builds", History: 5}, {Name: "instances", History: 1}, {Name: "queues", History: 1}, {Name: "reasks", History: 1}}; !reflect.DeepEqual(m.State, want) { // delivery.BuilderBuildsHistory
+	if want := []state{{Name: "builds", History: 5}, {Name: "hearing", History: 1}, {Name: "instances", History: 1}, {Name: "queues", History: 1}, {Name: "reasks", History: 1}}; !reflect.DeepEqual(m.State, want) { // delivery.BuilderBuildsHistory
 		t.Errorf("state = %+v, want %+v", m.State, want)
 	}
 	// 1887/A4c: the builder chooses no node, so it reads neither presence nor have, invokes no build module's tool and

modules/mesh-delivery-builder/module.json

@@ -20,6 +20,7 @@
   ],
   "state": [
     { "name": "builds", "history": 5 },
+    { "name": "hearing", "history": 1 },
     { "name": "instances", "history": 1 },
     { "name": "queues", "history": 1 },
     { "name": "reasks", "history": 1 }