Builder: ask again an ask its stream can no longer answer, and give it tools to read and ask again (issue 1945)

A build ask its seat's stream gave up on, dropped or ended with no step was
waited for with no end, and on restart taken over and waited for again: no
build node would ever answer it, and nothing could ask it again (trunk key
of mesh-build-node at 4e7b118e, building since 19:06; seven keys at 18:44).

At the step bound, a context's bound and no timer of its own, the builder
reads what the bus says of an ask with no step: gone from its stream, or
first in line while a node took a later ask. Such an ask is asked again
under a new id the key names, logged and said as build.asked-again, at most
three askings, then the key fails with why. The bus's max-deliveries
advisory would tell it sooner, but the builder is granted neither it nor the
workers' information, and the controller is not changed here.

builder_stuck reads every waiting key with its asks, askings and steps;
builder_ask_again writes the new reasks state, which the builder watches
and answers.

modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/main.go

@@ -0,0 +1,92 @@
+// Command mesh-delivery-builder-tools serves the builder's tools (novox/hq issue 1945): the runtime launches it for a call.
+// builder_stuck reads the builder's builds and reasks states and the build modules' steps through the builder's state and
+// reads; builder_ask_again writes one value of the reasks state, which the builder watches and answers. The builder is the
+// other program: it serves no tool (ADR 0437 §4).
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"os"
+	"strings"
+	"time"
+
+	stdio "git.novox.be/novox/mesh-sdk/go"
+
+	"mesh-delivery-builder/internal/build"
+)
+
+// Tools are the builder's two tools: stuck reads, askAgain writes the reasks value.
+func Tools(r build.StuckReader, askAgain func(key string, value []byte) error, now func() time.Time) []stdio.Tool {
+	str := func(args map[string]any, k string) string { s, _ := args[k].(string); return strings.TrimSpace(s) }
+	return []stdio.Tool{
+		{Name: build.StuckTool,
+			Description: "Every build key that waits (building or queued), oldest first: its module, fingerprint, step words and since when, and each ask it " +
+				"waits on by language with its id, when it was put, its asking (1 for the first; the builder asks an ask again when it can no longer be " +
+				"answered, at most 3 times) and what its build module's steps hold of it (none, or its state on its node since its start). The " +
+				"seat workers' delivery counts are not readable by the builder. With module, that module's keys only. Read-only.",
+			Input: map[string]any{"type": "object", "properties": map[string]any{
+				"module": map[string]any{"type": "string", "description": "only this module's keys"}}},
+			Run: func(args map[string]any) (any, error) { return r.Stuck(str(args, "module")) }},
+		{Name: build.AskAgainTool,
+			Description: "Ask a waiting build key's asks again now, under new ids the key then names, as the builder does by itself for an ask its build " +
+				"seat's stream gave up on. The builder answers in the reasks state, which " + build.StuckTool + " shows beside the key; a key no ask of " +
+				"which waits is answered so and nothing is asked.",
+			Input: map[string]any{"type": "object", "required": []string{"module", "fp", "why"}, "properties": map[string]any{
+				"module": map[string]any{"type": "string", "description": "the module's name"},
+				"fp":     map[string]any{"type": "string", "description": "its new-form fingerprint, src2:<hex>, as builder_stuck shows it"},
+				"why":    map[string]any{"type": "string", "description": "why it is asked again, in plain words"}}},
+			Run: func(args map[string]any) (any, error) {
+				key, value, err := build.NewReAsk(str(args, "module"), str(args, "fp"), str(args, "why"), stdio.CallerOf(args), now())
+				if err != nil {
+					return nil, err
+				}
+				if err := askAgain(key, value); err != nil {
+					return nil, fmt.Errorf("the ask could not be written to %s: %w", build.ReAsksBucket, err)
+				}
+				return map[string]any{"key": key, "asked": true,
+					"next": "the builder answers in " + build.ReAsksState + "." + key + "; " + build.StuckTool + " shows the answer and the new ask"}, nil
+			}},
+	}
+}
+
+func read(s stdio.KeptState) build.StateRead {
+	return build.StateRead{Keys: s.Keys, Get: func(key string) (json.RawMessage, time.Time, error) {
+		e, err := s.Get(key)
+		if err != nil || e == nil {
+			return nil, time.Time{}, err
+		}
+		return e.Value, written(s, key, e.Revision), nil
+	}}
+}
+
+// written is when the key's revision was written, from its history, or zero.
+func written(s stdio.KeptState, key string, revision uint64) time.Time {
+	h, err := s.History(key)
+	if err != nil {
+		return time.Time{}
+	}
+	for _, r := range h {
+		if r.Revision == revision {
+			return r.At
+		}
+	}
+	return time.Time{}
+}
+
+func main() {
+	steps := map[string]build.StateRead{}
+	for _, bm := range build.StuckBuildModules {
+		steps[bm] = read(stdio.State(bm + ".steps"))
+	}
+	reasks := stdio.State(build.ReAsksState)
+	r := build.StuckReader{Builds: read(stdio.State("builds")), ReAsks: read(reasks), Steps: steps}
+	put := func(key string, value []byte) error {
+		_, err := reasks.Put(key, json.RawMessage(value))
+		return err
+	}
+	if err := stdio.Serve("", Tools(r, put, time.Now)); err != nil {
+		fmt.Fprintf(os.Stderr, "[mesh-delivery-builder-tools] %v\n", err)
+		os.Exit(1)
+	}
+}

modules/mesh-delivery-builder/cmd/mesh-delivery-builder/held.go

@@ -8,57 +8,82 @@ import (
 	"github.com/nats-io/nats.go/jetstream"
 
 	"git.novox.be/novox/mesh-sdk/go/delivery"
+
+	"mesh-delivery-builder/internal/build"
 )
 
 // heldOnTheBus says whether an ask a build key names is on its build seat's stream, or a build node wrote a step for it
-// (issue 1910): read once, at the take-over of a key at start, never polled.
+// (issue 1910), read once at the take-over of a key at start; and, at the builder's check bound, whether an ask can still
+// be answered (issue 1945). Never polled.
 type heldOnTheBus struct{ js jetstream.JetStream }
 
 // Held is build.Builder's Held: a step key `<node>.<ask id>` in the build module's steps, or a message of the stream under
 // the ask's subject whose Nats-Msg-Id is the ask's id.
 func (h heldOnTheBus) Held(ctx context.Context, a delivery.StepAsk) (bool, error) {
-	module, err := delivery.BuildModuleOf(a.Language)
+	seen, err := h.seen(ctx, a, false)
 	if err != nil {
 		return false, err
 	}
+	return seen.Step || seen.OnStream, nil
+}
+
+// Gone is build.Builder's Gone: why the ask can no longer be answered (build.AskSeen.Why), or "".
+func (h heldOnTheBus) Gone(ctx context.Context, a delivery.StepAsk) (string, error) {
+	seen, err := h.seen(ctx, a, true)
+	if err != nil {
+		return "", err
+	}
+	return seen.Why(), nil
+}
+
+// seen reads what the bus holds of the ask; with later, a step of its build module started after the ask was put too,
+// when the ask waits first in line.
+func (h heldOnTheBus) seen(ctx context.Context, a delivery.StepAsk, later bool) (build.AskSeen, error) {
+	var seen build.AskSeen
+	module, err := delivery.BuildModuleOf(a.Language)
+	if err != nil {
+		return seen, err
+	}
 	pattern, err := delivery.AskStepWatch(a.ID)
 	if err != nil {
-		return false, err
+		return seen, err
 	}
 	steps, err := h.js.KeyValue(ctx, delivery.StepsBucket(module))
 	if err != nil {
-		return false, fmt.Errorf("the bucket %s: %w", delivery.StepsBucket(module), err)
+		return seen, fmt.Errorf("the bucket %s: %w", delivery.StepsBucket(module), err)
 	}
 	keys, err := steps.ListKeysFiltered(ctx, pattern)
 	if err != nil {
-		return false, err
+		return seen, err
 	}
 	for range keys.Keys() {
 		keys.Stop()
-		return true, nil // a node took it
+		seen.Step = true // a node took it
+		return seen, nil
 	}
 	subject, err := delivery.AskSubject(a.Language, a.Kind)
 	if err != nil {
-		return false, err
+		return seen, err
 	}
 	// The seat's stream by its name (delivery.AskStream), as the broker grants the builder its information and its
 	// messages by sequence and nothing wider (issue 1910).
 	name, err := delivery.AskStream(a.Language)
 	if err != nil {
-		return false, err
+		return seen, err
 	}
+	seen.Stream = name
 	stream, err := h.js.Stream(ctx, name)
 	if errors.Is(err, jetstream.ErrStreamNotFound) {
-		return false, nil
+		return seen, nil
 	}
 	if err != nil {
-		return false, err
+		return seen, err
 	}
 	// A seat's stream is a work queue, which takes no second consumer over its subjects: its messages are read by sequence
 	// instead, each a direct read that consumes nothing.
 	info, err := stream.Info(ctx)
 	if err != nil {
-		return false, err
+		return seen, err
 	}
 	for seq := info.State.FirstSeq; seq != 0 && seq <= info.State.LastSeq; seq++ {
 		m, err := stream.GetMsg(ctx, seq)
@@ -66,11 +91,37 @@ func (h heldOnTheBus) Held(ctx context.Context, a delivery.StepAsk) (bool, error
 			continue // taken and acknowledged, or removed
 		}
 		if err != nil {
-			return false, err
+			return seen, err
 		}
-		if m.Subject == subject && m.Header.Get(jetstream.MsgIDHeader) == a.ID {
-			return true, nil
+		if m.Subject != subject {
+			continue
 		}
+		if m.Header.Get(jetstream.MsgIDHeader) == a.ID {
+			seen.OnStream, seen.PutAt = true, m.Time
+			break
+		}
+		seen.Ahead++
 	}
-	return false, nil
+	if !later || !seen.OnStream || seen.Ahead > 0 {
+		return seen, nil
+	}
+	// First in line and not taken: did a build node take any ask put after it? Its stream then no longer delivers it.
+	all, err := steps.ListKeys(ctx)
+	if err != nil {
+		return seen, err
+	}
+	defer all.Stop()
+	for k := range all.Keys() {
+		e, err := steps.Get(ctx, k)
+		if err != nil {
+			continue
+		}
+		s, err := delivery.DecodeStep(e.Value())
+		if err != nil || !s.Started.After(seen.PutAt) {
+			continue // not a step (a node's pause key), or one started before the ask was put
+		}
+		seen.Later = &s
+		break
+	}
+	return seen, nil
 }

modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go

@@ -169,9 +169,12 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
 	}
 	defer calls.Close()
 	b := &build.Builder{
-		Claimer:    build.Claimer{Store: kvStore{builds}, By: cred.Node + "." + cred.Module},
-		Asks:       buildrun.Asker{Publisher: asks, Waiter: asks, Bound: bound},
-		Held:       heldOnTheBus{js},
+		Claimer: build.Claimer{Store: kvStore{builds}, By: cred.Node + "." + cred.Module},
+		Asks:    buildrun.Asker{Publisher: asks, Waiter: asks, Bound: bound},
+		Held:    heldOnTheBus{js},
+		// An ask that can no longer be answered is asked again (issue 1945), read at the step bound: a context's bound.
+		Gone:       heldOnTheBus{js},
+		CheckAfter: bound,
 		Toolchains: tools,
 		Manifests:  manifestReader{manifests},
 		Logf:       logger.Printf,
@@ -198,6 +201,11 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
 		}
 		b.Trunk = trunks.head
 	}
+	b.Said = func(e build.AskedAgain) {
+		if err := emitAskedAgain(context.WithoutCancel(ctx), js, cred, e); err != nil {
+			logger.Printf("%v", err)
+		}
+	}
 	// **Said once every build module answered a reference, taken away when stopped** (issue 1886, ADR 0522): the
 	// controller delivers from this builder's keys only while an instance says it writes references. A bucket not there
 	// yet (the manifest's state not made) is said, and the controller's plan waits for the keys, named (no build-agent
@@ -290,8 +298,23 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
 	if err != nil {
 		return err
 	}
+	// A person asks a key's build again with builder_ask_again, which writes the reasks state (issue 1945): watched, as
+	// every other input. A bucket not there yet (the manifest's state not made) is said.
+	errs := make(chan error, 4)
+	switch kv, err := js.KeyValue(ctx, build.ReAsksBucket); {
+	case errors.Is(err, jetstream.ErrBucketNotFound):
+		logger.Printf("the bucket %s does not exist yet: builder_ask_again is not heard until the builder starts again", build.ReAsksBucket)
+	case err != nil:
+		return fmt.Errorf("the bucket %s: %w", build.ReAsksBucket, err)
+	default:
+		eachReAsk := func(e jetstream.KeyValueEntry) { reAsked(ctx, b, kv, e, logger.Printf) }
+		rw, err := settle(ctx, kv, eachReAsk)
+		if err != nil {
+			return err
+		}
+		go func() { errs <- follow(ctx, rw, eachReAsk) }()
+	}
 	logger.Printf("started on %s", cred.Node)
-	errs := make(chan error, 3)
 	go func() { errs <- follow(ctx, w, each) }()
 	go func() { errs <- follow(ctx, cw, eachCandidate) }()
 	if sw != nil {

modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reasks.go

@@ -0,0 +1,75 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"strings"
+	"time"
+
+	"github.com/nats-io/nats.go"
+	"github.com/nats-io/nats.go/jetstream"
+
+	"mesh-delivery-builder/internal/build"
+)
+
+// askAgainer is the part of the builder a reasks value asks through.
+type askAgainer interface {
+	AskAgain(key, why string) []string
+}
+
+// reAsked answers one value of the reasks state (the tool builder_ask_again, issue 1945): a value not yet answered asks the
+// key's waiting lanes again and is written back, by compare-and-set at its revision, with what the builder did. A value
+// answered already (this builder's own write) or deleted is left.
+func reAsked(ctx context.Context, b askAgainer, kv jetstream.KeyValue, e jetstream.KeyValueEntry, logf func(string, ...any)) {
+	if e.Operation() != jetstream.KeyValuePut {
+		return
+	}
+	r, err := build.DecodeReAsk(e.Value())
+	if err != nil {
+		logf("%s.%s holds what is not an ask to ask again, left: %v", build.ReAsksState, e.Key(), err)
+		return
+	}
+	if r.Answer != "" {
+		return
+	}
+	asked := b.AskAgain(e.Key(), r.Why)
+	if len(asked) == 0 {
+		r.Answer = "no ask of this key waits in this builder now (it is not building, or its build has not reached its ask yet): nothing was asked; builder_stuck shows where it stands"
+	} else {
+		r.Answer = "asked again: the " + strings.Join(asked, ", ") + " ask, under a new id the key names"
+	}
+	r.Answered = time.Now().UTC()
+	logf("%s: asked again by %s (%s): %s", e.Key(), r.By, r.Why, r.Answer)
+	data, err := json.Marshal(r)
+	if err != nil {
+		return
+	}
+	if _, err := kv.Update(ctx, e.Key(), data, e.Revision()); err != nil && !errors.Is(err, jetstream.ErrKeyExists) && !errors.Is(err, jetstream.ErrKeyRevisionMismatch) {
+		logf("%s.%s: the answer could not be written: %v", build.ReAsksState, e.Key(), err)
+	}
+}
+
+// emitAskedAgain says the builder's event build.asked-again (issue 1945): mesh.mod.<module>.event.build.asked-again,
+// with the SDK's envelope headers and the new ask's id as the bus's dedup id.
+func emitAskedAgain(ctx context.Context, js jetstream.JetStream, cred credential, e build.AskedAgain) error {
+	data, err := json.Marshal(e)
+	if err != nil {
+		return err
+	}
+	msg := nats.NewMsg("mesh.mod." + cred.Module + ".event." + build.AskedAgainEvent)
+	id := "asked-again-" + e.New
+	msg.Header.Set("x-event-id", id)
+	msg.Header.Set("x-source", cred.Module)
+	msg.Header.Set("x-node", cred.Node)
+	msg.Header.Set("x-time", time.Now().UTC().Format(time.RFC3339Nano))
+	msg.Header.Set("content-type", "application/json")
+	msg.Data = data
+	pub, cancel := context.WithTimeout(ctx, 10*time.Second)
+	defer cancel()
+	if _, err := js.PublishMsg(pub, msg, jetstream.WithMsgID(id)); err != nil {
+		return fmt.Errorf("saying %s for %s: %w", build.AskedAgainEvent, e.Key, err)
+	}
+	return nil
+}

modules/mesh-delivery-builder/internal/build/askagain.go

@@ -0,0 +1,235 @@
+package build
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"regexp"
+	"strconv"
+	"strings"
+	"time"
+
+	"git.novox.be/novox/mesh-sdk/go/buildstep"
+	"git.novox.be/novox/mesh-sdk/go/delivery"
+)
+
+// **An ask that can no longer be answered is asked again** (novox/hq issue 1945). A build seat's stream gives an ask up
+// once it was delivered as often as its worker may (delivery.AskMaxDeliver), and a message given up on, dropped or
+// terminated with no step written is answered by no build node: the lane waiting for its step would wait for ever, and on
+// restart take over the same id and wait again. The builder notices it at the check bound (CheckAfter, the step bound),
+// a context's bound and no timer of its own: it asks Gone, which reads the bus, and asks such an ask again under a new
+// id, written into the key, said in its log and as the event build.asked-again, at most Askings times; then the key is
+// failed with why. A person asks the same by hand with the tool builder_ask_again (AskAgain).
+
+// DefaultAskings is how many times a build module's part of a build is asked, the first included, when Builder.Askings is 0.
+const DefaultAskings = 3
+
+// AskedAgainEvent is the event the builder says when it asked an ask again: mesh.mod.mesh-delivery-builder.event.<this>.
+const AskedAgainEvent = "build.asked-again"
+
+// AskedAgain is the body of AskedAgainEvent.
+type AskedAgain struct {
+	Key         string    `json:"key"`
+	Module      string    `json:"module"`
+	FP          string    `json:"fp"`
+	BuildModule string    `json:"build_module"`
+	Language    string    `json:"language"`
+	Old         string    `json:"old"`
+	New         string    `json:"new"`
+	Asking      int       `json:"asking"`
+	Askings     int       `json:"askings"`
+	Why         string    `json:"why"`
+	At          time.Time `json:"at"`
+}
+
+// AskSeen is what the bus says of an ask whose step has not ended, as the builder may read it: its build seat's stream
+// (information and messages by sequence) and its build module's steps. The builder may not read the seat workers'
+// delivery counts (granted to the delivery view alone), so a message given up on is told by what the workers did after it.
+type AskSeen struct {
+	Stream string
+	// Step is whether a build node wrote a step for the ask (running: the step bound of buildrun.Asker judges it).
+	Step bool
+	// OnStream is whether the stream still holds the ask's message; PutAt is when it was put, Ahead how many messages of
+	// its subject wait before it.
+	OnStream bool
+	PutAt    time.Time
+	Ahead    int
+	// Later is a step of the same build module that started after the ask was put, when one did.
+	Later *delivery.Step
+}
+
+// Why is why the ask can no longer be answered, or "" when it can: it has a step, waits behind others of its kind, or no
+// build node took any ask since it was put (the nodes are busy or paused: a backlog, never a fault, ADR 0525 §9).
+func (s AskSeen) Why() string {
+	switch {
+	case s.Step:
+		return ""
+	case !s.OnStream:
+		return fmt.Sprintf("no build node wrote a step for it and %s no longer holds it: it was given up on or dropped", s.Stream)
+	case s.Ahead > 0 || s.Later == nil:
+		return ""
+	default:
+		return fmt.Sprintf("it waits first in line on %s since %s while %s took a later ask at %s: its stream no longer delivers it (given up after its most deliveries)",
+			s.Stream, s.PutAt.UTC().Format(time.RFC3339), s.Later.Node, s.Later.Started.UTC().Format(time.RFC3339))
+	}
+}
+
+func (b *Builder) askings() int {
+	if b.Askings > 0 {
+		return b.Askings
+	}
+	return DefaultAskings
+}
+
+var askingWords = regexp.MustCompile(`asked (\S+) again \(asking (\d+)\)`)
+
+// askingOf is the asking a key's step words name for a build module's ask, 1 when none.
+func askingOf(step, buildModule string) int {
+	for _, m := range askingWords.FindAllStringSubmatch(step, -1) {
+		if m[1] == buildModule {
+			if n, err := strconv.Atoi(m[2]); err == nil && n > 0 {
+				return n
+			}
+		}
+	}
+	return 1
+}
+
+// AskAgain asks the waiting asks of a build key again now, under new ids (the tool builder_ask_again): it answers the
+// languages whose lane waits, none when no lane of this builder waits on the key.
+func (b *Builder) AskAgain(key, why string) []string {
+	b.init()
+	b.mu.Lock()
+	defer b.mu.Unlock()
+	var asked []string
+	for language, c := range b.waiting[key] {
+		select {
+		case c <- why:
+		default: // one is already asked
+		}
+		asked = append(asked, language)
+	}
+	return asked
+}
+
+// Waiting are the build keys whose lanes wait for a step now, each with its languages.
+func (b *Builder) Waiting() map[string][]string {
+	b.init()
+	b.mu.Lock()
+	defer b.mu.Unlock()
+	out := map[string][]string{}
+	for key, ls := range b.waiting {
+		for l := range ls {
+			out[key] = append(out[key], l)
+		}
+	}
+	return out
+}
+
+func (b *Builder) listen(key, language string) chan string {
+	b.init()
+	c := make(chan string, 1)
+	b.mu.Lock()
+	if b.waiting[key] == nil {
+		b.waiting[key] = map[string]chan string{}
+	}
+	b.waiting[key][language] = c
+	b.mu.Unlock()
+	return c
+}
+
+func (b *Builder) unlisten(key, language string, c chan string) {
+	b.mu.Lock()
+	if b.waiting[key][language] == c {
+		delete(b.waiting[key], language)
+		if len(b.waiting[key]) == 0 {
+			delete(b.waiting, key)
+		}
+	}
+	b.mu.Unlock()
+}
+
+var (
+	errCheck = errors.New("the check bound passed")
+	errAgain = errors.New("asked again")
+)
+
+type byHand struct{ why string }
+
+func (h byHand) Error() string { return "asked again by hand: " + h.why }
+
+// answer waits for the step of the lane's ask, and asks it again while it can no longer be answered (Gone) or a person
+// asked it again (AskAgain); it answers the step and the ask that has it.
+func (b *Builder) answer(ctx context.Context, l *lane, ask delivery.StepAsk, asked *askNotes, module, fp string) (delivery.Step, delivery.StepAsk, error) {
+	key, _ := delivery.BuilderBuildKey(module, fp)
+	again := b.listen(key, ask.Language)
+	defer b.unlisten(key, ask.Language, again)
+	asking := askingOf(asked.step, l.module)
+	for {
+		step, why, err := b.waitOnce(ctx, ask, again)
+		switch {
+		case errors.Is(err, errCheck):
+			continue
+		case errors.Is(err, errAgain):
+			if asking >= b.askings() {
+				return step, ask, fmt.Errorf("the ask %s could not be answered and was asked %d times, the most it is: %s", ask.ID, asking, why)
+			}
+			asking++
+			old := ask.ID
+			ask.ID, ask.At = buildstep.NewID(), time.Now()
+			if err := asked.note(ctx, ask.Language, ask.ID, fmt.Sprintf("asked %s again (asking %d): %s", l.module, asking, why)); err != nil {
+				return step, ask, fmt.Errorf("the ask of %s asked again could not be written to the key: %w", l.module, err)
+			}
+			if err := b.Asks.Put(ctx, ask); err != nil {
+				return step, ask, err
+			}
+			b.logf("%s: the ask %s of %s is asked again as %s (asking %d of %d): %s", key, old, l.module, ask.ID, asking, b.askings(), why)
+			if b.Said != nil {
+				b.Said(AskedAgain{Key: key, Module: module, FP: fp, BuildModule: l.module, Language: ask.Language, Old: old, New: ask.ID,
+					Asking: asking, Askings: b.askings(), Why: why, At: ask.At})
+			}
+		default:
+			return step, ask, err
+		}
+	}
+}
+
+// waitOnce waits for the ask's step until it ends, a person asks it again, or the check bound passes: then it asks Gone
+// and answers errAgain with why when the ask can no longer be answered, errCheck when it waits on.
+func (b *Builder) waitOnce(ctx context.Context, ask delivery.StepAsk, again <-chan string) (delivery.Step, string, error) {
+	hand, cancel := context.WithCancelCause(ctx)
+	defer cancel(nil)
+	go func() {
+		select {
+		case why := <-again:
+			cancel(byHand{why})
+		case <-hand.Done():
+		}
+	}()
+	wait := hand
+	if b.Gone != nil && b.CheckAfter > 0 {
+		var stop context.CancelFunc
+		wait, stop = context.WithTimeoutCause(hand, b.CheckAfter, errCheck)
+		defer stop()
+	}
+	step, err := b.Asks.Wait(wait, ask)
+	if err == nil || ctx.Err() != nil {
+		return step, "", err
+	}
+	var h byHand
+	if errors.As(context.Cause(hand), &h) {
+		return step, strings.TrimSpace("asked again by hand: " + h.why), errAgain
+	}
+	if errors.Is(context.Cause(wait), errCheck) {
+		why, gerr := b.Gone.Gone(ctx, ask)
+		if gerr != nil {
+			b.logf("whether the ask %s can still be answered cannot be read, so it is waited for: %v", ask.ID, gerr)
+			return step, "", errCheck
+		}
+		if why == "" {
+			return step, "", errCheck
+		}
+		return step, why, errAgain
+	}
+	return step, "", err
+}

modules/mesh-delivery-builder/internal/build/run.go

@@ -56,6 +56,18 @@ type Builder struct {
 	// Trunk answers the head of a repository in the gate's source revision rev (source.main at that revision): the trunk a
 	// candidate's preview is merged onto (Candidate.Base). Nil builds no candidate's fingerprint, and says why.
 	Trunk func(ctx context.Context, rev uint64, repository string) (string, error)
+	// Gone says why an ask with no ended step can no longer be answered, or "" when it can (issue 1945): asked once the
+	// ask has waited CheckAfter, and again each CheckAfter it goes on waiting. An ask that cannot be answered is asked
+	// again under a new id, at most Askings times. Nil, or CheckAfter 0, never asks it.
+	Gone interface {
+		Gone(ctx context.Context, a delivery.StepAsk) (string, error)
+	}
+	CheckAfter time.Duration
+	// Askings is how many times one build module's part of a build is asked, the first included, before its key is failed
+	// with why; DefaultAskings when 0.
+	Askings int
+	// Said is told each ask asked again (the builder's event build.asked-again). Nil tells nobody.
+	Said func(AskedAgain)
 
 	once     sync.Once
 	sem      chan struct{}
@@ -75,6 +87,8 @@ type Builder struct {
 	// held are the done keys' references the store answered held, by `<key> <references>`, asked once per process: a
 	// blob collected later is found by the take-in, whose artifacts-gone forgets the key here (issue 1937).
 	held map[string]bool
+	// waiting are the lanes that wait for an ask's step, by build key and language: AskAgain asks through them.
+	waiting map[string]map[string]chan string
 }
 
 type planned struct {
@@ -91,7 +105,7 @@ func (b *Builder) init() {
 		b.sem = make(chan struct{}, limit)
 		b.cands, b.via, b.trunkFP = map[string]delivery.Candidate{}, map[string]via{}, map[string]bool{}
 		b.known, b.inflight, b.again, b.plans, b.tried = map[string]delivery.Module{}, map[string]bool{}, map[string]delivery.Module{}, map[string]planned{}, map[string]bool{}
-		b.published, b.held = map[string]bool{}, map[string]bool{}
+		b.published, b.held, b.waiting = map[string]bool{}, map[string]bool{}, map[string]map[string]chan string{}
 	})
 }
 
@@ -673,12 +687,14 @@ type askNotes struct {
 	b       *Builder
 	cl      *Claim
 	resumed map[string]string
+	// step is the step words of a key taken over at start: an ask asked again says its asking there (askingOf).
+	step string
 }
 
 func (b *Builder) notes(cl *Claim) *askNotes {
 	n := &askNotes{b: b, cl: cl}
 	if cl.Build.State == delivery.BuildBuilding && len(cl.Build.Asks) > 0 {
-		n.resumed = maps.Clone(cl.Build.Asks)
+		n.resumed, n.step = maps.Clone(cl.Build.Asks), cl.Build.Step
 	}
 	return n
 }
@@ -776,7 +792,7 @@ func (b *Builder) runLane(ctx context.Context, l *lane, repository string, m del
 			return
 		}
 	}
-	step, err := b.Asks.Wait(ctx, ask)
+	step, ask, err := b.answer(ctx, l, ask, asked, module, m.FP2)
 	if err != nil {
 		l.err = fmt.Errorf("%s: %w", l.module, err)
 		return

modules/mesh-delivery-builder/internal/build/run_test.go

@@ -616,27 +616,198 @@ func TestAFailedPublishOfACandidatesImageIsWrittenToTheKey(t *testing.T) { // re
 	}
 }
 
-// Review round 2 item 1: a build ask that waits in its stream longer than the step bound is not failed and not asked
-// again; the key is done when a node takes it.
-func TestABuildAskWaitingInItsStreamIsNotBounded(t *testing.T) {
-	r := newRig(t, goBundle)
-	r.asks.Answer = nil
-	r.b.Asks = buildrun.Asker{Publisher: r.asks, Waiter: r.asks, Bound: 20 * time.Millisecond}
-	r.b.Module(context.Background(), "m", moduleValue(t, "modules/m"))
-	for deadline := time.Now().Add(2 * time.Second); len(r.asks.Asks()) == 0; time.Sleep(time.Millisecond) {
+// goneAsks answers Gone from a function of the ask, and counts how often it was asked.
+type goneAsks struct {
+	mu  sync.Mutex
+	n   int
+	why func(a delivery.StepAsk) string
+}
+
+func (g *goneAsks) Gone(_ context.Context, a delivery.StepAsk) (string, error) {
+	g.mu.Lock()
+	g.n++
+	g.mu.Unlock()
+	return g.why(a), nil
+}
+
+func (g *goneAsks) asked() int {
+	g.mu.Lock()
+	defer g.mu.Unlock()
+	return g.n
+}
+
+// waitForAsks waits until n asks were put.
+func waitForAsks(t *testing.T, r *rig, n int) []delivery.StepAsk {
+	t.Helper()
+	for deadline := time.Now().Add(2 * time.Second); len(r.asks.Asks()) < n; time.Sleep(time.Millisecond) {
 		if time.Now().After(deadline) {
-			t.Fatal("no ask was put")
+			t.Fatalf("%d asks put, want %d", len(r.asks.Asks()), n)
 		}
 	}
-	time.Sleep(100 * time.Millisecond)
-	if v := valueOf(t, r.st, r.key); v.State != delivery.BuildBuilding {
-		t.Fatalf("the key = %+v while its ask waits, want building", v)
-	}
-	r.asks.End(r.answer(r.asks.Asks()[0]))
-	r.b.Wait()
-	if v := valueOf(t, r.st, r.key); v.State != delivery.BuildDone || len(r.asks.Asks()) != 1 {
-		t.Fatalf("the key = %+v after %d asks, want done from the one ask", v, len(r.asks.Asks()))
+	return r.asks.Asks()
+}
+
+// Review round 2 item 1: a build ask that waits in its stream longer than the step bound is not failed and not asked
+// again; the key is done when a node takes it. Issue 1945: an ask that can no longer be answered (its stream gave it up)
+// is asked again under a new id, said, at most Askings times; a person asks it again by hand the same way.
+func TestABuildAskWaitingInItsStreamIsNotBounded(t *testing.T) {
+	t.Run("waits in its stream: never asked again", func(t *testing.T) {
+		r := newRig(t, goBundle)
+		r.asks.Answer = nil
+		r.b.Asks = buildrun.Asker{Publisher: r.asks, Waiter: r.asks, Bound: 20 * time.Millisecond}
+		gone := &goneAsks{why: func(delivery.StepAsk) string { return "" }}
+		r.b.Gone, r.b.CheckAfter = gone, 5*time.Millisecond
+		r.b.Module(context.Background(), "m", moduleValue(t, "modules/m"))
+		waitForAsks(t, r, 1)
+		time.Sleep(100 * time.Millisecond)
+		if v := valueOf(t, r.st, r.key); v.State != delivery.BuildBuilding {
+			t.Fatalf("the key = %+v while its ask waits, want building", v)
+		}
+		if gone.asked() == 0 {
+			t.Fatal("whether the ask can still be answered was never read at the check bound") // 1945
+		}
+		r.asks.End(r.answer(r.asks.Asks()[0]))
+		r.b.Wait()
+		if v := valueOf(t, r.st, r.key); v.State != delivery.BuildDone || len(r.asks.Asks()) != 1 {
+			t.Fatalf("the key = %+v after %d asks, want done from the one ask", v, len(r.asks.Asks()))
+		}
+	})
+	for _, c := range []struct {
+		name    string
+		askings int
+		// step is the key's step words at start when it names an ask (a key taken over), "" for a fresh build.
+		step   string
+		byHand bool
+		always bool // Gone says every ask cannot be answered, not only the first
+		puts   int
+		done   bool
+		asking int // the asking the event says, 0 when none is said
+	}{
+		{name: "given up by its stream: asked again under a new id and done from it", puts: 2, done: true, asking: 2},
+		{name: "asked again by hand: the same, with no check bound", byHand: true, puts: 2, done: true, asking: 2},
+		{name: "never answerable: failed after its askings", askings: 2, always: true, puts: 2},
+		{name: "taken over at its last asking: failed, not asked again", askings: 3, always: true, step: "asked mesh-build-go again (asking 3): x", puts: 0},
+	} {
+		t.Run(c.name, func(t *testing.T) { // 1945
+			r := newRig(t, goBundle)
+			r.asks.Answer = nil
+			r.b.Askings = c.askings
+			var said []AskedAgain
+			var mu sync.Mutex
+			r.b.Said = func(e AskedAgain) { mu.Lock(); said = append(said, e); mu.Unlock() }
+			first := ""
+			if c.step != "" {
+				first = "mvl0frv18d1960"
+				data, _ := delivery.EncodeBuilderBuild(delivery.BuilderBuild{State: delivery.BuildBuilding, By: "novox.builder", Manifest: manifestD,
+					Step: c.step, Asks: map[string]string{delivery.AskGo: first}})
+				r.st.vals[r.key] = data
+			}
+			if !c.byHand {
+				r.b.Gone = &goneAsks{why: func(a delivery.StepAsk) string {
+					if c.always || a.ID == first || first == "" {
+						return "it waits first in line while a node took a later ask"
+					}
+					return ""
+				}}
+				r.b.CheckAfter = 5 * time.Millisecond
+			}
+			r.b.Module(context.Background(), "m", moduleValue(t, "modules/m"))
+			if c.byHand {
+				waitForAsks(t, r, 1)
+				for deadline := time.Now().Add(2 * time.Second); len(r.b.AskAgain(r.key, "stuck since 19:06")) == 0; time.Sleep(time.Millisecond) {
+					if time.Now().After(deadline) {
+						t.Fatal("no lane of the key waits")
+					}
+				}
+			}
+			if c.done {
+				asks := waitForAsks(t, r, c.puts)
+				if first == "" && !c.byHand {
+					first = asks[0].ID
+				}
+				last := asks[len(asks)-1]
+				if v := valueOf(t, r.st, r.key); v.Asks[delivery.AskGo] != last.ID || !strings.Contains(v.Step, "again (asking 2)") {
+					t.Fatalf("the key = %+v, want building naming the ask asked again %s", v, last.ID)
+				}
+				r.asks.End(r.answer(last))
+			}
+			r.b.Wait()
+			v := valueOf(t, r.st, r.key)
+			if n := len(r.asks.Asks()); n != c.puts {
+				t.Errorf("%d asks put, want %d", n, c.puts)
+			}
+			if c.done && v.State != delivery.BuildDone {
+				t.Errorf("the key = %+v, want done from the ask asked again", v)
+			}
+			if !c.done && (v.State != delivery.BuildFailed || !strings.Contains(v.Error, "could not be answered and was asked")) {
+				t.Errorf("the key = %+v, want failed saying the ask could not be answered", v)
+			}
+			mu.Lock()
+			defer mu.Unlock()
+			if c.asking > 0 && (len(said) != 1 || said[0].Asking != c.asking || said[0].New == said[0].Old || said[0].Key != r.key || said[0].Why == "") {
+				t.Errorf("said %+v, want once at asking %d", said, c.asking)
+			}
+		})
 	}
+	t.Run("what the bus says of an ask", func(t *testing.T) { // 1945
+		put := time.Date(2026, 10, 25, 17, 6, 0, 0, time.UTC)
+		later := &delivery.Step{Node: "g14", Started: put.Add(time.Minute)}
+		for _, c := range []struct {
+			seen AskSeen
+			want string // a part of the why, "" when it can be answered
+		}{
+			{AskSeen{Stream: "S", Step: true}, ""},
+			{AskSeen{Stream: "S"}, "no longer holds it"},
+			{AskSeen{Stream: "S", OnStream: true, PutAt: put, Later: later}, "no longer delivers it"},
+			{AskSeen{Stream: "S", OnStream: true, PutAt: put, Ahead: 2, Later: later}, ""},
+			{AskSeen{Stream: "S", OnStream: true, PutAt: put}, ""},
+		} {
+			got := c.seen.Why()
+			if (c.want == "") != (got == "") || !strings.Contains(got, c.want) {
+				t.Errorf("%+v: why %q, want %q", c.seen, got, c.want)
+			}
+		}
+	})
+	t.Run("the tool builder_stuck", func(t *testing.T) { // 1945
+		at := time.Date(2026, 10, 25, 17, 6, 0, 0, time.UTC)
+		state := func(vals map[string]string) StateRead {
+			return StateRead{Keys: func() ([]string, error) {
+				var ks []string
+				for k := range vals {
+					ks = append(ks, k)
+				}
+				return ks, nil
+			}, Get: func(k string) (json.RawMessage, time.Time, error) {
+				if v, ok := vals[k]; ok {
+					return json.RawMessage(v), at, nil
+				}
+				return nil, time.Time{}, nil
+			}}
+		}
+		key, _ := delivery.BuilderBuildKey("m", fp1)
+		done, _ := delivery.BuilderBuildKey("d", fp1)
+		building, _ := delivery.EncodeBuilderBuild(delivery.BuilderBuild{State: delivery.BuildBuilding, By: "novox.builder", Manifest: manifestD,
+			Step: "asked mesh-build-go again (asking 2): x", Asks: map[string]string{delivery.AskGo: "mvo53jcm70b5f7", delivery.AskImage: "mvo53jcm70b5f8"}})
+		finished, _ := delivery.EncodeBuilderBuild(delivery.BuilderBuild{State: delivery.BuildDone, Manifest: manifestD, Step: "built"})
+		running, _ := json.Marshal(delivery.Step{V: delivery.Version, ID: "mvo53jcm70b5f8", Tool: "delivery ask", Node: "g14", State: delivery.StepRunning, Started: at})
+		_, reask, _ := NewReAsk("m", fp1, "stuck", "shanks.lead", at)
+		r := StuckReader{Builds: state(map[string]string{key: string(building), done: string(finished)}), ReAsks: state(map[string]string{key: string(reask)}),
+			Steps: map[string]StateRead{buildrun.GoBuildModule: state(map[string]string{}), ImageBuildModule: state(map[string]string{"g14.mvo53jcm70b5f8": string(running)})},
+			Now:   func() time.Time { return at.Add(time.Hour) }}
+		out, err := r.Stuck("")
+		if err != nil {
+			t.Fatal(err)
+		}
+		stuck := out["stuck"].([]Stuck)
+		if len(stuck) != 1 || stuck[0].Key != key || stuck[0].Age != "1h0m0s" || stuck[0].NoneStep != 1 || stuck[0].ReAsk == nil || stuck[0].ReAsk.Why != "stuck" {
+			t.Fatalf("stuck = %+v, want the building key alone, an hour old, one ask with no step, with its ask again", stuck)
+		}
+		a := stuck[0].Asks
+		if len(a) != 2 || a[0].Language != delivery.AskGo || a[0].Asking != 2 || a[0].Step != "none" || a[0].Asked.IsZero() ||
+			a[1].Asking != 1 || !strings.HasPrefix(a[1].Step, "running on g14") {
+			t.Fatalf("asks = %+v, want go at asking 2 with no step and image at asking 1 running on g14", a)
+		}
+	})
 }
 
 // heldAsks says an ask is held when its id is among held.

modules/mesh-delivery-builder/internal/build/stuck.go

@@ -0,0 +1,200 @@
+package build
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"sort"
+	"strconv"
+	"strings"
+	"time"
+
+	"git.novox.be/novox/mesh-sdk/go/delivery"
+	"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
+)
+
+// The builder's tools (novox/hq issue 1945; the instruction "tools are how a module is spoken to"): builder_stuck reads
+// every key that waits on an ask, builder_ask_again asks a key's build again now. The tools bundle serves them through
+// the tool runner; the builder's process serves no tool (ADR 0437 §4), so builder_ask_again writes the ask into the
+// builder's state reasks, which the builder watches and answers there.
+const (
+	StuckTool    = "builder_stuck"
+	AskAgainTool = "builder_ask_again"
+	// ReAsksState is the builder's state of asks to ask again: the bucket mesh-delivery-builder_reasks, keyed as the build key.
+	ReAsksState  = "reasks"
+	ReAsksBucket = delivery.BuilderModule + "_" + ReAsksState
+)
+
+// ReAsk is a value of the reasks state: why a person asked the key's build again, who and when, and, once the builder
+// read it, its answer.
+type ReAsk struct {
+	V        int       `json:"v"`
+	Why      string    `json:"why"`
+	By       string    `json:"by,omitempty"`
+	At       time.Time `json:"at"`
+	Answer   string    `json:"answer,omitempty"`
+	Answered time.Time `json:"answered,omitzero"`
+}
+
+// DecodeReAsk reads a reasks value.
+func DecodeReAsk(data []byte) (ReAsk, error) {
+	var r ReAsk
+	if err := json.Unmarshal(data, &r); err != nil {
+		return r, err
+	}
+	if r.V != 1 {
+		return r, fmt.Errorf("a reasks value of version %d, not 1", r.V)
+	}
+	return r, nil
+}
+
+// NewReAsk is the value builder_ask_again writes for a module's fingerprint: its key and the value.
+func NewReAsk(module, fp, why, by string, at time.Time) (string, []byte, error) {
+	key, err := delivery.BuilderBuildKey(module, fp)
+	if err != nil {
+		return "", nil, err
+	}
+	if strings.TrimSpace(why) == "" {
+		return "", nil, errors.New("say why the build is asked again (why)")
+	}
+	data, err := json.Marshal(ReAsk{V: 1, Why: strings.TrimSpace(why), By: by, At: at.UTC()})
+	return key, data, err
+}
+
+// StateRead is a state the tools read: its keys, and a key's value with when it was written (zero when not known).
+type StateRead struct {
+	Keys func() ([]string, error)
+	Get  func(key string) (json.RawMessage, time.Time, error)
+}
+
+// StuckAsk is one ask a key waits on.
+type StuckAsk struct {
+	Language    string `json:"language"`
+	BuildModule string `json:"build_module"`
+	ID          string `json:"id"`
+	Asking      int    `json:"asking"`
+	// Asked is when the ask was put, read from its id; Step is what its build module's steps hold of it: none, or its
+	// state on its node since its start.
+	Asked time.Time `json:"asked,omitzero"`
+	Step  string    `json:"step"`
+}
+
+// Stuck is one build key that waits on an ask.
+type Stuck struct {
+	Key      string     `json:"key"`
+	Module   string     `json:"module"`
+	FP       string     `json:"fp"`
+	State    string     `json:"state"`
+	Step     string     `json:"step"`
+	By       string     `json:"by"`
+	Since    time.Time  `json:"since,omitzero"`
+	Age      string     `json:"age,omitempty"`
+	Asks     []StuckAsk `json:"asks"`
+	ReAsk    *ReAsk     `json:"reask,omitempty"`
+	Unasked  bool       `json:"unasked,omitempty"`
+	NoneStep int        `json:"asks_with_no_step"`
+}
+
+// StuckReader reads what builder_stuck answers.
+type StuckReader struct {
+	Builds StateRead
+	ReAsks StateRead
+	// Steps are the build modules' steps states, by build module.
+	Steps map[string]StateRead
+	Now   func() time.Time
+}
+
+// Stuck is every key building or queued with its asks, oldest first; with module, that module's only.
+func (r StuckReader) Stuck(module string) (map[string]any, error) {
+	keys, err := r.Builds.Keys()
+	if err != nil {
+		return nil, err
+	}
+	now := time.Now
+	if r.Now != nil {
+		now = r.Now
+	}
+	stepKeys := map[string][]string{}
+	for bm, s := range r.Steps {
+		ks, err := s.Keys()
+		if err != nil {
+			return nil, fmt.Errorf("the steps of %s: %w", bm, err)
+		}
+		stepKeys[bm] = ks
+	}
+	out := []Stuck{}
+	for _, key := range keys {
+		mod, fp, err := delivery.ParseBuilderBuildKey(key)
+		if err != nil || (module != "" && mod != module) {
+			continue
+		}
+		raw, at, err := r.Builds.Get(key)
+		if err != nil || raw == nil {
+			continue
+		}
+		v, err := delivery.DecodeBuilderBuild(raw)
+		if err != nil || (v.State != delivery.BuildBuilding && v.State != delivery.BuildQueued) {
+			continue
+		}
+		s := Stuck{Key: key, Module: mod, FP: fp, State: v.State, Step: v.Step, By: v.By, Since: at, Asks: []StuckAsk{}}
+		if !at.IsZero() {
+			s.Age = now().Sub(at).Round(time.Second).String()
+		}
+		for language, id := range v.Asks {
+			bm, _ := delivery.BuildModuleOf(language)
+			a := StuckAsk{Language: language, BuildModule: bm, ID: id, Asking: askingOf(v.Step, bm), Step: "none"}
+			if t, ok := idTime(id); ok {
+				a.Asked = t
+			}
+			if st, ok := r.Steps[bm]; ok {
+				for _, k := range stepKeys[bm] {
+					if !strings.HasSuffix(k, "."+id) {
+						continue
+					}
+					if raw, _, err := st.Get(k); err == nil && raw != nil {
+						if step, err := delivery.DecodeStep(raw); err == nil {
+							a.Step = fmt.Sprintf("%s on %s since %s", step.State, step.Node, step.Started.UTC().Format(time.RFC3339))
+						}
+					}
+				}
+			}
+			if a.Step == "none" {
+				s.NoneStep++
+			}
+			s.Asks = append(s.Asks, a)
+		}
+		sort.Slice(s.Asks, func(i, j int) bool { return s.Asks[i].Language < s.Asks[j].Language })
+		s.Unasked = v.State == delivery.BuildBuilding && len(v.Asks) == 0
+		if r.ReAsks.Get != nil {
+			if raw, _, err := r.ReAsks.Get(key); err == nil && raw != nil {
+				if ra, err := DecodeReAsk(raw); err == nil {
+					s.ReAsk = &ra
+				}
+			}
+		}
+		out = append(out, s)
+	}
+	sort.Slice(out, func(i, j int) bool {
+		if !out[i].Since.Equal(out[j].Since) {
+			return out[i].Since.Before(out[j].Since)
+		}
+		return out[i].Key < out[j].Key
+	})
+	return map[string]any{"stuck": out, "count": len(out),
+		"delivery_counts": "not readable by the builder: the build seats' workers' information is granted to the delivery view alone (its backlog); an ask given up on is told by the builder at its check bound and asked again"}, nil
+}
+
+// StuckBuildModules are the build modules whose steps builder_stuck reads.
+var StuckBuildModules = []string{buildrun.GoBuildModule, buildrun.NodeBuildModule, ImageBuildModule}
+
+// idTime is when a step id was made (buildstep.NewID: the time in milliseconds in base 36, then six hex digits).
+func idTime(id string) (time.Time, bool) {
+	if len(id) <= 6 {
+		return time.Time{}, false
+	}
+	ms, err := strconv.ParseInt(id[:len(id)-6], 36, 64)
+	if err != nil || ms <= 0 {
+		return time.Time{}, false
+	}
+	return time.UnixMilli(ms).UTC(), true
+}

modules/mesh-delivery-builder/manifest_test.go

@@ -23,18 +23,21 @@ type state struct {
 }
 
 type artifact struct {
-	Name     string `json:"name"`
-	Kind     string `json:"kind"`
-	Language string `json:"language"`
-	System   string `json:"system"`
-	From     string `json:"from"`
-	Binary   string `json:"binary"`
+	Name     string   `json:"name"`
+	Kind     string   `json:"kind"`
+	Language string   `json:"language"`
+	System   string   `json:"system"`
+	From     string   `json:"from"`
+	Binary   string   `json:"binary"`
+	Loads    []string `json:"loads"`
 }
 
 type manifest struct {
 	Module   string   `json:"module"`
 	Version  string   `json:"version"`
 	Slug     string   `json:"slug"`
+	Tools    []string `json:"tools"`
+	Emits    []string `json:"emits"`
 	Consumes []string `json:"consumes"`
 	State    []state  `json:"state"`
 	Reads    []string `json:"reads"`
@@ -84,7 +87,7 @@ func TestManifestDeclarations(t *testing.T) { // 1607/AC1
 	if m.Module != "mesh-delivery-builder" {
 		t.Errorf("module = %q", m.Module)
 	}
-	if want := []state{{Name: "builds", History: 5}, {Name: "instances", History: 1}}; !reflect.DeepEqual(m.State, want) { // delivery.BuilderBuildsHistory
+	if want := []state{{Name: "builds", History: 5}, {Name: "instances", History: 1}, {Name: "reasks", History: 1}}; !reflect.DeepEqual(m.State, want) { // delivery.BuilderBuildsHistory
 		t.Errorf("state = %+v, want %+v", m.State, want)
 	}
 	// 1887/A4c: the builder chooses no node, so it reads neither presence nor have, invokes no build module's tool and
@@ -112,7 +115,14 @@ func TestManifestDeclarations(t *testing.T) { // 1607/AC1
 	if m.SecretsOwner != "mesh-delivery-builder" {
 		t.Errorf("secrets-owner = %q", m.SecretsOwner)
 	}
-	for _, k := range []string{"streams", "logs", "emits", "claims", "provides", "requires", "defines-seats", "tools", "seat", "seats", "bus", "config"} {
+	// 1945: the builder's tools, served by its tools bundle, and the event it says when it asks an ask again.
+	if want := []string{"builder_ask_again", "builder_stuck"}; !reflect.DeepEqual(sorted(m.Tools), want) {
+		t.Errorf("tools = %v, want %v", m.Tools, want)
+	}
+	if want := []string{"build.asked-again"}; !reflect.DeepEqual(m.Emits, want) {
+		t.Errorf("emits = %v, want %v", m.Emits, want)
+	}
+	for _, k := range []string{"streams", "logs", "claims", "provides", "requires", "defines-seats", "seat", "seats", "bus", "config"} {
 		if _, ok := top[k]; ok {
 			t.Errorf("the manifest declares %q: the builder has none", k)
 		}
@@ -127,7 +137,7 @@ func TestManifestProcess(t *testing.T) { // 1607/AC1
 			procs = append(procs, r)
 		}
 		if r["type"] == "tool" || r["type"] == "seat" {
-			t.Errorf("resource %v: the builder serves no tool and holds no seat", r)
+			t.Errorf("resource %v: the builder's tools are its bundle's, and it holds no seat", r)
 		}
 	}
 	if len(procs) != 1 {
@@ -149,8 +159,13 @@ func TestManifestProcess(t *testing.T) { // 1607/AC1
 	if _, bus := p["bus"]; bus {
 		t.Errorf("the process declares a bus: it runs on its own broker credential")
 	}
-	if len(m.Build.Artifacts) != 1 {
-		t.Fatalf("artifacts = %+v, want the program alone", m.Build.Artifacts)
+	if len(m.Build.Artifacts) != 2 {
+		t.Fatalf("artifacts = %+v, want the program and the tools", m.Build.Artifacts)
+	}
+	if a := m.Build.Artifacts[1]; a.Name != "tools" || a.Kind != "bundle" || a.Binary != "mesh-delivery-builder-tools" || !reflect.DeepEqual(a.Loads, []string{"mesh-delivery-builder-tools"}) {
+		t.Errorf("tools artifact = %+v", a) // 1945
+	} else if st, err := os.Stat(a.From); err != nil || !st.IsDir() {
+		t.Errorf("the tools' source %s: %v", a.From, err)
 	}
 	a := m.Build.Artifacts[0]
 	if st, err := os.Stat(a.From); err != nil || !st.IsDir() || a.Binary != "mesh-delivery-builder" || !strings.HasSuffix(a.From, a.Binary) || a.Kind != "bundle" || a.Language != "go" {

modules/mesh-delivery-builder/module.json

@@ -2,6 +2,13 @@
   "module": "mesh-delivery-builder",
   "version": "1",
   "slug": "builder",
+  "tools": [
+    "builder_ask_again",
+    "builder_stuck"
+  ],
+  "emits": [
+    "build.asked-again"
+  ],
   "consumes": [
     "mesh-build-go.toolchain.reported",
     "mesh-build-node.toolchain.reported",
@@ -12,7 +19,8 @@
   ],
   "state": [
     { "name": "builds", "history": 5 },
-    { "name": "instances", "history": 1 }
+    { "name": "instances", "history": 1 },
+    { "name": "reasks", "history": 1 }
   ],
   "reads": [
     "mesh-build-go.steps",
@@ -78,6 +86,17 @@
         "system": "arch",
         "from": "cmd/mesh-delivery-builder",
         "binary": "mesh-delivery-builder"
+      },
+      {
+        "name": "tools",
+        "kind": "bundle",
+        "language": "go",
+        "system": "arch",
+        "from": "cmd/mesh-delivery-builder-tools",
+        "binary": "mesh-delivery-builder-tools",
+        "loads": [
+          "mesh-delivery-builder-tools"
+        ]
       }
     ]
   }