Merge pull request 'builder_build_again: a person can build a failed key again, as the builder does by itself (issue 1126)' (#95) from fix/1126-build-again into main

modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/main.go

@@ -7,6 +7,7 @@ package main
 import (
 	"context"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"os"
 	"strings"
@@ -24,8 +25,8 @@ import (
 type queueAsker func(key string, value []byte) (json.RawMessage, error)
 
 // Tools are the builder's tools: stuck reads, askAgain writes the reasks value, queue writes a queues value and waits for
-// the builder's read.
+// the builder's read, buildAgain writes a reasks value of mode build and waits for the builder's answer.
-func Tools(r build.StuckReader, askAgain func(key string, value []byte) error, queue queueAsker, now func() time.Time) []stdio.Tool {
+func Tools(r build.StuckReader, askAgain func(key string, value []byte) error, queue, buildAgain queueAsker, now func() time.Time) []stdio.Tool {
 	str := func(args map[string]any, k string) string { s, _ := args[k].(string); return strings.TrimSpace(s) }
 	return []stdio.Tool{
 		{Name: build.StuckTool,
@@ -57,6 +58,42 @@ func Tools(r build.StuckReader, askAgain func(key string, value []byte) error, q
 				return map[string]any{"key": key, "asked": true,
 					"next": "the builder answers in " + build.ReAsksState + "." + key + "; " + build.StuckTool + " shows the answer and the new ask"}, nil
 			}},
+		{Name: build.BuildAgainTool,
+			Description: "Build a failed key again (rebuild a module, build a failed key again): the builder writes the key of module and " +
+				"fingerprint building as a new revision of the same key and asks the build seats, as it does by itself when it looks at a module " +
+				"again (ADR 0429 clause 18). Answers the new revision and the ask id of each build module (go, node, image). Refuses a key that " +
+				"is not failed, and says why: done (nothing to do), or waiting (building or queued: " + build.AskAgainTool + " asks its waiting asks " +
+				"again); a key failed for a reason the builder never retries on the same fingerprint (a pin missing, a version taken, digests that " +
+				"differ) and a fingerprint its module no longer has are refused too. Use it for a key builder_stuck does not show, which failed.",
+			Input: map[string]any{"type": "object", "required": []string{"module", "fp", "why"}, "properties": map[string]any{
+				"module": map[string]any{"type": "string", "description": "the module's name"},
+				"fp":     map[string]any{"type": "string", "description": "its new-form fingerprint, src2:<hex>"},
+				"why":    map[string]any{"type": "string", "description": "why it is built again, in plain words"}}},
+			Run: func(args map[string]any) (any, error) {
+				key, value, err := build.NewBuildAgain(str(args, "module"), str(args, "fp"), str(args, "why"), stdio.CallerOf(args), now())
+				if err != nil {
+					return nil, err
+				}
+				raw, err := buildAgain(key, value)
+				if err != nil {
+					return nil, err
+				}
+				r, err := build.DecodeReAsk(raw)
+				if err != nil {
+					return nil, err
+				}
+				if r.Refused {
+					return nil, errors.New(r.Answer)
+				}
+				if r.Failed != "" {
+					return nil, errors.New(r.Answer)
+				}
+				if r.Revision == 0 && !r.Pending {
+					return nil, fmt.Errorf("the builder answered neither a new revision nor a refusal (an older builder?): %s", r.Answer)
+				}
+				return map[string]any{"key": key, "revision": r.Revision, "asks": r.Asks, "answer": r.Answer,
+					"next": build.StuckTool + " shows the key and its asks while they wait"}, nil
+			}},
 		{Name: build.SeatQueueTool,
 			Description: "What waits on a build seat's stream (SEAT_NODE_BUILD_<LANGUAGE>), as the builder reads it: the stream's message count and " +
 				"sequences, and per kind (delivery, check, recheck) its messages oldest first, at most limit each, with sequence, ask id, when asked " +
@@ -87,8 +124,17 @@ func Tools(r build.StuckReader, askAgain func(key string, value []byte) error, q
 
 // askQueue writes the queues value at key and waits, on a watch of the key, at most bound for the builder's answer.
 func askQueue(s stdio.KeptState, bound time.Duration) queueAsker {
+	return askKept(s, build.QueuesState, build.QueuesBucket, "the read", bound, func(v []byte) bool {
+		a, err := build.DecodeQueueAsk(v)
+		return err == nil && !a.Answered.IsZero()
+	})
+}
+
+// askKept writes a value at key of a state the builder watches and answers there, and waits, on a watch of the key, at most
+// bound for a value written after it that answered says is the builder's answer.
+func askKept(s stdio.KeptState, state, bucket, what string, bound time.Duration, answered func([]byte) bool) queueAsker {
 	return func(key string, value []byte) (json.RawMessage, error) {
-		answered := make(chan json.RawMessage, 1)
+		heard := make(chan json.RawMessage, 1)
 		var rev uint64
 		var mu sync.Mutex
 		if err := s.Watch(key, func(c stdio.StateChange) error {
@@ -98,36 +144,34 @@ func askQueue(s stdio.KeptState, bound time.Duration) queueAsker {
 			if c.Op != "put" || mine == 0 || c.Revision <= mine {
 				return nil
 			}
-			if a, err := build.DecodeQueueAsk(c.Value); err == nil && !a.Answered.IsZero() {
+			if answered(c.Value) {
 				select {
-				case answered <- c.Value:
+				case heard <- c.Value:
 				default:
 				}
 			}
 			return nil
 		}); err != nil {
-			return nil, fmt.Errorf("watching %s.%s: %w", build.QueuesState, key, err)
+			return nil, fmt.Errorf("watching %s.%s: %w", state, key, err)
 		}
 		r, err := s.Put(key, json.RawMessage(value))
 		if err != nil {
-			return nil, fmt.Errorf("the read could not be written to %s: %w", build.QueuesBucket, err)
+			return nil, fmt.Errorf("%s could not be written to %s: %w", what, bucket, err)
 		}
 		mu.Lock()
 		rev = r
 		mu.Unlock()
 		// The watch may have heard the answer before the revision was known: read it once too.
-		if e, err := s.Get(key); err == nil && e != nil && e.Revision > r {
+		if e, err := s.Get(key); err == nil && e != nil && e.Revision > r && answered(e.Value) {
-			if a, err := build.DecodeQueueAsk(e.Value); err == nil && !a.Answered.IsZero() {
+			return e.Value, nil
-				return e.Value, nil
-			}
 		}
 		wait, cancel := context.WithTimeout(context.Background(), bound) // a context's bound, no timer (ADR 0437 §8)
 		defer cancel()
 		select {
-		case v := <-answered:
+		case v := <-heard:
 			return v, nil
 		case <-wait.Done():
-			return nil, fmt.Errorf("the builder did not answer %s.%s within %s: it is not running, or cannot read %s", build.QueuesState, key, bound, build.QueuesBucket)
+			return nil, fmt.Errorf("the builder did not answer %s.%s within %s: it is not running, or cannot read %s", state, key, bound, bucket)
 		}
 	}
 }
@@ -174,7 +218,11 @@ func main() {
 		_, err := reasks.Put(key, json.RawMessage(value))
 		return err
 	}
-	if err := stdio.Serve("", Tools(r, put, askQueue(stdio.State(build.QueuesState), 25*time.Second), time.Now)); err != nil {
+	buildAgain := askKept(reasks, build.ReAsksState, build.ReAsksBucket, "the request", 25*time.Second, func(v []byte) bool {
+		a, err := build.DecodeReAsk(v)
+		return err == nil && !a.Answered.IsZero()
+	})
+	if err := stdio.Serve("", Tools(r, put, askQueue(stdio.State(build.QueuesState), 25*time.Second), buildAgain, time.Now)); err != nil {
 		fmt.Fprintf(os.Stderr, "[mesh-delivery-builder-tools] %v\n", err)
 		os.Exit(1)
 	}

modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reasks.go

@@ -18,8 +18,13 @@ import (
 // askAgainer is the part of the builder a reasks value asks through.
 type askAgainer interface {
 	AskAgain(key, why string) []string
+	BuildAgain(ctx context.Context, module, fp, why, by string, bound time.Duration) build.BuildAgainResult
 }
 
+// buildAgainBound is how long the builder waits for the claim and the asks of a key built again by hand; the tool waits
+// longer for the builder's answer.
+const buildAgainBound = 15 * time.Second
+
 // reAsked answers one value of the reasks state (the tool builder_ask_again, issue 1945): a value not yet answered asks the
 // key's waiting lanes again and is written back, by compare-and-set at its revision, with what the builder did. A value
 // answered already (this builder's own write) or deleted is left.
@@ -35,6 +40,11 @@ func reAsked(ctx context.Context, b askAgainer, kv jetstream.KeyValue, e jetstre
 	if r.Answer != "" {
 		return
 	}
+	if r.Mode == build.ModeBuild {
+		// The answer comes when the claim and the asks are written, so it is made beside the watch, not in it.
+		go builtAgain(ctx, b, kv, e, r, logf)
+		return
+	}
 	asked := b.AskAgain(e.Key(), r.Why)
 	if len(asked) == 0 {
 		r.Answer = "no ask of this key waits in this builder now (it is not building, or its build has not reached its ask yet): nothing was asked; builder_stuck shows where it stands"
@@ -74,3 +84,33 @@ func emitAskedAgain(ctx context.Context, js jetstream.JetStream, cred process.Ac
 	}
 	return nil
 }
+
+// builtAgain answers a reasks value of mode build (the tool builder_build_again, issue 1126 note 72): the failed key is
+// built again as a new revision, and the value is written back with the revision and the asks put, or why nothing was done.
+func builtAgain(ctx context.Context, b askAgainer, kv jetstream.KeyValue, e jetstream.KeyValueEntry, r build.ReAsk, logf func(string, ...any)) {
+	res := b.BuildAgain(ctx, r.Module, r.FP, r.Why, r.By, buildAgainBound)
+	switch {
+	case res.Refused != "":
+		r.Answer, r.Refused = res.Refused, true
+	case res.Failed != "":
+		r.Answer, r.Revision, r.Failed = fmt.Sprintf("the key is written building as revision %d, and the build failed before any ask was put: %s", res.Revision, res.Failed), res.Revision, res.Failed
+	default:
+		r.Pending = res.Revision == 0
+		r.Answer, r.Revision, r.Asks = "built again: the key is written building as revision "+fmt.Sprint(res.Revision)+" and its asks are put", res.Revision, res.Asks
+		if res.Pending != "" {
+			r.Answer = res.Pending
+			if res.Revision != 0 {
+				r.Answer = "built again: the key is written building as revision " + fmt.Sprint(res.Revision) + "; " + res.Pending
+			}
+		}
+	}
+	r.Answered = time.Now().UTC()
+	logf("%s: build again asked by %s (%s): %s", e.Key(), r.By, r.Why, r.Answer)
+	data, err := json.Marshal(r)
+	if err != nil {
+		return
+	}
+	if _, err := kv.Update(ctx, e.Key(), data, e.Revision()); err != nil && !errors.Is(err, jetstream.ErrKeyExists) && !errors.Is(err, jetstream.ErrKeyRevisionMismatch) {
+		logf("%s.%s: the answer could not be written: %v", build.ReAsksState, e.Key(), err)
+	}
+}

modules/mesh-delivery-builder/internal/build/buildagain.go

@@ -0,0 +1,183 @@
+package build
+
+import (
+	"context"
+	"fmt"
+	"maps"
+	"strings"
+	"time"
+
+	"git.novox.be/novox/mesh-sdk/go/delivery"
+	"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
+)
+
+// **A failed key is built again by hand** (novox/hq issue 1126 note 72; ADR 0429 clause 18: "a fingerprint is built again
+// only when its build key failed, as a new revision of the same key"). The builder does it by itself whenever it looks at a
+// module again (a modules value, a change of the toolchains, artifacts-gone, a moved trunk): Claimer.Claim writes a failed
+// key building as a new revision and the build asks the build seats. Nothing looks at an unchanged module on its own, so a key
+// that failed (a node refused the build, a toolchain was missing) stayed failed until the module changed. The tool
+// builder_build_again (BuildAgain) makes the builder look at one module now, which does exactly that and no more: it keeps
+// the claim's rules, so a key failed for a reason the builder never retries on the same fingerprint stays refused.
+
+// BuildAgainResult is the builder's answer to a request to build a key again.
+type BuildAgainResult struct {
+	// Refused is why nothing was done, "" when the key was written building as a new revision.
+	Refused string
+	// Revision is the revision the claim wrote; Asks are the ask ids put by language, as far as they were put within the bound.
+	Revision uint64
+	Asks     map[string]string
+	// Pending says the builder had not claimed the key, or not put every ask, when it answered: not a refusal.
+	Pending string
+	// Failed is the error of the key when the build ended failed before its asks were put (a plan that cannot be read, a
+	// toolchain not agreed): the new revision was written, and nothing was asked.
+	Failed string
+}
+
+// handWatch follows one request to build a key again until its asks are put.
+type handWatch struct {
+	rev   uint64
+	lanes int
+	asks  map[string]string
+	done  chan struct{}
+	shut  bool
+	// ended is set when the look ended before the asks were all put.
+	ended bool
+}
+
+// settle closes done once the claim is written and every lane has put its ask. Call with b.mu held.
+func (w *handWatch) settle() {
+	if !w.shut && w.rev > 0 && w.lanes >= 0 && len(w.asks) >= w.lanes {
+		w.shut = true
+		close(w.done)
+	}
+}
+
+func (b *Builder) hand(key string, f func(w *handWatch)) {
+	b.mu.Lock()
+	defer b.mu.Unlock()
+	if w := b.hands[key]; w != nil {
+		f(w)
+	}
+}
+
+func (b *Builder) handClaimed(key string, rev uint64) {
+	b.hand(key, func(w *handWatch) { w.rev = rev; w.settle() })
+}
+
+func (b *Builder) handLanes(key string, n int) {
+	b.hand(key, func(w *handWatch) { w.lanes = n; w.settle() })
+}
+
+func (b *Builder) handAsk(key, language, id string) {
+	b.hand(key, func(w *handWatch) { w.asks[language] = id; w.settle() })
+}
+
+// handEnded ends the watch of a request whose look ended: it wrote no claim, or the build ended.
+func (b *Builder) handEnded(key string) {
+	b.hand(key, func(w *handWatch) {
+		if !w.shut {
+			w.shut, w.ended = true, true
+			close(w.done)
+		}
+	})
+}
+
+// BuildAgain builds the failed key of module at fp again, as the builder does by itself: the key is written building as a
+// new revision and the build module asks are put on the build seats. It answers once the asks are put, or at bound. A key
+// that is not failed is refused with why: done (nothing to do) or waiting (builder_ask_again asks its waiting asks again).
+// ctx is the builder's own context: the build it starts lives on it.
+func (b *Builder) BuildAgain(ctx context.Context, module, fp, why, by string, bound time.Duration) BuildAgainResult {
+	b.init()
+	key, err := delivery.BuilderBuildKey(module, fp)
+	if err != nil {
+		return BuildAgainResult{Refused: err.Error()}
+	}
+	value, _, found, err := b.Claimer.Store.Get(ctx, key)
+	if err != nil {
+		return BuildAgainResult{Refused: fmt.Sprintf("%s cannot be read: %v", key, err)}
+	}
+	if !found {
+		return BuildAgainResult{Refused: fmt.Sprintf("%s has no build key: nothing failed. The builder builds a fingerprint by itself once the indexer writes it to its module", key)}
+	}
+	cur, err := delivery.DecodeBuilderBuild(value)
+	if err != nil {
+		return BuildAgainResult{Refused: fmt.Sprintf("%s holds what is not a build: %v", key, err)}
+	}
+	switch {
+	case cur.State == delivery.BuildDone:
+		return BuildAgainResult{Refused: fmt.Sprintf("%s is done: nothing to do", key)}
+	case cur.State == delivery.BuildBuilding || cur.State == delivery.BuildQueued:
+		return BuildAgainResult{Refused: fmt.Sprintf("%s is waiting (%s: %s): %s asks its waiting asks again", key, cur.State, cur.Step, AskAgainTool)}
+	case cur.State != delivery.BuildFailed:
+		return BuildAgainResult{Refused: fmt.Sprintf("%s is %q, not failed", key, cur.State)}
+	case delivery.FailedUntilChanged(cur.Error) || strings.HasPrefix(cur.Error, buildrun.DigestsDifferPrefix):
+		return BuildAgainResult{Refused: fmt.Sprintf("%s failed for a reason the builder does not retry on the same fingerprint, it waits for another fingerprint: %s", key, cur.Error)}
+	}
+	// One critical section: the key is refused when it is looked at or asked already, the module still has the fingerprint,
+	// and the watch and the look are registered before anything else can take the key.
+	w := &handWatch{lanes: -1, asks: map[string]string{}, done: make(chan struct{})}
+	b.mu.Lock()
+	m, known := b.known[module]
+	switch {
+	case !known || m.FP2 != fp:
+		b.mu.Unlock()
+		return BuildAgainResult{Refused: fmt.Sprintf("module %s does not have the fingerprint %s now (a newer one superseded it, or its modules value is not read yet): the builder builds the module's current fingerprint", module, fp)}
+	case m.Manifest == "":
+		b.mu.Unlock()
+		return BuildAgainResult{Refused: fmt.Sprintf("module %s names no manifest at %s yet, so the builder looks at nothing", module, fp)}
+	case b.inflight[key] || b.hands[key] != nil:
+		b.mu.Unlock()
+		return BuildAgainResult{Refused: fmt.Sprintf("%s is being looked at by this builder now (or asked again by hand): wait for it to end", key)}
+	}
+	b.hands[key] = w
+	b.inflight[key] = true
+	b.mu.Unlock()
+	defer func() {
+		b.mu.Lock()
+		if b.hands[key] == w {
+			delete(b.hands, key)
+		}
+		b.mu.Unlock()
+	}()
+	b.logf("%s: failed (%s); built again by hand, asked by %s: %s", key, cur.Error, by, why)
+	b.startOnce(ctx, key, module, m)
+	wait, stop := context.WithTimeout(ctx, bound) // a context's bound for a person's answer (ADR 0437 §8)
+	defer stop()
+	select {
+	case <-w.done:
+	case <-wait.Done():
+	}
+	b.mu.Lock()
+	res := BuildAgainResult{Revision: w.rev, Asks: maps.Clone(w.asks)}
+	ended, shut, lanes := w.ended, w.shut, w.lanes
+	b.mu.Unlock()
+	switch {
+	case res.Revision == 0 && shut:
+		res.Refused = fmt.Sprintf("the builder looked at %s and wrote nothing (a toolchain moved, the claim was lost or could not be written): its log says why", key)
+	case res.Revision == 0:
+		res.Pending = fmt.Sprintf("the builder has not claimed %s within %s: it waits behind other builds; builder_stuck shows where it stands", key, bound)
+	case ended && len(res.Asks) < max(lanes, 0)+boolInt(lanes < 0):
+		// The look ended before every ask was put: say how the key stands, not that it was asked.
+		if v, _, found, err := b.Claimer.Store.Get(ctx, key); err == nil && found {
+			if cur, err := delivery.DecodeBuilderBuild(v); err == nil {
+				switch cur.State {
+				case delivery.BuildFailed:
+					res.Failed = cur.Error
+				case delivery.BuildDone:
+				default:
+					res.Pending = fmt.Sprintf("the look ended with the key %s (%s) before every ask was put: its log says why", cur.State, cur.Step)
+				}
+			}
+		}
+	case !shut:
+		res.Pending = fmt.Sprintf("the key is building at revision %d, and not every ask was put within %s; builder_stuck shows the asks", res.Revision, bound)
+	}
+	return res
+}
+
+func boolInt(b bool) int {
+	if b {
+		return 1
+	}
+	return 0
+}

modules/mesh-delivery-builder/internal/build/run.go

@@ -89,6 +89,8 @@ type Builder struct {
 	held map[string]bool
 	// waiting are the lanes that wait for an ask's step, by build key and language: AskAgain asks through them.
 	waiting map[string]map[string]chan string
+	// hands are the requests to build a key again by hand that wait for the claim and its asks (BuildAgain), by build key.
+	hands map[string]*handWatch
 }
 
 type planned struct {
@@ -106,6 +108,7 @@ func (b *Builder) init() {
 		b.cands, b.via, b.trunkFP = map[string]delivery.Candidate{}, map[string]via{}, map[string]bool{}
 		b.known, b.inflight, b.again, b.plans, b.tried = map[string]delivery.Module{}, map[string]bool{}, map[string]delivery.Module{}, map[string]planned{}, map[string]bool{}
 		b.published, b.held, b.waiting = map[string]bool{}, map[string]bool{}, map[string]map[string]chan string{}
+		b.hands = map[string]*handWatch{}
 	})
 }
 
@@ -372,9 +375,35 @@ func (b *Builder) consider(ctx context.Context, module string, m delivery.Module
 	}()
 }
 
+// startOnce looks at the module for a key the caller has set inflight already (BuildAgain): the by-hand look runs once, and
+// a look the builder queued for the key meanwhile (again) still runs after it, as consider's loop does.
+func (b *Builder) startOnce(ctx context.Context, key, module string, m delivery.Module) {
+	b.wg.Add(1)
+	go func() {
+		defer b.wg.Done()
+		b.sem <- struct{}{}
+		defer func() { <-b.sem }()
+		for {
+			b.try(ctx, module, m)
+			b.mu.Lock()
+			next, more := b.again[key]
+			delete(b.again, key)
+			if !more || ctx.Err() != nil {
+				delete(b.inflight, key)
+				b.mu.Unlock()
+				return
+			}
+			b.mu.Unlock()
+			m = next // a look the builder queued meanwhile is not lost (consider's rule); it is not a second by-hand build
+		}
+	}()
+}
+
 // try claims the fingerprint and builds it, writing the key done or failed. A build that ends because ctx did is left
 // building by this builder: it takes it again when it starts (ADR 0478 clause 5).
 func (b *Builder) try(ctx context.Context, module string, m delivery.Module) {
+	key, _ := delivery.BuilderBuildKey(module, m.FP2)
+	defer b.handEnded(key)
 	plan, planErr, read := b.planOf(ctx, m.Manifest)
 	nothing := read && planErr == nil && len(plan.Tasks) == 0
 	if planErr == nil && !nothing {
@@ -385,7 +414,6 @@ func (b *Builder) try(ctx context.Context, module string, m delivery.Module) {
 			return
 		}
 	}
-	key, _ := delivery.BuilderBuildKey(module, m.FP2)
 	b.mu.Lock()
 	trunk := b.trunkFP[key]
 	first := !b.tried[key]
@@ -400,6 +428,7 @@ func (b *Builder) try(ctx context.Context, module string, m delivery.Module) {
 	if cl == nil {
 		return
 	}
+	b.handClaimed(key, cl.Rev)
 	if cl.Unasked != nil {
 		b.logf("%v", cl.Unasked)
 	}
@@ -535,6 +564,9 @@ func (b *Builder) build(ctx context.Context, module string, m delivery.Module, p
 		// One ask builds each fingerprint's part of a build module, on whichever node pulls it (ADR 0525 §3).
 		lanes = append(lanes, &lane{module: bm, tasks: tasks, toolchain: toolchain, digests: map[string]string{}, references: map[string]string{}})
 	}
+	if key, err := delivery.BuilderBuildKey(module, m.FP2); err == nil {
+		b.handLanes(key, len(lanes))
+	}
 	var wg sync.WaitGroup
 	for _, l := range lanes {
 		wg.Add(1)
@@ -717,7 +749,11 @@ func (n *askNotes) note(ctx context.Context, language, id, step string) error {
 		v.Asks = map[string]string{}
 	}
 	v.Asks[language] = id
-	return n.b.Claimer.Finish(ctx, n.cl, v)
+	if err := n.b.Claimer.Finish(ctx, n.cl, v); err != nil {
+		return err
+	}
+	n.b.handAsk(n.cl.Key, language, id)
+	return nil
 }
 
 // runLane puts the lane's part of the build as one ask on its build seat's stream (the delivery stream for a trunk key, the

modules/mesh-delivery-builder/internal/build/run_test.go

@@ -694,7 +694,48 @@ func TestABuildAskWaitingInItsStreamIsNotBounded(t *testing.T) {
 		puts   int
 		done   bool
 		asking int // the asking the event says, 0 when none is said
+		// again, when set, is a request to build the key again by hand (BuildAgain, issue 1126 note 72) of the key as
+		// it stands (key, absent when nil), the module known at known's fingerprint ("" for the rig's): refuse is a part of why it
+		// is refused, "" when it is built again as a new revision and its ask id answered.
+		again  bool
+		key    *delivery.BuilderBuild
+		known  string
+		refuse string
+		// edit changes the module the builder has read, busy marks the key looked at (1) or asked by hand (2), failed is a part of
+		// the error the key then holds when its build ends before its asks are put.
+		edit   func(m *delivery.Module)
+		busy   int
+		failed string
+		// during is a look of the builder's own at the module while the by-hand build runs: it still runs afterwards.
+		during bool
 	}{
+		{name: "build again: a look the builder queued during the by-hand build still runs afterwards", again: true, during: true,
+			key: &delivery.BuilderBuild{State: delivery.BuildFailed, Error: "ace: no tun"}},
+		{name: "build again: a key looked at by the builder now is refused, not built twice", again: true, refuse: "is being looked at", busy: 1,
+			key: &delivery.BuilderBuild{State: delivery.BuildFailed, Error: "ace: no tun"}},
+		{name: "build again: a key already asked by hand is refused, its watch kept", again: true, refuse: "is being looked at", busy: 2,
+			key: &delivery.BuilderBuild{State: delivery.BuildFailed, Error: "ace: no tun"}},
+		{name: "build again: a module with no manifest is refused", again: true, refuse: "names no manifest", edit: func(m *delivery.Module) { m.Manifest = "" },
+			key: &delivery.BuilderBuild{State: delivery.BuildFailed, Error: "ace: no tun"}},
+		{name: "build again: a toolchain that moved writes nothing and is said at once", again: true, refuse: "wrote nothing",
+			edit: func(m *delivery.Module) {
+				m.Toolchains = map[string]delivery.ToolchainUsed{"go": {Version: "go1.0", Recipe: "1"}}
+			},
+			key: &delivery.BuilderBuild{State: delivery.BuildFailed, Error: "ace: no tun"}},
+		{name: "build again: a build that fails before its asks says the key failed, not that it was asked", again: true, failed: "cannot be read",
+			edit: func(m *delivery.Module) { m.Manifest = mdigest('b') },
+			key:  &delivery.BuilderBuild{State: delivery.BuildFailed, Error: "ace: no tun"}},
+		{name: "build again: a failed key is written building as a new revision and asks the seat", again: true,
+			key: &delivery.BuilderBuild{State: delivery.BuildFailed, Error: "ace: the step failed: no tun"}},
+		{name: "build again: a done key is refused, nothing to do", again: true, refuse: "is done: nothing to do",
+			key: &delivery.BuilderBuild{State: delivery.BuildDone, Step: "built"}},
+		{name: "build again: a building key is refused, builder_ask_again is for it", again: true, refuse: "is waiting",
+			key: &delivery.BuilderBuild{State: delivery.BuildBuilding, By: "novox.builder", Step: "asked mesh-build-go", Asks: map[string]string{delivery.AskGo: "mvl0frv18d1960"}}},
+		{name: "build again: a key failed for a pin missing is refused, it waits for another fingerprint", again: true, refuse: "does not retry",
+			key: &delivery.BuilderBuild{State: delivery.BuildFailed, Error: delivery.NotPinned + "web"}},
+		{name: "build again: a key with no build is refused", again: true, refuse: "has no build key"},
+		{name: "build again: a fingerprint its module no longer has is refused", again: true, refuse: "does not have the fingerprint", known: "src2:" + strings.Repeat("9", 64),
+			key: &delivery.BuilderBuild{State: delivery.BuildFailed, Error: "ace: no tun"}},
 		{name: "given up by its stream: asked again under a new id and done from it", puts: 2, done: true, asking: 2},
 		{name: "asked again by hand: the same, with no check bound", byHand: true, puts: 2, done: true, asking: 2},
 		{name: "given up into DEAD_LETTERS: its wait ends failed with the words and it is asked again at once", dead: true, puts: 2, done: true, asking: 2},
@@ -703,6 +744,74 @@ func TestABuildAskWaitingInItsStreamIsNotBounded(t *testing.T) {
 	} {
 		t.Run(c.name, func(t *testing.T) { // 1945
 			r := newRig(t, goBundle)
+			if c.again {
+				var rev uint64
+				if c.key != nil {
+					c.key.Manifest = manifestD
+					rev = putBuild(t, r.st, r.key, *c.key)
+				}
+				m, err := delivery.DecodeModule(moduleValue(t, "modules/m"))
+				if err != nil {
+					t.Fatal(err)
+				}
+				if c.known != "" {
+					m.FP2 = c.known
+				}
+				if c.edit != nil {
+					c.edit(&m)
+				}
+				r.b.init()
+				var stored atomic.Int64
+				if c.during {
+					r.asks.Answer = nil
+					r.b.Stored = func(context.Context, []string) ([]string, error) { stored.Add(1); return nil, nil }
+				}
+				switch c.busy {
+				case 1:
+					r.b.inflight[r.key] = true
+				case 2:
+					r.b.hands[r.key] = &handWatch{}
+				}
+				r.b.known["m"] = m // the builder has read the modules value, and looks at nothing until asked
+				log := &logged{}
+				r.b.Logf = log.logf
+				res := r.b.BuildAgain(context.Background(), "m", fp1, "no tun on ace", "shanks.lead", 2*time.Second)
+				if c.during {
+					// The trunk's modules value arrives while the by-hand build waits for its ask: it is queued, and run after it.
+					r.b.Module(context.Background(), "m", moduleValue(t, "modules/m"))
+					r.asks.End(r.answer(waitForAsks(t, r, 1)[0]))
+					r.b.Wait()
+					if v := valueOf(t, r.st, r.key); v.State != delivery.BuildDone || res.Revision <= rev || stored.Load() != 1 {
+						t.Fatalf("key %+v, answer %+v, %d asks of the store; want built from the new revision and the queued look run once after it", v, res, stored.Load())
+					}
+					return
+				}
+				r.b.Wait()
+				after := r.st.revs[r.key]
+				if c.refuse != "" {
+					if !strings.Contains(res.Refused, c.refuse) || res.Revision != 0 || len(r.asks.Asks()) != 0 || after != rev {
+						t.Fatalf("refused %q (revision %d, %d asks, key at revision %d was %d); want refused with %q and nothing written or asked", res.Refused, res.Revision, len(r.asks.Asks()), after, rev, c.refuse)
+					}
+					return
+				}
+				if c.failed != "" {
+					if !strings.Contains(res.Failed, c.failed) || res.Revision <= rev || len(res.Asks) != 0 || len(r.asks.Asks()) != 0 {
+						t.Fatalf("answer %+v after %d asks; want a new revision (was %d), no ask and the key's failure %q", res, len(r.asks.Asks()), rev, c.failed)
+					}
+					return
+				}
+				asks := r.asks.Asks()
+				if res.Refused != "" || res.Pending != "" || res.Revision <= rev || len(asks) != 1 || res.Asks[delivery.AskGo] != asks[0].ID {
+					t.Fatalf("answer %+v after %d asks; want the new revision (was %d) and the ask id of the one ask", res, len(asks), rev)
+				}
+				if v := valueOf(t, r.st, r.key); v.State != delivery.BuildDone || after <= res.Revision {
+					t.Errorf("the key = %+v at revision %d; want built from the new revision %d", v, after, res.Revision)
+				}
+				if !log.has(r.key, "built again by hand", "no tun on ace", "shanks.lead") {
+					t.Errorf("the why is not logged: %v", log.lines)
+				}
+				return
+			}
 			r.asks.Answer = nil
 			r.b.Askings = c.askings
 			var said []AskedAgain

modules/mesh-delivery-builder/internal/build/stuck.go

@@ -22,6 +22,8 @@ import (
 const (
 	StuckTool    = "builder_stuck"
 	AskAgainTool = "builder_ask_again"
+	// BuildAgainTool builds a failed key again by hand (issue 1126 note 72): it writes a reasks value of Mode ModeBuild.
+	BuildAgainTool = "builder_build_again"
 	// ReAsksState is the builder's state of asks to ask again: the bucket mesh-delivery-builder_reasks, keyed as the build key.
 	ReAsksState  = "reasks"
 	ReAsksBucket = delivery.BuilderModule + "_" + ReAsksState
@@ -36,8 +38,24 @@ type ReAsk struct {
 	At       time.Time `json:"at"`
 	Answer   string    `json:"answer,omitempty"`
 	Answered time.Time `json:"answered,omitzero"`
+	// Mode is ModeBuild for a request to build a failed key again (BuildAgainTool); empty for an ask asked again.
+	Mode   string `json:"mode,omitempty"`
+	Module string `json:"module,omitempty"`
+	FP     string `json:"fp,omitempty"`
+	// Refused, Revision and Asks are the builder's answer to a ModeBuild request: why it did nothing, or the key's new
+	// revision (building) and the ask ids it put by language.
+	Refused bool `json:"refused,omitempty"`
+	// Pending says the builder answered before the key was claimed or all its asks were put: neither a refusal nor done.
+	// Failed is the error of a key written building that failed before its asks were put.
+	Pending  bool              `json:"pending,omitempty"`
+	Failed   string            `json:"failed,omitempty"`
+	Revision uint64            `json:"revision,omitempty"`
+	Asks     map[string]string `json:"asks,omitempty"`
 }
 
+// ModeBuild is the Mode of a reasks value that asks the key to be built again, not its waiting asks to be asked again.
+const ModeBuild = "build"
+
 // DecodeReAsk reads a reasks value.
 func DecodeReAsk(data []byte) (ReAsk, error) {
 	var r ReAsk
@@ -63,6 +81,19 @@ func NewReAsk(module, fp, why, by string, at time.Time) (string, []byte, error)
 	return key, data, err
 }
 
+// NewBuildAgain is the value builder_build_again writes for a module's fingerprint: its key and the value.
+func NewBuildAgain(module, fp, why, by string, at time.Time) (string, []byte, error) {
+	key, err := delivery.BuilderBuildKey(module, fp)
+	if err != nil {
+		return "", nil, err
+	}
+	if strings.TrimSpace(why) == "" {
+		return "", nil, errors.New("say why the key is built again (why)")
+	}
+	data, err := json.Marshal(ReAsk{V: 1, Why: strings.TrimSpace(why), By: by, At: at.UTC(), Mode: ModeBuild, Module: module, FP: fp})
+	return key, data, err
+}
+
 // StateRead is a state the tools read: its keys, and a key's value with when it was written (zero when not known).
 type StateRead struct {
 	Keys func() ([]string, error)

modules/mesh-delivery-builder/manifest_test.go

@@ -116,7 +116,7 @@ func TestManifestDeclarations(t *testing.T) { // 1607/AC1
 		t.Errorf("secrets-owner = %q", m.SecretsOwner)
 	}
 	// 1945: the builder's tools (builder_seat_queue: note 62), served by its tools bundle, and the event it says when it asks an ask again.
-	if want := []string{"builder_ask_again", "builder_seat_queue", "builder_stuck"}; !reflect.DeepEqual(sorted(m.Tools), want) {
+	if want := []string{"builder_ask_again", "builder_build_again", "builder_seat_queue", "builder_stuck"}; !reflect.DeepEqual(sorted(m.Tools), want) {
 		t.Errorf("tools = %v, want %v", m.Tools, want)
 	}
 	if want := []string{"build.asked-again"}; !reflect.DeepEqual(m.Emits, want) {

modules/mesh-delivery-builder/module.json

@@ -4,6 +4,7 @@
   "slug": "builder",
   "tools": [
     "builder_ask_again",
+    "builder_build_again",
     "builder_seat_queue",
     "builder_stuck"
   ],