Keep builds on the bus under their module, so no lookup reads the bucket whole (issue 1121, ADR 0427)

Milestone 5 moved 8,599 build records onto the bus under bare build ids, each
carrying its module's whole manifest. Every build lookup then read the whole
bucket, and a delivery plan's send did that per module per machine while it
held the plans and the machine's declaration: the send of build-agent never
finished, and every other act and the status waited behind it.

Builds now live under <module>.build.<id> (requests under req.<id>, id.<id>
names a build's module), with pointer keys newest, worked, held,
commit.<sha> and source.<reuse key> that move only to a newer build and are
read as hints, repaired when wrong. Manifests are kept once in the same
bucket under manifest.sha256-<hex>, so no grant the running bus lacks is
needed. The old keys are laid out once at start, idempotently, and left
unread. The calls a plan look makes on the silent clocks while it holds the
plans are bounded, since the vendored kv.Keys ignores its context.

internal/inventory/chain_requests.go

@@ -35,7 +35,14 @@ func (i *Inventory) chainRecordBuildRequest(ctx context.Context, c *chain.Chain,
 	if a.At.IsZero() {
 		a.At = time.Now().UTC()
 	}
-	key := chainBuildKey(a.ID)
+	key := chain.RequestKey(a.ID)
+	// An outcome under a module that came first has no request beside it: the request is kept on it, and its own record says
+	// it was answered (issue 1121).
+	if ptr, found, err := c.Builds.GetRaw(ctx, chain.IDKey(a.ID)); err != nil {
+		return err
+	} else if found && string(ptr.Value) != chain.RequestPrefix {
+		return i.keepAnsweredRequest(ctx, c, string(ptr.Value), a)
+	}
 	for try := 0; try < casTries; try++ {
 		_, err := c.Builds.Create(ctx, key, askedRecord(a))
 		if err == nil {
@@ -65,8 +72,35 @@ func (i *Inventory) chainRecordBuildRequest(ctx context.Context, c *chain.Chain,
 	return fmt.Errorf("build request %s is not kept on the bus: it kept moving under the write", a.ID)
 }
 
+// keepAnsweredRequest puts a request on the outcome that answered it before the request was kept, and keeps the request's
+// own record as answered.
+func (i *Inventory) keepAnsweredRequest(ctx context.Context, c *chain.Chain, module string, a BuildRequest) error {
+	outKey := chain.BuildKey(module, a.ID)
+	for try := 0; try < casTries; try++ {
+		out, found, err := c.Builds.Get(ctx, outKey)
+		if err != nil || !found {
+			return err
+		}
+		if out.Value.Request == nil {
+			out.Value.Request = toRequest(a)
+			if _, err := c.Builds.Update(ctx, outKey, out.Value, out.Revision); errors.Is(err, chain.ErrMoved) {
+				continue
+			} else if err != nil {
+				return fmt.Errorf("build request %s is not kept on the bus: %w", a.ID, err)
+			}
+		}
+		rec := askedRecord(a)
+		rec.State, rec.Module, rec.Failed, rec.At = chain.BuildAnswered, module, out.Value.Failed, out.Value.At
+		if _, err := c.Builds.Create(ctx, chain.RequestKey(a.ID), rec); err != nil && !errors.Is(err, chain.ErrExists) {
+			return fmt.Errorf("build request %s is not kept on the bus: %w", a.ID, err)
+		}
+		return nil
+	}
+	return fmt.Errorf("build request %s is not kept: it kept moving under the write", a.ID)
+}
+
 func (i *Inventory) chainBuildRequestByID(ctx context.Context, c *chain.Chain, id string) (BuildRequest, bool, error) {
-	k, found, err := c.Builds.Get(ctx, chainBuildKey(id))
+	k, found, err := c.Builds.Get(ctx, chain.RequestKey(id))
 	if err != nil || !found || k.Value.Request == nil {
 		return BuildRequest{}, false, err
 	}
@@ -75,7 +109,7 @@ func (i *Inventory) chainBuildRequestByID(ctx context.Context, c *chain.Chain, i
 
 // chainMarkRequest changes a request that has no outcome yet, by compare-and-set; one answered, or none kept, is left.
 func (i *Inventory) chainMarkRequest(ctx context.Context, c *chain.Chain, id string, mark func(*chain.Request)) error {
-	key := chainBuildKey(id)
+	key := chain.RequestKey(id)
 	for try := 0; try < casTries; try++ {
 		cur, found, err := c.Builds.Get(ctx, key)
 		if err != nil || !found || cur.Value.State != chain.BuildAsked || cur.Value.Request == nil {
@@ -94,13 +128,14 @@ func (i *Inventory) chainMarkRequest(ctx context.Context, c *chain.Chain, id str
 // chainRequests is every request kept, newest first, each with its outcome where its build was heard. Requests older than
 // KeptFor are not listed: the bucket keeps them, and no reader needs them.
 func (i *Inventory) chainRequests(ctx context.Context, c *chain.Chain) ([]RequestOutcome, error) {
-	all, err := c.Builds.All(ctx)
+	all, err := requestRecords(ctx, c)
 	if err != nil {
-		return nil, fmt.Errorf("the builds cannot be read from the bus: %w", err)
+		return nil, err
 	}
 	oldest := time.Now().Add(-KeptFor)
 	var out []RequestOutcome
-	for id, k := range all {
+	for _, k := range all {
+		id := strings.TrimPrefix(k.Key, chain.RequestPrefix+".")
 		r := k.Value.Request
 		if r == nil || !r.At.After(oldest) {
 			continue
@@ -120,6 +155,11 @@ func (i *Inventory) chainRequests(ctx context.Context, c *chain.Chain) ([]Reques
 	return out, nil
 }
 
+// requestRecords are the records under `req.`: the requests, and the builds that knew no module.
+func requestRecords(ctx context.Context, c *chain.Chain) ([]chain.Kept[chain.Build], error) {
+	return buildsIn(ctx, c, chain.RequestPrefix+".*")
+}
+
 // RequestsToAskAgain are the requests the move found never handed over (issue 981 note 7 Q5), not yet asked again: each is
 // asked once, and never twice.
 func (i *Inventory) RequestsToAskAgain(ctx context.Context) ([]BuildRequest, error) {
@@ -127,12 +167,13 @@ func (i *Inventory) RequestsToAskAgain(ctx context.Context) ([]BuildRequest, err
 	if err != nil || !on {
 		return nil, err
 	}
-	all, err := c.Builds.All(ctx)
+	all, err := requestRecords(ctx, c)
 	if err != nil {
 		return nil, err
 	}
 	var out []BuildRequest
-	for id, k := range all {
+	for _, k := range all {
+		id := strings.TrimPrefix(k.Key, chain.RequestPrefix+".")
 		if k.Value.State == chain.BuildAsked && k.Value.Request != nil && k.Value.Request.Reask == chain.ReaskPending {
 			out = append(out, fromRequest(id, *k.Value.Request))
 		}
@@ -148,12 +189,12 @@ func (i *Inventory) ClaimToAskAgain(ctx context.Context, id string) (bool, error
 	if err != nil || !on {
 		return false, err
 	}
-	cur, found, err := c.Builds.Get(ctx, chainBuildKey(id))
+	cur, found, err := c.Builds.Get(ctx, chain.RequestKey(id))
 	if err != nil || !found || cur.Value.Request == nil || cur.Value.Request.Reask != chain.ReaskPending {
 		return false, err
 	}
 	cur.Value.Request.Reask = chain.ReaskClaimed
-	if _, err := c.Builds.Update(ctx, chainBuildKey(id), cur.Value, cur.Revision); err != nil {
+	if _, err := c.Builds.Update(ctx, chain.RequestKey(id), cur.Value, cur.Revision); err != nil {
 		if errors.Is(err, chain.ErrMoved) {
 			return false, nil
 		}