Hold every refused module of a first merge and ask each again by its own path (issue 1173)

A held merge was asked again by the rule of a first merge, which finds nothing once a sibling module of the
same merge registered, so it stayed held forever; and a second refusal in a merge already held fell through to
a failed build. The hold now keeps a list of module, path and word, a refusal in a held waiting merge joins it,
and the delivery of build-agent asks each held path again on the merge's Approve warrant.

cmd/mesh-controller/first_builds.go

@@ -103,7 +103,7 @@ func (f following) resumeFirstBuilds(ctx context.Context, d deliveryMerge) error
 	}
 	for _, w := range waiting {
 		k := announcedOf(w)
-		if !k.FirstBuild || k.Held != nil || !strings.EqualFold(k.Owner+"/"+k.Repo, d.Owner+"/"+d.Repo) || k.Number != d.Number ||
+		if !k.FirstBuild || len(k.Held) > 0 || !strings.EqualFold(k.Owner+"/"+k.Repo, d.Owner+"/"+d.Repo) || k.Number != d.Number ||
 			k.HeadSHA != d.HeadSHA {
 			continue
 		}
@@ -177,18 +177,55 @@ func firstBuildsOf(ctx context.Context, inv catalogueReader, m link.SourceMoved,
 // and its merge is kept waiting for a build-agent that reads it (novox/hq ADR 0482 §5, issue 1173).
 type heldFirstBuild struct {
 	Module string    `json:"module"`
+	Path   string    `json:"path"`
 	Word   string    `json:"word"`
 	At     time.Time `json:"at"`
 }
 
-// judgeRefusedFirstBuild is the hold a first build's outcome puts its merge under: nil unless the build failed on a word
-// of its manifest the running build-agent does not know.
-func judgeRefusedFirstBuild(result link.BuildResult, req inventory.BuildRequest, merge inventory.BatchedMerge) *heldFirstBuild {
+// judgeRefusedFirstBuild is the hold a first build's outcome puts its merge under: every module the merge holds already,
+// and this one, unless it is among them; nil unless the build failed on a word of its manifest the running build-agent
+// does not know, and was asked for a first merge, answered by the record of merges no plan answers or put back to waiting
+// by an earlier hold (a second refusal in one merge is held too, review of issue 1173).
+func judgeRefusedFirstBuild(result link.BuildResult, req inventory.BuildRequest, merge inventory.BatchedMerge) []heldFirstBuild {
 	word := refusedWord(result.Failed)
-	if word == "" || req.For != "merge" || merge.Plan != inventory.WaitingRecord {
+	if word == "" || req.For != "merge" {
 		return nil
 	}
-	return &heldFirstBuild{Module: req.Name(), Word: word, At: time.Now().UTC()}
+	var held []heldFirstBuild
+	switch k := announcedOf(merge); {
+	case merge.Plan == inventory.WaitingRecord:
+		// Answered when its builds were asked: a hold from before (a build asked again) is over, and starts anew.
+	case merge.Plan == "" && len(k.Held) > 0:
+		held = k.Held
+	default:
+		return nil
+	}
+	this := heldFirstBuild{Module: req.Name(), Path: strings.Trim(req.Path, "/"), Word: word, At: time.Now().UTC()}
+	for i, h := range held {
+		if strings.Trim(h.Path, "/") == this.Path {
+			out := append([]heldFirstBuild(nil), held...)
+			out[i] = this
+			return out
+		}
+	}
+	return append(append([]heldFirstBuild(nil), held...), this)
+}
+
+// heldDirsOf is the directories a held merge asks again: its held modules' own, each once, "." for the repository's root.
+func heldDirsOf(k keptMerge) []string {
+	var dirs []string
+	seen := map[string]bool{}
+	for _, h := range k.Held {
+		dir := strings.Trim(h.Path, "/")
+		if dir == "" {
+			dir = "."
+		}
+		if !seen[dir] {
+			seen[dir] = true
+			dirs = append(dirs, dir)
+		}
+	}
+	return dirs
 }
 
 // unknownWord is how the strict reading of a manifest says a word is not one it knows: encoding/json's own words, which
@@ -219,7 +256,7 @@ func holdRefusedFirstBuild(ctx context.Context, inv *inventory.Inventory, result
 		return false, err
 	}
 	held := judgeRefusedFirstBuild(result, req, merge)
-	if held == nil {
+	if len(held) == 0 {
 		return false, nil
 	}
 	k := announcedOf(merge)
@@ -231,9 +268,15 @@ func holdRefusedFirstBuild(ctx context.Context, inv *inventory.Inventory, result
 	if err := inv.HoldMerge(ctx, merge.Repository, merge.Commit, event); err != nil {
 		return false, err
 	}
+	this := held[len(held)-1]
+	for _, h := range held {
+		if h.Path == strings.Trim(req.Path, "/") {
+			this = h
+		}
+	}
 	fmt.Printf("%s: the first build of %s was refused for the word %q, which the running build-agent does not read; the "+
-		"merge %.8s of %s is kept waiting for a build-agent that reads it, and the build is asked again when one is delivered\n",
-		result.ID, held.Module, held.Word, merge.Commit, merge.Repository)
+		"merge %.8s of %s is kept waiting for a build-agent that reads it (holding %s), and the build is asked again when "+
+		"one is delivered\n", result.ID, this.Module, this.Word, merge.Commit, merge.Repository, heldNames(held))
 	return true, nil
 }
 
@@ -257,7 +300,7 @@ func firstBuildObservations(waiting []inventory.BatchedMerge, now time.Time) []c
 	var out []conditions.Observation
 	for _, w := range waiting {
 		k := announcedOf(w)
-		if k.Held != nil {
+		if len(k.Held) > 0 {
 			out = append(out, heldFirstBuildObservation(k))
 			continue
 		}
@@ -283,22 +326,34 @@ func firstBuildObservations(waiting []inventory.BatchedMerge, now time.Time) []c
 	return out
 }
 
-// heldFirstBuildObservation names a first merge held for a build-agent that reads its manifest: at once, since nothing
-// is lost and nothing times out, but the operator can see what the merge waits for.
+// heldNames is the modules a merge holds, each with the word it waits for: "watchman (key-ttl), view (per-machine)".
+func heldNames(held []heldFirstBuild) string {
+	names := make([]string, 0, len(held))
+	for _, h := range held {
+		names = append(names, fmt.Sprintf("%s (%q)", h.Module, h.Word))
+	}
+	return strings.Join(names, ", ")
+}
+
+// heldFirstBuildObservation names a first merge held for a build-agent that reads its manifests: at once, since nothing
+// is lost and nothing times out, but the operator can see what the merge waits for. One condition names every module held.
 func heldFirstBuildObservation(k keptMerge) conditions.Observation {
 	repository := k.Owner + "/" + k.Repo
+	modules := make([]string, 0, len(k.Held))
+	for _, h := range k.Held {
+		modules = append(modules, h.Module)
+	}
+	held := heldNames(k.Held)
 	return conditions.Observation{Scope: conditions.ScopeMerge, ID: fmt.Sprintf("%s_%.8s", repository, k.Commit),
 		Kind: kindFirstBuildHeld, Severity: conditions.Warning,
-		Summary: fmt.Sprintf("the merge %.8s of %s adds the module %s, whose manifest uses the word %q: the running "+
-			"build-agent does not read it, so the merge waits for a build-agent that reads it", k.Commit, repository,
-			k.Held.Module, k.Held.Word),
-		Said: fmt.Sprintf("%s at %s waits for a build-agent that reads %q (module %s)", repository, k.Commit, k.Held.Word,
-			k.Held.Module),
-		Headline: fmt.Sprintf("The first build of %s waits for a newer build-agent", k.Held.Module),
-		Explanation: fmt.Sprintf("The merge %.8s into %s adds the module %s, and its manifest uses the word %q, which "+
-			"the build-agent now running does not know. The merge is kept, not failed: the first build is asked again "+
-			"when a delivery of build-agent ends.", k.Commit, repository, k.Held.Module, k.Held.Word),
-		Resolved: "a delivery of build-agent ends and the first build registers the module, or a hand `build` does",
+		Summary: fmt.Sprintf("the merge %.8s of %s adds modules whose manifests use words the running build-agent does not "+
+			"read: %s; the merge waits for a build-agent that reads them", k.Commit, repository, held),
+		Said:     fmt.Sprintf("%s at %s waits for a build-agent that reads the words of %s", repository, k.Commit, held),
+		Headline: fmt.Sprintf("The first build of %s waits for a newer build-agent", strings.Join(modules, ", ")),
+		Explanation: fmt.Sprintf("The merge %.8s into %s adds modules whose manifests use words the build-agent now "+
+			"running does not know: %s. The merge is kept, not failed: their first builds are asked again when a delivery "+
+			"of build-agent ends.", k.Commit, repository, held),
+		Resolved: "a delivery of build-agent ends and the first builds register the modules, or a hand `build` does",
 	}
 }
 
@@ -329,29 +384,32 @@ func askHeldFirstBuildsAgain(ctx context.Context, open *stores, p inventory.Plan
 	asked := 0
 	for _, w := range waiting {
 		k := announcedOf(w)
-		if k.Held == nil {
+		if len(k.Held) == 0 {
 			continue
 		}
-		entries, err := inv.Catalogued(ctx)
+		// Its own held modules, not the rule of a first merge: a module of the merge that passed is registered by now,
+		// and the repository no longer counts as one the mesh builds nothing from (review of issue 1173).
+		why, err := approvedAnnounced(ctx, inv, k.Owner, k.Repo, keptMerge{SourceMoved: k.SourceMoved})
 		if err != nil {
-			fmt.Printf("%s delivered build-agent, and the catalogue could not be read: %v\n", p.ID, err)
-			return asked
-		}
-		read, err := readForPlanning(ctx, inv)
-		if err != nil {
-			fmt.Printf("%s delivered build-agent, and what the builds read could not be read: %v\n", p.ID, err)
-			return asked
-		}
-		added, why, err := firstBuildsOf(ctx, inv, k.SourceMoved, entries, read)
-		if err != nil || len(added) == 0 {
 			fmt.Printf("%s delivered build-agent, and the merge %.8s of %s held for it asks no build now (%v): it stays "+
 				"waiting\n", p.ID, w.Commit, w.Repository, err)
 			continue
 		}
-		fmt.Printf("%s delivered build-agent: the first build of %s, held for the word %q, is asked again\n", p.ID,
-			k.Held.Module, k.Held.Word)
-		if err := askFirstBuilds(ctx, inv, k.SourceMoved, time.Now().UTC(), added, why); err != nil {
-			fmt.Printf("  and could not be asked: %v\n", err)
+		dirs := heldDirsOf(k)
+		fmt.Printf("%s delivered build-agent: the first builds of %s are asked again\n", p.ID, heldNames(k.Held))
+		paths := make([]string, 0, len(dirs))
+		for _, dir := range dirs {
+			if dir == "." {
+				dir = ""
+			}
+			paths = append(paths, dir)
+		}
+		repository := k.Owner + "/" + k.Repo
+		askBuilds(withFirstBuildWarrant(ctx, repository, paths, why), inv, k.SourceMoved,
+			buildSource{Repository: repository, Seat: gitSeat}, dirs)
+		// Answered: it no longer waits, so this end asks nothing twice; a refusal that comes again holds it anew.
+		if err := inv.AnswerMerge(ctx, w.Repository, w.Commit, inventory.WaitingRecord, false); err != nil {
+			fmt.Printf("  and the merge could not be marked answered: %v\n", err)
 			continue
 		}
 		asked++