Build everything behind its source, in one command

novox/hq ADR 0010 replaced a pipeline with a comparison, and named the risk:
losing the question "did my change go out?". The mesh could already answer
which modules are behind their source — and then a person read that list and
retyped each repository, which is a person being the loop, and the loop is the
thing the pipeline was doing before it was taken away.

The mirror of `push --behind`, with the same argument and the same refusal to
combine the two forms: naming a repository and asking which need building are
different requests.

One failing does not stop the others, for the same reason one broken module no
longer blocks a machine's whole declaration: a mesh where one bad repository
holds back nine good ones is a mesh where nobody dares add the tenth.

Each is built from its own recorded ref rather than the commit the mesh
happened to notice — pinning to that would quietly turn a tracked branch into
a pin.

cmd/mesh-control/main.go

@@ -146,6 +146,7 @@ func usage() {
   settings set <module> <file> --node <n>   ...or for one machine
   settings clear <module> [--node <n>]      take a layer away
   build <repository> [--ref R]   have a build machine build it, and record what came out
+  build --behind                 build every module the mesh holds older than its source
   builds [<module>]              what has been built lately, and what came of it
   builder issue <name>           a broker account for a build machine, scoped to build work
   licence add|list|use|key       model access, under the name a person calls it
@@ -942,7 +943,7 @@ func moduleCommand(ctx context.Context, args []string) error {
 			}
 		}
 		if stale > 0 {
-			fmt.Printf("\n%d module(s) behind their source — `build <repository>` to catch up\n", stale)
+			fmt.Printf("\n%d module(s) behind their source — `build --behind` to catch up\n", stale)
 		}
 		return nil
 
@@ -2043,95 +2044,32 @@ func buildCommand(ctx context.Context, args []string) error {
 	ref := set.String("ref", "", "the branch, tag or commit to build")
 	wait := set.Duration("wait", 10*time.Minute, "how long to wait for a builder to answer")
 	dryRun := set.Bool("dry-run", false, "build and print the manifest, recording nothing")
+	// Every module whose source has moved, rather than one named repository.
+	//
+	// **The mirror of `push --behind`, and the same argument** (novox/hq ADR 0010): the mesh
+	// already knows which modules are behind their source, so making a person read that list and
+	// retype each repository is asking them to be the loop. Naming a repository and asking which
+	// ones need building are different requests, so they are not combined.
+	behind := set.Bool("behind", false, "every module the mesh holds older than its source has")
 	positionals, err := parseAround(set, args)
 	if err != nil {
 		return err
 	}
+	if *behind {
+		if len(positionals) != 0 {
+			return errors.New("build <repository> or build --behind, not both: one names a " +
+				"repository and the other asks which need building")
+		}
+		return buildBehind(ctx, *wait)
+	}
 	if len(positionals) != 1 {
 		return errors.New("build <repository> [--ref R] [--wait D] [--dry-run]")
 	}
 
-	ident, err := openIdentity(ctx)
-	if err != nil {
-		return err
-	}
-	defer ident.Close()
-
-	server, err := link.Connect(nil, nil)
-	if err != nil {
-		return err
-	}
-	defer server.Close()
-
-	// Correlated by something the control plane makes, not by the module's name: two builds of one
-	// module can be in flight, and the second answer is not the first one's.
-	request := link.BuildRequest{
-		ID:         fmt.Sprintf("%s-%d", "build", time.Now().UnixNano()),
-		Repository: positionals[0],
-		Ref:        *ref,
-	}
-	fmt.Printf("asked for %s", request.Repository)
-	if *ref != "" {
-		fmt.Printf(" at %s", *ref)
-	}
-	fmt.Println()
-
-	result, err := link.RequestBuild(ctx, server.Channel(), request, *wait)
-	if err != nil {
-		return err
-	}
-
-	// Kept before it is judged. A failed build that leaves no trace is indistinguishable from one
-	// nobody asked for, and the difference is the whole of whether somebody should be looking at
-	// something.
-	inv, err := openInventory(ctx)
-	if err != nil {
-		return err
-	}
-	defer inv.Close()
-	if err := inv.RecordBuild(ctx, buildFrom(result)); err != nil {
-		return err
-	}
-
-	if result.Failed != "" {
-		// The builder's own words. Wrapping them in something about the control plane would put
-		// two explanations between a person and a build log.
-		return fmt.Errorf("%s could not build %s:\n%s", result.On, result.Repository, result.Failed)
-	}
-
-	for _, made := range result.Made {
-		fmt.Printf("  %-12s %s %s\n", made.Name, made.Kind, made.Reference)
-	}
-
-	// Parsed with the same parser a hand-written manifest goes through. A second path would be a
-	// second thing to disagree about what a manifest is.
-	manifest, err := catalogue.ParseManifest(result.Manifest)
-	if err != nil {
-		return fmt.Errorf("%s built %s and what came back is not a manifest: %w",
-			result.On, result.Repository, err)
-	}
-
 	if *dryRun {
-		body, err := json.MarshalIndent(manifest, "", "  ")
+		return buildAndShow(ctx, positionals[0], *ref, *wait)
-		if err != nil {
-			return err
-		}
-		fmt.Println(string(body))
-		return nil
 	}
-
+	return buildOne(ctx, positionals[0], *ref, *wait)
-	// Recorded with where it came from, so "is this current?" is answerable without building it
-	// again (novox/hq ADR 0009).
-	if err := inv.RegisterModule(ctx, manifest, inventory.Source{
-		Repository: result.Repository, Ref: result.Ref,
-		BuiltFrom: result.Commit, Head: result.Commit,
-	}); err != nil {
-		return err
-	}
-	fmt.Printf("\n%s %s, built on %s from %s\n",
-		manifest.Module, manifest.Version, result.On, short(result.Commit))
-	fmt.Printf("  run `assign <node> %s` to put it somewhere\n", manifest.Module)
-	return nil
 }
 
 // buildFrom turns what a builder said into what the mesh keeps.
@@ -2416,3 +2354,178 @@ func keyFor(ctx context.Context, licence, node, module string) (string, error) {
 	defer held.Close()
 	return held.KeyFor(ctx, licence, node, module)
 }
+
+// buildBehind builds every module the mesh holds older than its source has.
+//
+// **This is the loop novox/hq ADR 0010 replaced a pipeline with, closed.** The mesh already
+// records where each module came from and what its source last had; until this, a person read
+// that list and retyped each repository — which is a person being the loop, and the thing a
+// pipeline was doing before it was taken away.
+//
+// Each is built and recorded on its own. **One failing does not stop the others**, for the same
+// reason one broken module no longer blocks a machine's whole declaration: a mesh where one bad
+// repository holds back nine good ones is a mesh where nobody dares add the tenth.
+func buildBehind(ctx context.Context, wait time.Duration) error {
+	inv, err := openInventory(ctx)
+	if err != nil {
+		return err
+	}
+	defer inv.Close()
+
+	held, err := inv.Catalogued(ctx)
+	if err != nil {
+		return err
+	}
+	var stale []inventory.Entry
+	for _, e := range held {
+		if !e.Source.Current() {
+			stale = append(stale, e)
+		}
+	}
+	if len(stale) == 0 {
+		// Said rather than doing nothing quietly: "nothing needed building" and "this did not
+		// run" must never look the same.
+		fmt.Println("every module the mesh holds is what its source last had")
+		return nil
+	}
+
+	fmt.Printf("%d module(s) behind their source:\n", len(stale))
+	for _, e := range stale {
+		fmt.Printf("  %s  %s < %s\n",
+			e.Manifest.Module, short(e.Source.BuiltFrom), short(e.Source.Head))
+	}
+	fmt.Println()
+
+	var failed []string
+	for _, e := range stale {
+		fmt.Printf("--- %s\n", e.Manifest.Module)
+		// Its own recorded ref, not its head commit: a module tracking a branch should be built
+		// from that branch, and pinning to the commit the mesh happened to notice would quietly
+		// turn a tracked branch into a pin.
+		if err := buildOne(ctx, e.Source.Repository, e.Source.Ref, wait); err != nil {
+			fmt.Printf("  %v\n", err)
+			failed = append(failed, e.Manifest.Module)
+		}
+	}
+
+	if len(failed) > 0 {
+		return fmt.Errorf("%d of %d could not be built: %s",
+			len(failed), len(stale), strings.Join(failed, ", "))
+	}
+	fmt.Printf("\n%d module(s) built. `push --behind` sends them to the machines running them\n",
+		len(stale))
+	return nil
+}
+
+// buildOne asks a build machine for one repository and records everything that came back.
+//
+// Separated from the command so `--behind` can walk a list without a second path to the same act.
+func buildOne(ctx context.Context, repository, ref string, wait time.Duration) error {
+	ident, err := openIdentity(ctx)
+	if err != nil {
+		return err
+	}
+	defer ident.Close()
+
+	server, err := link.Connect(nil, nil)
+	if err != nil {
+		return err
+	}
+	defer server.Close()
+
+	// Correlated by something the control plane makes, not by the module's name: two builds of one
+	// module can be in flight, and the second answer is not the first one's.
+	request := link.BuildRequest{
+		ID:         fmt.Sprintf("%s-%d", "build", time.Now().UnixNano()),
+		Repository: repository,
+		Ref:        ref,
+	}
+	fmt.Printf("asked for %s", request.Repository)
+	if ref != "" {
+		fmt.Printf(" at %s", ref)
+	}
+	fmt.Println()
+
+	result, err := link.RequestBuild(ctx, server.Channel(), request, wait)
+	if err != nil {
+		return err
+	}
+
+	// Kept before it is judged. A failed build that leaves no trace is indistinguishable from one
+	// nobody asked for, and the difference is the whole of whether somebody should be looking at
+	// something.
+	inv, err := openInventory(ctx)
+	if err != nil {
+		return err
+	}
+	defer inv.Close()
+	if err := inv.RecordBuild(ctx, buildFrom(result)); err != nil {
+		return err
+	}
+
+	if result.Failed != "" {
+		// The builder's own words. Wrapping them in something about the control plane would put
+		// two explanations between a person and a build log.
+		return fmt.Errorf("%s could not build %s:\n%s", result.On, result.Repository, result.Failed)
+	}
+
+	for _, made := range result.Made {
+		fmt.Printf("  %-12s %s %s\n", made.Name, made.Kind, made.Reference)
+	}
+
+	// Parsed with the same parser a hand-written manifest goes through. A second path would be a
+	// second thing to disagree about what a manifest is.
+	manifest, err := catalogue.ParseManifest(result.Manifest)
+	if err != nil {
+		return fmt.Errorf("%s built %s and what came back is not a manifest: %w",
+			result.On, result.Repository, err)
+	}
+
+	// Recorded with where it came from, so "is this current?" is answerable without building it
+	// again (novox/hq ADR 0009).
+	if err := inv.RegisterModule(ctx, manifest, inventory.Source{
+		Repository: result.Repository, Ref: result.Ref,
+		BuiltFrom: result.Commit, Head: result.Commit,
+	}); err != nil {
+		return err
+	}
+	fmt.Printf("\n%s %s, built on %s from %s\n",
+		manifest.Module, manifest.Version, result.On, short(result.Commit))
+	fmt.Printf("  run `assign <node> %s` to put it somewhere\n", manifest.Module)
+	return nil
+}
+
+// buildAndShow builds and prints the manifest without recording anything.
+func buildAndShow(ctx context.Context, repository, ref string, wait time.Duration) error {
+	ident, err := openIdentity(ctx)
+	if err != nil {
+		return err
+	}
+	defer ident.Close()
+	server, err := link.Connect(nil, nil)
+	if err != nil {
+		return err
+	}
+	defer server.Close()
+
+	result, err := link.RequestBuild(ctx, server.Channel(), link.BuildRequest{
+		ID: fmt.Sprintf("%s-%d", "build", time.Now().UnixNano()), Repository: repository, Ref: ref,
+	}, wait)
+	if err != nil {
+		return err
+	}
+	if result.Failed != "" {
+		return fmt.Errorf("%s could not build %s:\n%s", result.On, result.Repository, result.Failed)
+	}
+	manifest, err := catalogue.ParseManifest(result.Manifest)
+	if err != nil {
+		return fmt.Errorf("%s built %s and what came back is not a manifest: %w",
+			result.On, result.Repository, err)
+	}
+	body, err := json.MarshalIndent(manifest, "", "  ")
+	if err != nil {
+		return err
+	}
+	fmt.Println(string(body))
+	return nil
+}