push --behind, and a builder told where to publish

`status` says which machines are not doing what they were told, and
nothing acted on it: a machine that refused or failed stayed wrong until
somebody ran push again naming it.

`push --behind` sends only to machines whose last report was not a clean
apply. A command rather than a timer, deliberately: a scheduler is then a
scheduler over this, where building the scheduler first would have meant
two paths to one act with nothing to compare them against.

Naming a machine and asking which machines need one are different
requests, so `push <node> --behind` is refused rather than guessed. With
nothing behind it says so, because "nothing needed one" and "this did not
run" must never look the same. A machine failing the same way for six
hours is pushed to anyway and said about — refusing would leave no way to
retry after fixing the cause, and this is a command somebody ran.

Proven in the lab: a machine is broken with a package that does not
exist, `push --behind` names it and not the machine that is fine, the
module is corrected, and the machine recovers without anybody naming it.

And the builder can be told where to publish rather than configured. A
builder that is a module requires an artifact store, and the mesh writes
it the same binding any consumer of any provision gets. A binding with no
address is refused rather than falling back to anything — that would
publish to a store on the wrong machine and be found out much later. The
variable remains for a builder run by a person, which is how it is still
run while being developed.

cmd/mesh-builder/main.go

@@ -47,7 +47,8 @@ It consumes build requests and answers with what it made. Nothing is listened on
 is dialled except the broker.
 
   MESH_BROKER_AMQP   where the broker is, with this builder's own credential
-  MESH_REGISTRY      host:port to publish artifacts to
+  MESH_REGISTRY      host:port to publish artifacts to, when the mesh has not said
+  MESH_BINDING       a file the mesh wrote saying where the artifact store is
   MESH_WORKSPACE     where to clone and build (default: a temporary directory)
 `
 
@@ -67,10 +68,9 @@ func run() error {
 	if amqpURL == "" {
 		return fmt.Errorf("no MESH_BROKER_AMQP: a builder with no broker has nothing to build")
 	}
-	registry := strings.TrimSpace(os.Getenv("MESH_REGISTRY"))
+	registry, err := whereToPublish()
-	if registry == "" {
+	if err != nil {
-		return fmt.Errorf(
+		return err
-			"no MESH_REGISTRY: a built artifact nobody can fetch is not built")
 	}
 	workspace := os.Getenv("MESH_WORKSPACE")
 	if workspace == "" {
@@ -212,3 +212,51 @@ func short(commit string) string {
 	}
 	return commit
 }
+
+// whereToPublish is the artifact store this builder uses.
+//
+// **Preferably from the mesh.** A builder that is a module requires an artifact store, and the
+// mesh writes it a file saying which machine answers that and on what port — the same binding any
+// consumer of any provision gets. Reading it means the address is not a setting somebody keeps in
+// step by hand, and moving the store is an ordinary reassignment rather than an edit on every
+// build machine.
+//
+// The environment variable remains for a builder run by a person, which is how this started and
+// how it is still run while being developed.
+func whereToPublish() (string, error) {
+	binding := strings.TrimSpace(os.Getenv("MESH_BINDING"))
+	if binding == "" {
+		registry := strings.TrimSpace(os.Getenv("MESH_REGISTRY"))
+		if registry == "" {
+			return "", fmt.Errorf("neither MESH_BINDING nor MESH_REGISTRY: a built artifact " +
+				"nobody can fetch is not built")
+		}
+		return registry, nil
+	}
+
+	raw, err := os.ReadFile(binding)
+	if err != nil {
+		return "", fmt.Errorf("cannot read what the mesh said about the artifact store: %w", err)
+	}
+	var told struct {
+		From   string         `json:"from"`
+		At     string         `json:"at"`
+		Serves map[string]any `json:"serves"`
+	}
+	if err := json.Unmarshal(raw, &told); err != nil {
+		return "", fmt.Errorf("%s is not a binding: %w", binding, err)
+	}
+	if told.At == "" {
+		// The provider is not on the private network, so there is no name to reach it by. Said
+		// rather than falling back to the machine's own name, which would publish to a store on
+		// the wrong machine and be found out much later.
+		return "", fmt.Errorf(
+			"%s says the artifact store is on %q and gives no address for it", binding, told.From)
+	}
+	port, ok := told.Serves["port"]
+	if !ok {
+		return "", fmt.Errorf("%s says nothing about which port the artifact store answers on",
+			binding)
+	}
+	return fmt.Sprintf("%s:%v", told.At, port), nil
+}

cmd/mesh-builder/where_test.go

@@ -0,0 +1,76 @@
+package main
+
+import (
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+)
+
+// Where a builder publishes.
+//
+// Preferably from the mesh: a builder that is a module requires an artifact store, and the mesh
+// writes it a binding saying which machine answers and on what port. Reading it means the address
+// is not a setting somebody keeps in step by hand.
+
+func binding(t *testing.T, body string) string {
+	t.Helper()
+	path := filepath.Join(t.TempDir(), "artifact-store.json")
+	if err := os.WriteFile(path, []byte(body), 0o644); err != nil {
+		t.Fatal(err)
+	}
+	return path
+}
+
+func TestTheMeshSaysWhereToPublish(t *testing.T) {
+	t.Setenv("MESH_BINDING", binding(t, `{"binding":1,"provision":"artifact-store",
+		"from":"anchor","at":"anchor.internal","serves":{"port":5000,"scheme":"http"}}`))
+	where, err := whereToPublish()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if where != "anchor.internal:5000" {
+		t.Fatalf("got %q", where)
+	}
+}
+
+func TestABindingWithNoAddressIsRefused(t *testing.T) {
+	// The provider is not on the private network, so there is no name to reach it by. Falling
+	// back to anything would publish to a store on the wrong machine and be found out much later.
+	t.Setenv("MESH_BINDING", binding(t, `{"binding":1,"from":"anchor","serves":{"port":5000}}`))
+	_, err := whereToPublish()
+	if err == nil {
+		t.Fatal("a binding with nowhere to reach was accepted")
+	}
+	if !strings.Contains(err.Error(), "anchor") {
+		t.Fatalf("the failure does not name the machine: %v", err)
+	}
+}
+
+func TestABindingWithNoPortIsRefused(t *testing.T) {
+	t.Setenv("MESH_BINDING", binding(t, `{"binding":1,"from":"a","at":"a.internal","serves":{}}`))
+	if _, err := whereToPublish(); err == nil {
+		t.Fatal("a binding saying nothing about a port was accepted")
+	}
+}
+
+func TestTheVariableStillWorksForABuilderRunByAPerson(t *testing.T) {
+	// Which is how this started and how it is still run while being developed.
+	t.Setenv("MESH_BINDING", "")
+	t.Setenv("MESH_REGISTRY", "127.0.0.1:5000")
+	where, err := whereToPublish()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if where != "127.0.0.1:5000" {
+		t.Fatalf("got %q", where)
+	}
+}
+
+func TestNeitherIsRefusedRatherThanGuessed(t *testing.T) {
+	t.Setenv("MESH_BINDING", "")
+	t.Setenv("MESH_REGISTRY", "")
+	if _, err := whereToPublish(); err == nil {
+		t.Fatal("a builder with nowhere to publish reported somewhere")
+	}
+}

cmd/mesh-control/main.go

@@ -145,7 +145,7 @@ func usage() {
   pin <node> <provision> <from>  which node this one gets a provision from
   unpin <node> <provision>       put that question back
   plan <node> [--files|--json]     what that node would run, and why
-  push [<node>]                  send a node everything it should be
+  push [<node>] [--behind]       send a node everything it should be, or only those that need it
   version                        what this binary is
 
 Each context reaches its own store through its own credential (novox/hq ADR 0008), named
@@ -1406,8 +1406,28 @@ func planCommand(ctx context.Context, args []string) error {
 // half-configured for as long as that lasts — and the two are computed from the same picture of
 // the mesh, so sending them apart would let them disagree.
 func pushCommand(ctx context.Context, args []string) error {
+	set := flag.NewFlagSet("push", flag.ContinueOnError)
+	// Only the machines that need it.
+	//
+	// **A command rather than a timer, to begin with.** Something that re-pushes on a schedule is
+	// a scheduler over this, and building the scheduler first would mean two paths to one act
+	// with nothing to compare them against. A person can run this; so can cron; so can whatever
+	// eventually watches.
+	behind := set.Bool("behind", false,
+		"only machines whose last declaration was refused or partly failed")
+	positionals, err := parseAround(set, args)
+	if err != nil {
+		return err
+	}
+	args = positionals
 	if len(args) > 1 {
-		return errors.New("push [<node>] — one node, or all of them")
+		return errors.New("push [<node>] [--behind] — one node, or all of them")
+	}
+	if len(args) == 1 && *behind {
+		// Naming a machine and asking for the ones that need it are two different requests, and
+		// guessing which was meant would sometimes push to a machine somebody did not name.
+		return errors.New("push <node> or push --behind, not both: one names a machine and the " +
+			"other asks which machines need one")
 	}
 	inv, err := openInventory(ctx)
 	if err != nil {
@@ -1428,6 +1448,25 @@ func pushCommand(ctx context.Context, args []string) error {
 	if err != nil {
 		return err
 	}
+
+	// Which machines are not in the state they were sent, when that is what was asked for.
+	var needsOne map[string]inventory.Doing
+	if *behind {
+		wrong, err := inv.NotDoingWhatTheyWereTold(ctx)
+		if err != nil {
+			return err
+		}
+		needsOne = map[string]inventory.Doing{}
+		for _, d := range wrong {
+			needsOne[d.Node] = d
+		}
+		if len(needsOne) == 0 {
+			// Said rather than doing nothing quietly. "Nothing needed one" and "this did not run"
+			// must never look the same.
+			fmt.Println("every machine is doing what it was told")
+			return nil
+		}
+	}
 	gens, err := generators(ctx, inv)
 	if err != nil {
 		return err
@@ -1453,6 +1492,20 @@ func pushCommand(ctx context.Context, args []string) error {
 		if len(args) == 1 && n.Name != args[0] {
 			continue
 		}
+		if *behind {
+			doing, needs := needsOne[n.Name]
+			if !needs {
+				continue
+			}
+			// A machine that has been failing the same way for a long time is not going to stop
+			// because it was asked again. Said, and pushed to anyway — refusing would leave no
+			// way to retry after fixing the cause, and this is a command somebody ran.
+			if since := time.Since(doing.At); since > 6*time.Hour {
+				fmt.Printf("%s has been %s since %s; pushing again anyway, but the cause is "+
+					"unlikely to be timing\n",
+					n.Name, doing.Outcome, doing.At.Local().Format("2006-01-02 15:04"))
+			}
+		}
 		plan, settings, err := planFor(ctx, inv, n.Name)
 		if err != nil {
 			refusals = append(refusals, fmt.Sprintf("%s:\n%v", n.Name, err))