Hand a replaced resource over to the process that replaces it (hq issue 213)

The controller moves from a container to a process on the one machine
that runs it (novox/hq issue 213). Every orphan is removed before anything
is applied, so the container would go first and nothing would answer the
mesh's verbs while the process was fetched, unpacked and started — and
never again, if it did not start.

- a process may say what it `replaces`: resources the declaration no
  longer declares. Such an orphan is kept through the up-front sweep and
  removed right after the process applied and is up: active and running
  at two looks ten seconds apart, the same main process, no restart in
  between (stricter than ADR 0184's second look, which reads a unit
  waiting to restart as running). If the process failed, was skipped
  behind its module's step, or is not running, the orphan stays running
  and recorded, reported kept, and the next apply hands it over.
  Refused: naming something still declared, itself, an empty id, one
  thing named by two processes, and `replaces` on a step or a schedule.
- beyond #85's oneshot unit for a step: a step written ./name runs its
  own bundle's binary (tested), and is started, never enabled.
- a run-once process that fails gates its module, as a run-once
  container already did, so a version whose preparation failed is not
  started.
- an unchanged run-once process is not run again, and an unchanged
  scheduled one is kept up by its timer: both were "a daemon that had
  stopped" and were started on every apply.

internal/apply/handover_test.go

@@ -213,23 +213,18 @@ func TestWithoutReplacesAnOrphanStillGoesFirst(t *testing.T) {
 	}
 }
 
-// novox/hq issue 213: a step is run by its unit — as the process it prepares for is — so it has
-// that process's user, directory and environment, and `./name` is its own bundle's binary. It used
-// to be run directly by the host, as root, in the host's directory, with no environment.
-func TestAStepIsRunByItsUnitWithItsUserAndEnvironment(t *testing.T) {
+// novox/hq issue 213, beyond what the oneshot unit (process_step_test.go) already holds: a step
+// written `./name` runs its own bundle's binary — the controller's preparation is its own binary —
+// and a step is started, never enabled.
+func TestAStepRunsItsOwnBundlesBinaryAndIsNotEnabled(t *testing.T) {
 	onAMachine(t)
 	body, digest := anArchive(t, map[string]string{"mesh-controller": "#!/bin/sh\n"})
 	m := &aMachine{running: true}
 	d := declare(t, `{"id":"mesh-controller.controller-prepare","type":"process","name":"mesh-controller-prepare",
-	  "source":"`+serving(t, body)+`","digest":"`+digest+`","run":["./mesh-controller","prepare"],"run-once":true,
-	  "env":{"MESH_STORE_INVENTORY_FILE":"/var/lib/mesh/x/inventory"}}`)
-	report, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginDeclared, m.run, nil, nil)
-	if err != nil {
+	  "source":"`+serving(t, body)+`","digest":"`+digest+`","run":["./mesh-controller","prepare"],"run-once":true}`)
+	if _, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginDeclared, m.run, nil, nil); err != nil {
 		t.Fatalf("the step failed: %v", err)
 	}
-	if m.index("systemctl start mesh-controller-prepare.service") < 0 {
-		t.Fatalf("the step was not started through its unit: %v", m.commands)
-	}
 	for _, c := range m.commands {
 		if strings.HasPrefix(c, "./") || strings.HasPrefix(c, "systemctl enable") {
 			t.Errorf("the step was run directly or enabled: %v", m.commands)
@@ -239,22 +234,9 @@ func TestAStepIsRunByItsUnitWithItsUserAndEnvironment(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	for _, want := range []string{
-		"Type=oneshot",
-		"ExecStart=" + filepath.Join(daemonRoot, "mesh-controller-prepare", "mesh-controller") + " prepare",
-		`Environment="MESH_STORE_INVENTORY_FILE=/var/lib/mesh/x/inventory"`,
-	} {
-		if !strings.Contains(string(unit), want) {
-			t.Errorf("the step's unit does not say %q:\n%s", want, unit)
-		}
-	}
-	for _, not := range []string{"Restart=always", "WantedBy="} {
-		if strings.Contains(string(unit), not) {
-			t.Errorf("the step's unit says %q, which makes it a service:\n%s", not, unit)
-		}
-	}
-	if o := outcomeOf(report, "mesh-controller.controller-prepare"); o.Action != "created" {
-		t.Errorf("the step's outcome is %+v", o)
+	want := "ExecStart=" + filepath.Join(daemonRoot, "mesh-controller-prepare", "mesh-controller") + " prepare"
+	if !strings.Contains(string(unit), want) {
+		t.Errorf("the step does not run its own bundle's binary (%q):\n%s", want, unit)
 	}
 }
 

internal/apply/process.go

@@ -123,15 +123,14 @@ func applyProcess(ctx context.Context, r *declaration.Process, run Runner,
 
 	// **A step is run to completion, not installed.** What follows it is gated on it finishing,
 	// so the machine is not asked to start something that needed a migration that did not happen.
-	// The record that it ran is the digest, which is why the identity above includes the command.
+	// Nothing is left behind to ask afterwards: the record that it ran is the digest, which is why
+	// the identity above includes the command.
 	//
-	// **Run by its unit, as the process it prepares for is run** (novox/hq issue 213). It used to be
-	// run directly by the host: as root, in the host's own directory, with none of its environment —
-	// so a step reading its store's connection from a file its environment names found no variable,
-	// `./name` was looked for beside the host rather than in the bundle, and a step that must not be
-	// root was. A oneshot unit carries the same user, directory and environment as a daemon's, and
-	// starting one waits for it to exit and fails when it did not exit cleanly. It is neither enabled
-	// nor left running; its output is in the journal under its own name.
+	// **Run as the unit a daemon would be, once** (novox/hq design 38 WP4c). Run directly, the
+	// step started in the host's own working directory, without its environment, its environment
+	// files or its user — `node bootstrap/index.js` resolved from wherever the host ran and was told
+	// none of the words it was declared with. A oneshot unit carries all four exactly as a daemon's
+	// does, and starting one waits for it to finish and fails when it fails.
 	if r.RunOnce {
 		unit := filepath.Join(unitDir, r.Name+".service")
 		if err := os.WriteFile(unit, []byte(unitFor(r)), 0o644); err != nil {
@@ -141,7 +140,7 @@ func applyProcess(ctx context.Context, r *declaration.Process, run Runner,
 			return out, err
 		}
 		if _, err := run(ctx, "systemctl", "start", r.Name+".service"); err != nil {
-			return out, fmt.Errorf("the %s step did not complete: %w", r.Name, err)
+			return out, fmt.Errorf("the %s step did not complete (journalctl -u %s.service says why): %w", r.Name, r.Name, err)
 		}
 		out.Action = "created"
 		if previous.Wrote != "" {
@@ -240,9 +239,8 @@ func unitFor(r *declaration.Process) string {
 	}
 	fmt.Fprintf(&b, "ExecStart=%s\n", strings.Join(runFrom(r), " "))
 	if r.Schedule != "" || r.RunOnce {
-		// Started by its timer, or by the host as a step, and expected to finish. Restarting it
-		// would have it run continuously, which is the opposite of either; and a step is not
-		// installed, so it is not wanted by anything at boot.
+		// Started by its timer, or once by the host, and expected to finish. Restarting it would have it run
+		// continuously between fires, which is the opposite of a schedule.
 		b.WriteString("Type=oneshot\n")
 		b.WriteString("\n")
 		return strings.Replace(b.String(), "Type=simple\n", "", 1)

internal/apply/process_step_test.go

@@ -0,0 +1,81 @@
+package apply
+
+import (
+	"context"
+	"errors"
+	"os"
+	"os/user"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/novox/mesh-host/internal/store"
+)
+
+// A run-once process is a step run where, how and as whom it was declared (novox/hq design 38
+// WP4c): its bundle's directory, its environment and environment files, its user. Run directly,
+// it started in the host's own directory with none of them.
+func TestARunOnceProcessRunsAsItsOneshotUnit(t *testing.T) {
+	units, bundles := t.TempDir(), t.TempDir()
+	wasUnits, wasBundles := unitDir, daemonRoot
+	unitDir, daemonRoot = units, bundles
+	t.Cleanup(func() { unitDir, daemonRoot = wasUnits, wasBundles })
+
+	me, err := user.Current()
+	if err != nil {
+		t.Fatal(err)
+	}
+	body, digest := anArchive(t, map[string]string{"bootstrap/index.js": "console.log(1)\n"})
+	var commands []string
+	fail := false
+	run := func(ctx context.Context, name string, args ...string) (string, error) {
+		commands = append(commands, name+" "+strings.Join(args, " "))
+		if fail && name == "systemctl" && len(args) > 0 && args[0] == "start" {
+			return "", errors.New("exit status 1")
+		}
+		return "", nil
+	}
+	d := declare(t, `{"id":"mosquitto.bootstrap","type":"process","name":"mosquitto-bootstrap","source":"`+serving(t, body)+
+		`","digest":"`+digest+`","run":["node","bootstrap/index.js"],"run-once":true,"user":"`+me.Username+`",`+
+		`"env":{"MESH_ADMIN":"mesh-admin"},"env-file":["/var/lib/mesh/mosquitto/bootstrap.env"]}`)
+
+	if _, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginDeclared, run, nil, nil); err != nil {
+		t.Fatal(err)
+	}
+	unit, err := os.ReadFile(filepath.Join(units, "mosquitto-bootstrap.service"))
+	if err != nil {
+		t.Fatalf("no unit was written for the step: %v", err)
+	}
+	for _, want := range []string{
+		"WorkingDirectory=" + filepath.Join(bundles, "mosquitto-bootstrap"),
+		"EnvironmentFile=/var/lib/mesh/mosquitto/bootstrap.env",
+		`Environment="MESH_ADMIN=mesh-admin"`,
+		"User=" + me.Username,
+		"Type=oneshot",
+		"ExecStart=node bootstrap/index.js",
+	} {
+		if !strings.Contains(string(unit), want) {
+			t.Errorf("the step's unit lacks %q:\n%s", want, unit)
+		}
+	}
+	for _, never := range []string{"Restart=always", "[Install]", "Type=simple"} {
+		if strings.Contains(string(unit), never) {
+			t.Errorf("a step's unit says %q:\n%s", never, unit)
+		}
+	}
+	joined := strings.Join(commands, "; ")
+	if !strings.Contains(joined, "systemctl start mosquitto-bootstrap.service") {
+		t.Errorf("the step was not started as its unit: %s", joined)
+	}
+	if strings.Contains(joined, "node bootstrap/index.js") || strings.Contains(joined, "enable mosquitto-bootstrap") {
+		t.Errorf("the step was run directly or enabled: %s", joined)
+	}
+
+	// A step that fails fails the apply, and is not recorded as done.
+	fail = true
+	d2 := declare(t, `{"id":"mosquitto.bootstrap","type":"process","name":"mosquitto-bootstrap","source":"`+serving(t, body)+
+		`","digest":"`+digest+`","run":["node","bootstrap/index.js"],"run-once":true,"env":{"MESH_ADMIN":"changed"}}`)
+	if _, _, err := Apply(context.Background(), archHost(t), d2, store.State{}, store.OriginDeclared, run, nil, nil); err == nil {
+		t.Error("a step that failed did not fail the apply")
+	}
+}