mesh-host/internal/apply/runonce_test.go

package apply

import (
	"context"
	"errors"
	"path/filepath"
	"strings"
	"testing"

	"github.com/novox/mesh-host/internal/declaration"
	"github.com/novox/mesh-host/internal/store"
)

// A run-once container is a step, not a service (novox/hq ADR 0052): the host runs it to
// completion, requires exit 0, records that it ran, and — because a failed step gates the apply —
// starts whatever the declaration places after it only once the step has finished. These tests
// defend that, each named for the claim it holds up.

// nameOf returns the value after --name in a docker run argument list.
func nameOf(args []string) string {
	for i, a := range args {
		if a == "--name" && i+1 < len(args) {
			return args[i+1]
		}
	}
	return ""
}

func TestARunOnceStepIsRunToCompletionNotLeftRunning(t *testing.T) {
	// The difference between a step and a service is that a step is run in the foreground and its
	// exit code is the answer. So the host must not pass --detach (which returns before the
	// container exits) nor --restart (a step that is restarted is not a step).
	var runArgs []string
	run := func(ctx context.Context, name string, args ...string) (string, error) {
		switch args[0] {
		case "info":
			return "27.0\n", nil
		case "run":
			runArgs = args
			return "", nil // ran and exited 0
		case "rm":
			return "", nil
		}
		return "", nil
	}
	d := parseTrusted(t, `{"declaration":1,"resources":[
	  {"id":"seed","type":"container","name":"seed","image":"`+pinned+`","run-once":true}
	]}`)

	report, state, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, run, nil, nil)
	if err != nil {
		t.Fatalf("a run-once step that exited 0 failed the apply: %v", err)
	}
	if runArgs == nil {
		t.Fatal("the run-once step was never run")
	}
	if got := strings.Join(runArgs, " "); strings.Contains(got, "--detach") {
		t.Errorf("a run-once step was detached, so its exit could not be observed: %s", got)
	}
	if got := strings.Join(runArgs, " "); strings.Contains(got, "--restart") {
		t.Errorf("a run-once step was given a restart policy, which makes it a service: %s", got)
	}
	if report.Outcomes[0].Action != "created" {
		t.Errorf("a completed run-once step was not reported created: %+v", report.Outcomes[0])
	}
	// It ran, so it was recorded — and the record is the digest of the declaration, which is what
	// keeps a re-apply from running it again.
	applied, ok := state.Find("seed")
	if !ok {
		t.Fatal("a completed run-once step was not recorded")
	}
	if applied.Wrote != containerSpec(d.Resources[0].(*declaration.Container), inputs{}) {
		t.Errorf("the run-once record is not the declaration's digest: %q", applied.Wrote)
	}
}

func TestARunOnceStepThatExitsNonZeroFailsTheApply(t *testing.T) {
	// Run in the foreground, a non-zero exit is an error the runtime hands back. That must fail
	// the apply, not be swallowed — a seed that did not happen leaves the machine unready.
	run := func(ctx context.Context, name string, args ...string) (string, error) {
		switch args[0] {
		case "info":
			return "27.0\n", nil
		case "run":
			return "", errors.New("exit status 1")
		case "rm":
			return "", nil
		}
		return "", nil
	}
	d := parseTrusted(t, `{"declaration":1,"resources":[
	  {"id":"seed","type":"container","name":"seed","image":"`+pinned+`","run-once":true}
	]}`)

	_, state, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, run, nil, nil)
	if err == nil {
		t.Fatal("a run-once step that did not complete was accepted")
	}
	if !strings.Contains(err.Error(), "did not complete") {
		t.Errorf("failed for the wrong reason: %v", err)
	}
	if _, recorded := state.Find("seed"); recorded {
		t.Error("a run-once step that did not complete was recorded as done")
	}
}

func TestAFailedRunOnceStepGatesWhatFollows(t *testing.T) {
	// The whole of how "before the broker starts" is enforced: the step is declared first, and a
	// step that did not complete stops the apply reaching the container that depends on it — the
	// mirror of a failed action stopping what follows.
	var startedNames []string
	run := func(ctx context.Context, name string, args ...string) (string, error) {
		switch args[0] {
		case "info":
			return "27.0\n", nil
		case "container":
			return "false\t\n", errors.New("no such container")
		case "run":
			startedNames = append(startedNames, nameOf(args))
			if nameOf(args) == "seed" {
				return "", errors.New("exit status 1") // the step fails
			}
			return "deadbeef\n", nil
		case "rm":
			return "", nil
		}
		return "", nil
	}
	d := parseTrusted(t, `{"declaration":1,"resources":[
	  {"id":"seed","type":"container","name":"seed","image":"`+pinned+`","run-once":true},
	  {"id":"broker","type":"container","name":"broker","image":"`+pinned+`"}
	]}`)

	_, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, run, nil, nil)
	if err == nil {
		t.Fatal("a failed run-once step did not fail the apply")
	}
	var applyErr *Error
	if !errors.As(err, &applyErr) {
		t.Fatalf("got %T", err)
	}
	if !applyErr.Gated {
		t.Error("the failure does not say that nothing after the step was attempted")
	}
	for _, n := range startedNames {
		if n == "broker" {
			t.Fatal("the broker was started even though its run-once step did not complete")
		}
	}
}

func TestARunOnceStepAlreadyCompletedIsNotReRun(t *testing.T) {
	// Its record of having happened is the digest of its declaration. A re-apply that finds the
	// digest already recorded does nothing — a step is not reconciled toward, it is run once.
	d := parseTrusted(t, `{"declaration":1,"resources":[
	  {"id":"seed","type":"container","name":"seed","image":"`+pinned+`","run-once":true}
	]}`)
	want := containerSpec(d.Resources[0].(*declaration.Container), inputs{})

	var ran bool
	run := func(ctx context.Context, name string, args ...string) (string, error) {
		switch args[0] {
		case "info":
			return "27.0\n", nil
		case "run":
			ran = true
			return "", nil
		case "rm":
			return "", nil
		}
		return "", nil
	}
	known := store.State{}
	known.Record(store.Applied{ID: "seed", Type: "container", Origin: store.OriginCarried, Target: "seed", Wrote: want})

	report, _, err := Apply(context.Background(), archHost(t), d, known, store.OriginCarried, run, nil, nil)
	if err != nil {
		t.Fatalf("re-applying a completed run-once step failed: %v", err)
	}
	if ran {
		t.Error("a run-once step already completed for this declaration was run again")
	}
	if report.Changed() {
		t.Errorf("a re-applied run-once step reported a change: %+v", report.Outcomes)
	}
}

func TestARunOnceStepReRunsWhenItsDeclarationChanged(t *testing.T) {
	// The marker is the declaration's digest, so a changed image or environment moves it and the
	// step runs again — a migration or a seed that changed is a different step.
	d := parseTrusted(t, `{"declaration":1,"resources":[
	  {"id":"seed","type":"container","name":"seed","image":"`+pinned+`","run-once":true,"env":{"CLIENT":"new-admin"}}
	]}`)

	var ran bool
	run := func(ctx context.Context, name string, args ...string) (string, error) {
		switch args[0] {
		case "info":
			return "27.0\n", nil
		case "run":
			ran = true
			return "", nil
		case "rm":
			return "", nil
		}
		return "", nil
	}
	// A record from an earlier, different declaration of the same step.
	known := store.State{}
	known.Record(store.Applied{ID: "seed", Type: "container", Origin: store.OriginCarried, Target: "seed", Wrote: "an-older-digest"})

	if _, _, err := Apply(context.Background(), archHost(t), d, known, store.OriginCarried, run, nil, nil); err != nil {
		t.Fatalf("apply failed: %v", err)
	}
	if !ran {
		t.Error("a run-once step whose declaration changed was not run again")
	}
}

func TestARunOnceStepRunsAgainWhenWhatItReadsChanged(t *testing.T) {
	// A step that fetches a fact from a provider names the binding file it reads. What a
	// container reads is part of its digest, so when the provider moved and the mesh rewrote the
	// file, the step's marker no longer matches and it runs again (novox/hq ADR 0099) — the same
	// rule that recreates a running container, read as "again" for a step.
	dir := t.TempDir()
	env := filepath.Join(dir, "acme.env")
	d := parseTrusted(t, `{"declaration":1,"resources":[
	  {"id":"env","type":"file","path":"`+env+`","content":"ACME_ROOTS=https://10.0.0.2/roots.pem\n"},
	  {"id":"trust","type":"container","name":"trust","image":"`+pinned+`","run-once":true,"restart-on":["env"]}
	]}`)
	// The record from when the provider was elsewhere: the step's digest against the old file.
	was := map[string]string{"env": declaredDigest(&declaration.File{Content: "ACME_ROOTS=https://10.0.0.1/roots.pem\n"})}
	known := store.State{}
	known.Record(store.Applied{ID: "trust", Type: "container", Origin: store.OriginCarried, Target: "trust",
		Wrote: containerSpec(d.Resources[1].(*declaration.Container), inputs{declares: was})})

	var ran bool
	run := func(ctx context.Context, name string, args ...string) (string, error) {
		switch args[0] {
		case "info":
			return "27.0\n", nil
		case "run":
			ran = true
			return "", nil
		}
		return "", nil
	}
	report, _, err := Apply(context.Background(), archHost(t), d, known, store.OriginCarried, run, nil, nil)
	if err != nil {
		t.Fatalf("apply failed: %v", err)
	}
	if !ran {
		t.Fatal("a run-once step whose named file changed was not run again")
	}
	if report.Outcomes[1].Action != "created" {
		t.Errorf("the re-run step was not reported as having run: %+v", report.Outcomes[1])
	}

	// And with the file unchanged, the step stays done: "again" is when something changed.
	ran = false
	now := map[string]string{"env": declaredDigest(d.Resources[0].(*declaration.File))}
	settled := store.State{}
	settled.Record(store.Applied{ID: "env", Type: "file", Origin: store.OriginCarried, Target: env, Wrote: now["env"]})
	settled.Record(store.Applied{ID: "trust", Type: "container", Origin: store.OriginCarried, Target: "trust",
		Wrote: containerSpec(d.Resources[1].(*declaration.Container), inputs{declares: now})})
	if _, _, err := Apply(context.Background(), archHost(t), d, settled, store.OriginCarried, run, nil, nil); err != nil {
		t.Fatalf("re-apply failed: %v", err)
	}
	if ran {
		t.Error("a run-once step whose named file did not change was run again")
	}
}

func TestAContainerNamingARunOnceStepIsRecreatedWhenItRan(t *testing.T) {
	// The service that consumes what a step made names the step: when the step ran this pass —
	// fetched a new root — the running container holds the old one, and its spec did not move,
	// so restart-on is what brings it back with the new one (novox/hq ADR 0099).
	d := parseTrusted(t, `{"declaration":1,"resources":[
	  {"id":"trust","type":"container","name":"trust","image":"`+pinned+`","run-once":true},
	  {"id":"server","type":"container","name":"server","image":"`+pinned+`","restart-on":["trust"]}
	]}`)
	declares := map[string]string{"trust": declaredDigest(d.Resources[0].(*declaration.Container))}
	spec := containerSpec(d.Resources[1].(*declaration.Container), inputs{declares: declares})

	var removed, created bool
	run := func(ctx context.Context, name string, args ...string) (string, error) {
		switch args[0] {
		case "info":
			return "27.0\n", nil
		case "container":
			// The server is up, made from exactly this spec — nothing but the step's run says
			// it must be replaced.
			return "true\t" + spec, nil
		case "rm":
			if len(args) > 0 && args[len(args)-1] == "server" {
				removed = true
			}
			return "", nil
		case "run":
			if args[len(args)-1] != "trust" && strings.Contains(strings.Join(args, " "), "--name server") {
				created = true
			}
			return "deadbeef\n", nil
		}
		return "", nil
	}
	report, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, run, nil, nil)
	if err != nil {
		t.Fatalf("apply failed: %v", err)
	}
	if !removed || !created {
		t.Fatalf("the container naming the step was not recreated after the step ran (removed=%v created=%v)", removed, created)
	}
	server := report.Outcomes[len(report.Outcomes)-1]
	if server.Action != "updated" || !strings.Contains(server.Detail, "trust") {
		t.Errorf("the recreation did not name the step as its reason: %+v", server)
	}
}

func TestAFailedStepGatesItsModuleAndNotTheMachine(t *testing.T) {
	// **The blast radius of a step is its module** (novox/hq ADR 0136). A step exists to make
	// something true before the next thing in its own module needs it — a store seeded before the
	// broker starts, a schema prepared before the version that needs it runs. Stopping the whole
	// apply is what this host's own loop calls holding a machine hostage, and it was already
	// rejected for every other shape (04-ISSUES/011): a module whose database is briefly
	// unreachable must not stop every module declared after it.
	var startedNames []string
	run := func(ctx context.Context, name string, args ...string) (string, error) {
		switch args[0] {
		case "info":
			return "27.0\n", nil
		case "container":
			return "false\t\n", errors.New("no such container")
		case "run":
			startedNames = append(startedNames, nameOf(args))
			if nameOf(args) == "catalogue-prepare" {
				return "", errors.New("exit status 1") // the schema could not be reached
			}
			return "deadbeef\n", nil
		case "rm":
			return "", nil
		}
		return "", nil
	}
	d := parseTrusted(t, `{"declaration":1,"resources":[
	  {"id":"mesh-catalog.runtime-prepare","type":"container","name":"catalogue-prepare","image":"`+pinned+`","run-once":true},
	  {"id":"mesh-catalog.runtime","type":"container","name":"catalogue","image":"`+pinned+`"},
	  {"id":"gitea.server","type":"container","name":"forge","image":"`+pinned+`"}
	]}`)

	report, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, run, nil, nil)
	if err == nil {
		t.Fatal("a failed step was not reported as a failure")
	}
	started := map[string]bool{}
	for _, n := range startedNames {
		started[n] = true
	}
	if started["catalogue"] {
		t.Error("the module's own workload ran although its step did not complete")
	}
	if !started["forge"] {
		t.Error("another module was not attempted, so one module's step held the machine hostage")
	}
	// And the machine's own account says which was not attempted, rather than leaving it to be
	// inferred from silence.
	var skipped string
	for _, o := range report.Outcomes {
		if o.Action == "skipped" {
			skipped = o.ID
		}
	}
	if skipped != "mesh-catalog.runtime" {
		t.Errorf("the report does not say what was not attempted: %q", skipped)
	}
}

func TestAResourcesOwnerIsReadToTheLastDot(t *testing.T) {
	// **A module's name may contain a dot.** `novox.be` is one on this mesh, so reading a resource's
	// owner to the first dot would make its resources belong to something called "novox" — and a gate
	// would skip whatever else happened to start that way. A resource's own id never contains one,
	// which is what makes the last dot the boundary.
	for identity, want := range map[string]string{
		"novox.be.server":              "novox.be",
		"mesh-catalog.runtime-prepare": "mesh-catalog",
		"gitea.admin-bootstrap":        "gitea",
	} {
		got, ours := moduleOf(identity)
		if !ours || got != want {
			t.Errorf("%q belongs to %q (%v), want %q", identity, got, ours, want)
		}
	}
	// What the mesh declares in its own right belongs to no module: the foundation's resources carry
	// no dot, and the adoption's are the mesh's.
	for _, identity := range []string{"container-runtime", "store-ready", "adoption.guard", ".server"} {
		if _, ours := moduleOf(identity); ours {
			t.Errorf("%q was read as a module's", identity)
		}
	}
}