package apply import ( "context" "errors" "path/filepath" "strings" "testing" "github.com/novox/mesh-host/internal/declaration" "github.com/novox/mesh-host/internal/store" ) // A run-once container is a step, not a service (novox/hq ADR 0052): the host runs it to // completion, requires exit 0, records that it ran, and — because a failed step gates the apply — // starts whatever the declaration places after it only once the step has finished. These tests // defend that, each named for the claim it holds up. // nameOf returns the value after --name in a docker run argument list. func nameOf(args []string) string { for i, a := range args { if a == "--name" && i+1 < len(args) { return args[i+1] } } return "" } func TestARunOnceStepIsRunToCompletionNotLeftRunning(t *testing.T) { // The difference between a step and a service is that a step is run in the foreground and its // exit code is the answer. So the host must not pass --detach (which returns before the // container exits) nor --restart (a step that is restarted is not a step). var runArgs []string run := func(ctx context.Context, name string, args ...string) (string, error) { switch args[0] { case "info": return "27.0\n", nil case "run": runArgs = args return "", nil // ran and exited 0 case "rm": return "", nil } return "", nil } d := parseTrusted(t, `{"declaration":1,"resources":[ {"id":"seed","type":"container","name":"seed","image":"`+pinned+`","run-once":true} ]}`) report, state, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, run, nil, nil) if err != nil { t.Fatalf("a run-once step that exited 0 failed the apply: %v", err) } if runArgs == nil { t.Fatal("the run-once step was never run") } if got := strings.Join(runArgs, " "); strings.Contains(got, "--detach") { t.Errorf("a run-once step was detached, so its exit could not be observed: %s", got) } if got := strings.Join(runArgs, " "); strings.Contains(got, "--restart") { t.Errorf("a run-once step was given a restart policy, which makes it a service: %s", got) } if report.Outcomes[0].Action != "created" { t.Errorf("a completed run-once step was not reported created: %+v", report.Outcomes[0]) } // It ran, so it was recorded — and the record is the digest of the declaration, which is what // keeps a re-apply from running it again. applied, ok := state.Find("seed") if !ok { t.Fatal("a completed run-once step was not recorded") } if applied.Wrote != containerSpec(d.Resources[0].(*declaration.Container), inputs{}) { t.Errorf("the run-once record is not the declaration's digest: %q", applied.Wrote) } } func TestARunOnceStepThatExitsNonZeroFailsTheApply(t *testing.T) { // Run in the foreground, a non-zero exit is an error the runtime hands back. That must fail // the apply, not be swallowed — a seed that did not happen leaves the machine unready. run := func(ctx context.Context, name string, args ...string) (string, error) { switch args[0] { case "info": return "27.0\n", nil case "run": return "", errors.New("exit status 1") case "rm": return "", nil } return "", nil } d := parseTrusted(t, `{"declaration":1,"resources":[ {"id":"seed","type":"container","name":"seed","image":"`+pinned+`","run-once":true} ]}`) _, state, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, run, nil, nil) if err == nil { t.Fatal("a run-once step that did not complete was accepted") } if !strings.Contains(err.Error(), "did not complete") { t.Errorf("failed for the wrong reason: %v", err) } if _, recorded := state.Find("seed"); recorded { t.Error("a run-once step that did not complete was recorded as done") } } func TestAFailedRunOnceStepGatesWhatFollows(t *testing.T) { // The whole of how "before the broker starts" is enforced: the step is declared first, and a // step that did not complete stops the apply reaching the container that depends on it — the // mirror of a failed action stopping what follows. var startedNames []string run := func(ctx context.Context, name string, args ...string) (string, error) { switch args[0] { case "info": return "27.0\n", nil case "container": return "false\t\n", errors.New("no such container") case "run": startedNames = append(startedNames, nameOf(args)) if nameOf(args) == "seed" { return "", errors.New("exit status 1") // the step fails } return "deadbeef\n", nil case "rm": return "", nil } return "", nil } d := parseTrusted(t, `{"declaration":1,"resources":[ {"id":"seed","type":"container","name":"seed","image":"`+pinned+`","run-once":true}, {"id":"broker","type":"container","name":"broker","image":"`+pinned+`"} ]}`) _, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, run, nil, nil) if err == nil { t.Fatal("a failed run-once step did not fail the apply") } var applyErr *Error if !errors.As(err, &applyErr) { t.Fatalf("got %T", err) } if !applyErr.Gated { t.Error("the failure does not say that nothing after the step was attempted") } for _, n := range startedNames { if n == "broker" { t.Fatal("the broker was started even though its run-once step did not complete") } } } func TestARunOnceStepAlreadyCompletedIsNotReRun(t *testing.T) { // Its record of having happened is the digest of its declaration. A re-apply that finds the // digest already recorded does nothing — a step is not reconciled toward, it is run once. d := parseTrusted(t, `{"declaration":1,"resources":[ {"id":"seed","type":"container","name":"seed","image":"`+pinned+`","run-once":true} ]}`) want := containerSpec(d.Resources[0].(*declaration.Container), inputs{}) var ran bool run := func(ctx context.Context, name string, args ...string) (string, error) { switch args[0] { case "info": return "27.0\n", nil case "run": ran = true return "", nil case "rm": return "", nil } return "", nil } known := store.State{} known.Record(store.Applied{ID: "seed", Type: "container", Origin: store.OriginCarried, Target: "seed", Wrote: want}) report, _, err := Apply(context.Background(), archHost(t), d, known, store.OriginCarried, run, nil, nil) if err != nil { t.Fatalf("re-applying a completed run-once step failed: %v", err) } if ran { t.Error("a run-once step already completed for this declaration was run again") } if report.Changed() { t.Errorf("a re-applied run-once step reported a change: %+v", report.Outcomes) } } func TestARunOnceStepReRunsWhenItsDeclarationChanged(t *testing.T) { // The marker is the declaration's digest, so a changed image or environment moves it and the // step runs again — a migration or a seed that changed is a different step. d := parseTrusted(t, `{"declaration":1,"resources":[ {"id":"seed","type":"container","name":"seed","image":"`+pinned+`","run-once":true,"env":{"CLIENT":"new-admin"}} ]}`) var ran bool run := func(ctx context.Context, name string, args ...string) (string, error) { switch args[0] { case "info": return "27.0\n", nil case "run": ran = true return "", nil case "rm": return "", nil } return "", nil } // A record from an earlier, different declaration of the same step. known := store.State{} known.Record(store.Applied{ID: "seed", Type: "container", Origin: store.OriginCarried, Target: "seed", Wrote: "an-older-digest"}) if _, _, err := Apply(context.Background(), archHost(t), d, known, store.OriginCarried, run, nil, nil); err != nil { t.Fatalf("apply failed: %v", err) } if !ran { t.Error("a run-once step whose declaration changed was not run again") } } func TestARunOnceStepRunsAgainWhenWhatItReadsChanged(t *testing.T) { // A step that fetches a fact from a provider names the binding file it reads. What a // container reads is part of its digest, so when the provider moved and the mesh rewrote the // file, the step's marker no longer matches and it runs again (novox/hq ADR 0099) — the same // rule that recreates a running container, read as "again" for a step. dir := t.TempDir() env := filepath.Join(dir, "acme.env") d := parseTrusted(t, `{"declaration":1,"resources":[ {"id":"env","type":"file","path":"`+env+`","content":"ACME_ROOTS=https://10.0.0.2/roots.pem\n"}, {"id":"trust","type":"container","name":"trust","image":"`+pinned+`","run-once":true,"restart-on":["env"]} ]}`) // The record from when the provider was elsewhere: the step's digest against the old file. was := map[string]string{"env": declaredDigest(&declaration.File{Content: "ACME_ROOTS=https://10.0.0.1/roots.pem\n"})} known := store.State{} known.Record(store.Applied{ID: "trust", Type: "container", Origin: store.OriginCarried, Target: "trust", Wrote: containerSpec(d.Resources[1].(*declaration.Container), inputs{declares: was})}) var ran bool run := func(ctx context.Context, name string, args ...string) (string, error) { switch args[0] { case "info": return "27.0\n", nil case "run": ran = true return "", nil } return "", nil } report, _, err := Apply(context.Background(), archHost(t), d, known, store.OriginCarried, run, nil, nil) if err != nil { t.Fatalf("apply failed: %v", err) } if !ran { t.Fatal("a run-once step whose named file changed was not run again") } if report.Outcomes[1].Action != "created" { t.Errorf("the re-run step was not reported as having run: %+v", report.Outcomes[1]) } // And with the file unchanged, the step stays done: "again" is when something changed. ran = false now := map[string]string{"env": declaredDigest(d.Resources[0].(*declaration.File))} settled := store.State{} settled.Record(store.Applied{ID: "env", Type: "file", Origin: store.OriginCarried, Target: env, Wrote: now["env"]}) settled.Record(store.Applied{ID: "trust", Type: "container", Origin: store.OriginCarried, Target: "trust", Wrote: containerSpec(d.Resources[1].(*declaration.Container), inputs{declares: now})}) if _, _, err := Apply(context.Background(), archHost(t), d, settled, store.OriginCarried, run, nil, nil); err != nil { t.Fatalf("re-apply failed: %v", err) } if ran { t.Error("a run-once step whose named file did not change was run again") } } func TestAContainerNamingARunOnceStepIsRecreatedWhenItRan(t *testing.T) { // The service that consumes what a step made names the step: when the step ran this pass — // fetched a new root — the running container holds the old one, and its spec did not move, // so restart-on is what brings it back with the new one (novox/hq ADR 0099). d := parseTrusted(t, `{"declaration":1,"resources":[ {"id":"trust","type":"container","name":"trust","image":"`+pinned+`","run-once":true}, {"id":"server","type":"container","name":"server","image":"`+pinned+`","restart-on":["trust"]} ]}`) declares := map[string]string{"trust": declaredDigest(d.Resources[0].(*declaration.Container))} spec := containerSpec(d.Resources[1].(*declaration.Container), inputs{declares: declares}) var removed, created bool run := func(ctx context.Context, name string, args ...string) (string, error) { switch args[0] { case "info": return "27.0\n", nil case "container": // The server is up, made from exactly this spec — nothing but the step's run says // it must be replaced. return "true\t" + spec, nil case "rm": if len(args) > 0 && args[len(args)-1] == "server" { removed = true } return "", nil case "run": if args[len(args)-1] != "trust" && strings.Contains(strings.Join(args, " "), "--name server") { created = true } return "deadbeef\n", nil } return "", nil } report, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, run, nil, nil) if err != nil { t.Fatalf("apply failed: %v", err) } if !removed || !created { t.Fatalf("the container naming the step was not recreated after the step ran (removed=%v created=%v)", removed, created) } server := report.Outcomes[len(report.Outcomes)-1] if server.Action != "updated" || !strings.Contains(server.Detail, "trust") { t.Errorf("the recreation did not name the step as its reason: %+v", server) } } func TestAFailedStepGatesItsModuleAndNotTheMachine(t *testing.T) { // **The blast radius of a step is its module** (novox/hq ADR 0136). A step exists to make // something true before the next thing in its own module needs it — a store seeded before the // broker starts, a schema prepared before the version that needs it runs. Stopping the whole // apply is what this host's own loop calls holding a machine hostage, and it was already // rejected for every other shape (04-ISSUES/011): a module whose database is briefly // unreachable must not stop every module declared after it. var startedNames []string run := func(ctx context.Context, name string, args ...string) (string, error) { switch args[0] { case "info": return "27.0\n", nil case "container": return "false\t\n", errors.New("no such container") case "run": startedNames = append(startedNames, nameOf(args)) if nameOf(args) == "catalogue-prepare" { return "", errors.New("exit status 1") // the schema could not be reached } return "deadbeef\n", nil case "rm": return "", nil } return "", nil } d := parseTrusted(t, `{"declaration":1,"resources":[ {"id":"mesh-catalog.runtime.prepare","type":"container","name":"catalogue-prepare","image":"`+pinned+`","run-once":true}, {"id":"mesh-catalog.runtime","type":"container","name":"catalogue","image":"`+pinned+`"}, {"id":"gitea.server","type":"container","name":"forge","image":"`+pinned+`"} ]}`) report, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, run, nil, nil) if err == nil { t.Fatal("a failed step was not reported as a failure") } started := map[string]bool{} for _, n := range startedNames { started[n] = true } if started["catalogue"] { t.Error("the module's own workload ran although its step did not complete") } if !started["forge"] { t.Error("another module was not attempted, so one module's step held the machine hostage") } // And the machine's own account says which was not attempted, rather than leaving it to be // inferred from silence. var skipped string for _, o := range report.Outcomes { if o.Action == "skipped" { skipped = o.ID } } if skipped != "mesh-catalog.runtime" { t.Errorf("the report does not say what was not attempted: %q", skipped) } }