jschoubben
a9c49724b5
A run-once container that does not complete stops the rest of that module's resources; everything else on the machine is attempted, as every other shape already is (novox/hq ADR 0136). An action still gates the machine — genesis is a row of them and they belong to no module. What was not attempted is reported as skipped, because that and 'nothing to do' are different answers. Without this, ADR 0135's derived preparation would let one module's unreachable database hold a machine hostage — the fault 04-ISSUES/011 removed for everything else, and the reason the catalogue migrates itself at start.
377 lines
14 KiB
Go
377 lines
14 KiB
Go
package apply
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"path/filepath"
|
|
"strings"
|
|
"testing"
|
|
|
|
"github.com/novox/mesh-host/internal/declaration"
|
|
"github.com/novox/mesh-host/internal/store"
|
|
)
|
|
|
|
// A run-once container is a step, not a service (novox/hq ADR 0052): the host runs it to
|
|
// completion, requires exit 0, records that it ran, and — because a failed step gates the apply —
|
|
// starts whatever the declaration places after it only once the step has finished. These tests
|
|
// defend that, each named for the claim it holds up.
|
|
|
|
// nameOf returns the value after --name in a docker run argument list.
|
|
func nameOf(args []string) string {
|
|
for i, a := range args {
|
|
if a == "--name" && i+1 < len(args) {
|
|
return args[i+1]
|
|
}
|
|
}
|
|
return ""
|
|
}
|
|
|
|
func TestARunOnceStepIsRunToCompletionNotLeftRunning(t *testing.T) {
|
|
// The difference between a step and a service is that a step is run in the foreground and its
|
|
// exit code is the answer. So the host must not pass --detach (which returns before the
|
|
// container exits) nor --restart (a step that is restarted is not a step).
|
|
var runArgs []string
|
|
run := func(ctx context.Context, name string, args ...string) (string, error) {
|
|
switch args[0] {
|
|
case "info":
|
|
return "27.0\n", nil
|
|
case "run":
|
|
runArgs = args
|
|
return "", nil // ran and exited 0
|
|
case "rm":
|
|
return "", nil
|
|
}
|
|
return "", nil
|
|
}
|
|
d := parseTrusted(t, `{"declaration":1,"resources":[
|
|
{"id":"seed","type":"container","name":"seed","image":"`+pinned+`","run-once":true}
|
|
]}`)
|
|
|
|
report, state, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, run, nil, nil)
|
|
if err != nil {
|
|
t.Fatalf("a run-once step that exited 0 failed the apply: %v", err)
|
|
}
|
|
if runArgs == nil {
|
|
t.Fatal("the run-once step was never run")
|
|
}
|
|
if got := strings.Join(runArgs, " "); strings.Contains(got, "--detach") {
|
|
t.Errorf("a run-once step was detached, so its exit could not be observed: %s", got)
|
|
}
|
|
if got := strings.Join(runArgs, " "); strings.Contains(got, "--restart") {
|
|
t.Errorf("a run-once step was given a restart policy, which makes it a service: %s", got)
|
|
}
|
|
if report.Outcomes[0].Action != "created" {
|
|
t.Errorf("a completed run-once step was not reported created: %+v", report.Outcomes[0])
|
|
}
|
|
// It ran, so it was recorded — and the record is the digest of the declaration, which is what
|
|
// keeps a re-apply from running it again.
|
|
applied, ok := state.Find("seed")
|
|
if !ok {
|
|
t.Fatal("a completed run-once step was not recorded")
|
|
}
|
|
if applied.Wrote != containerSpec(d.Resources[0].(*declaration.Container), inputs{}) {
|
|
t.Errorf("the run-once record is not the declaration's digest: %q", applied.Wrote)
|
|
}
|
|
}
|
|
|
|
func TestARunOnceStepThatExitsNonZeroFailsTheApply(t *testing.T) {
|
|
// Run in the foreground, a non-zero exit is an error the runtime hands back. That must fail
|
|
// the apply, not be swallowed — a seed that did not happen leaves the machine unready.
|
|
run := func(ctx context.Context, name string, args ...string) (string, error) {
|
|
switch args[0] {
|
|
case "info":
|
|
return "27.0\n", nil
|
|
case "run":
|
|
return "", errors.New("exit status 1")
|
|
case "rm":
|
|
return "", nil
|
|
}
|
|
return "", nil
|
|
}
|
|
d := parseTrusted(t, `{"declaration":1,"resources":[
|
|
{"id":"seed","type":"container","name":"seed","image":"`+pinned+`","run-once":true}
|
|
]}`)
|
|
|
|
_, state, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, run, nil, nil)
|
|
if err == nil {
|
|
t.Fatal("a run-once step that did not complete was accepted")
|
|
}
|
|
if !strings.Contains(err.Error(), "did not complete") {
|
|
t.Errorf("failed for the wrong reason: %v", err)
|
|
}
|
|
if _, recorded := state.Find("seed"); recorded {
|
|
t.Error("a run-once step that did not complete was recorded as done")
|
|
}
|
|
}
|
|
|
|
func TestAFailedRunOnceStepGatesWhatFollows(t *testing.T) {
|
|
// The whole of how "before the broker starts" is enforced: the step is declared first, and a
|
|
// step that did not complete stops the apply reaching the container that depends on it — the
|
|
// mirror of a failed action stopping what follows.
|
|
var startedNames []string
|
|
run := func(ctx context.Context, name string, args ...string) (string, error) {
|
|
switch args[0] {
|
|
case "info":
|
|
return "27.0\n", nil
|
|
case "container":
|
|
return "false\t\n", errors.New("no such container")
|
|
case "run":
|
|
startedNames = append(startedNames, nameOf(args))
|
|
if nameOf(args) == "seed" {
|
|
return "", errors.New("exit status 1") // the step fails
|
|
}
|
|
return "deadbeef\n", nil
|
|
case "rm":
|
|
return "", nil
|
|
}
|
|
return "", nil
|
|
}
|
|
d := parseTrusted(t, `{"declaration":1,"resources":[
|
|
{"id":"seed","type":"container","name":"seed","image":"`+pinned+`","run-once":true},
|
|
{"id":"broker","type":"container","name":"broker","image":"`+pinned+`"}
|
|
]}`)
|
|
|
|
_, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, run, nil, nil)
|
|
if err == nil {
|
|
t.Fatal("a failed run-once step did not fail the apply")
|
|
}
|
|
var applyErr *Error
|
|
if !errors.As(err, &applyErr) {
|
|
t.Fatalf("got %T", err)
|
|
}
|
|
if !applyErr.Gated {
|
|
t.Error("the failure does not say that nothing after the step was attempted")
|
|
}
|
|
for _, n := range startedNames {
|
|
if n == "broker" {
|
|
t.Fatal("the broker was started even though its run-once step did not complete")
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestARunOnceStepAlreadyCompletedIsNotReRun(t *testing.T) {
|
|
// Its record of having happened is the digest of its declaration. A re-apply that finds the
|
|
// digest already recorded does nothing — a step is not reconciled toward, it is run once.
|
|
d := parseTrusted(t, `{"declaration":1,"resources":[
|
|
{"id":"seed","type":"container","name":"seed","image":"`+pinned+`","run-once":true}
|
|
]}`)
|
|
want := containerSpec(d.Resources[0].(*declaration.Container), inputs{})
|
|
|
|
var ran bool
|
|
run := func(ctx context.Context, name string, args ...string) (string, error) {
|
|
switch args[0] {
|
|
case "info":
|
|
return "27.0\n", nil
|
|
case "run":
|
|
ran = true
|
|
return "", nil
|
|
case "rm":
|
|
return "", nil
|
|
}
|
|
return "", nil
|
|
}
|
|
known := store.State{}
|
|
known.Record(store.Applied{ID: "seed", Type: "container", Origin: store.OriginCarried, Target: "seed", Wrote: want})
|
|
|
|
report, _, err := Apply(context.Background(), archHost(t), d, known, store.OriginCarried, run, nil, nil)
|
|
if err != nil {
|
|
t.Fatalf("re-applying a completed run-once step failed: %v", err)
|
|
}
|
|
if ran {
|
|
t.Error("a run-once step already completed for this declaration was run again")
|
|
}
|
|
if report.Changed() {
|
|
t.Errorf("a re-applied run-once step reported a change: %+v", report.Outcomes)
|
|
}
|
|
}
|
|
|
|
func TestARunOnceStepReRunsWhenItsDeclarationChanged(t *testing.T) {
|
|
// The marker is the declaration's digest, so a changed image or environment moves it and the
|
|
// step runs again — a migration or a seed that changed is a different step.
|
|
d := parseTrusted(t, `{"declaration":1,"resources":[
|
|
{"id":"seed","type":"container","name":"seed","image":"`+pinned+`","run-once":true,"env":{"CLIENT":"new-admin"}}
|
|
]}`)
|
|
|
|
var ran bool
|
|
run := func(ctx context.Context, name string, args ...string) (string, error) {
|
|
switch args[0] {
|
|
case "info":
|
|
return "27.0\n", nil
|
|
case "run":
|
|
ran = true
|
|
return "", nil
|
|
case "rm":
|
|
return "", nil
|
|
}
|
|
return "", nil
|
|
}
|
|
// A record from an earlier, different declaration of the same step.
|
|
known := store.State{}
|
|
known.Record(store.Applied{ID: "seed", Type: "container", Origin: store.OriginCarried, Target: "seed", Wrote: "an-older-digest"})
|
|
|
|
if _, _, err := Apply(context.Background(), archHost(t), d, known, store.OriginCarried, run, nil, nil); err != nil {
|
|
t.Fatalf("apply failed: %v", err)
|
|
}
|
|
if !ran {
|
|
t.Error("a run-once step whose declaration changed was not run again")
|
|
}
|
|
}
|
|
|
|
func TestARunOnceStepRunsAgainWhenWhatItReadsChanged(t *testing.T) {
|
|
// A step that fetches a fact from a provider names the binding file it reads. What a
|
|
// container reads is part of its digest, so when the provider moved and the mesh rewrote the
|
|
// file, the step's marker no longer matches and it runs again (novox/hq ADR 0099) — the same
|
|
// rule that recreates a running container, read as "again" for a step.
|
|
dir := t.TempDir()
|
|
env := filepath.Join(dir, "acme.env")
|
|
d := parseTrusted(t, `{"declaration":1,"resources":[
|
|
{"id":"env","type":"file","path":"`+env+`","content":"ACME_ROOTS=https://10.0.0.2/roots.pem\n"},
|
|
{"id":"trust","type":"container","name":"trust","image":"`+pinned+`","run-once":true,"restart-on":["env"]}
|
|
]}`)
|
|
// The record from when the provider was elsewhere: the step's digest against the old file.
|
|
was := map[string]string{"env": declaredDigest(&declaration.File{Content: "ACME_ROOTS=https://10.0.0.1/roots.pem\n"})}
|
|
known := store.State{}
|
|
known.Record(store.Applied{ID: "trust", Type: "container", Origin: store.OriginCarried, Target: "trust",
|
|
Wrote: containerSpec(d.Resources[1].(*declaration.Container), inputs{declares: was})})
|
|
|
|
var ran bool
|
|
run := func(ctx context.Context, name string, args ...string) (string, error) {
|
|
switch args[0] {
|
|
case "info":
|
|
return "27.0\n", nil
|
|
case "run":
|
|
ran = true
|
|
return "", nil
|
|
}
|
|
return "", nil
|
|
}
|
|
report, _, err := Apply(context.Background(), archHost(t), d, known, store.OriginCarried, run, nil, nil)
|
|
if err != nil {
|
|
t.Fatalf("apply failed: %v", err)
|
|
}
|
|
if !ran {
|
|
t.Fatal("a run-once step whose named file changed was not run again")
|
|
}
|
|
if report.Outcomes[1].Action != "created" {
|
|
t.Errorf("the re-run step was not reported as having run: %+v", report.Outcomes[1])
|
|
}
|
|
|
|
// And with the file unchanged, the step stays done: "again" is when something changed.
|
|
ran = false
|
|
now := map[string]string{"env": declaredDigest(d.Resources[0].(*declaration.File))}
|
|
settled := store.State{}
|
|
settled.Record(store.Applied{ID: "env", Type: "file", Origin: store.OriginCarried, Target: env, Wrote: now["env"]})
|
|
settled.Record(store.Applied{ID: "trust", Type: "container", Origin: store.OriginCarried, Target: "trust",
|
|
Wrote: containerSpec(d.Resources[1].(*declaration.Container), inputs{declares: now})})
|
|
if _, _, err := Apply(context.Background(), archHost(t), d, settled, store.OriginCarried, run, nil, nil); err != nil {
|
|
t.Fatalf("re-apply failed: %v", err)
|
|
}
|
|
if ran {
|
|
t.Error("a run-once step whose named file did not change was run again")
|
|
}
|
|
}
|
|
|
|
func TestAContainerNamingARunOnceStepIsRecreatedWhenItRan(t *testing.T) {
|
|
// The service that consumes what a step made names the step: when the step ran this pass —
|
|
// fetched a new root — the running container holds the old one, and its spec did not move,
|
|
// so restart-on is what brings it back with the new one (novox/hq ADR 0099).
|
|
d := parseTrusted(t, `{"declaration":1,"resources":[
|
|
{"id":"trust","type":"container","name":"trust","image":"`+pinned+`","run-once":true},
|
|
{"id":"server","type":"container","name":"server","image":"`+pinned+`","restart-on":["trust"]}
|
|
]}`)
|
|
declares := map[string]string{"trust": declaredDigest(d.Resources[0].(*declaration.Container))}
|
|
spec := containerSpec(d.Resources[1].(*declaration.Container), inputs{declares: declares})
|
|
|
|
var removed, created bool
|
|
run := func(ctx context.Context, name string, args ...string) (string, error) {
|
|
switch args[0] {
|
|
case "info":
|
|
return "27.0\n", nil
|
|
case "container":
|
|
// The server is up, made from exactly this spec — nothing but the step's run says
|
|
// it must be replaced.
|
|
return "true\t" + spec, nil
|
|
case "rm":
|
|
if len(args) > 0 && args[len(args)-1] == "server" {
|
|
removed = true
|
|
}
|
|
return "", nil
|
|
case "run":
|
|
if args[len(args)-1] != "trust" && strings.Contains(strings.Join(args, " "), "--name server") {
|
|
created = true
|
|
}
|
|
return "deadbeef\n", nil
|
|
}
|
|
return "", nil
|
|
}
|
|
report, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, run, nil, nil)
|
|
if err != nil {
|
|
t.Fatalf("apply failed: %v", err)
|
|
}
|
|
if !removed || !created {
|
|
t.Fatalf("the container naming the step was not recreated after the step ran (removed=%v created=%v)", removed, created)
|
|
}
|
|
server := report.Outcomes[len(report.Outcomes)-1]
|
|
if server.Action != "updated" || !strings.Contains(server.Detail, "trust") {
|
|
t.Errorf("the recreation did not name the step as its reason: %+v", server)
|
|
}
|
|
}
|
|
|
|
func TestAFailedStepGatesItsModuleAndNotTheMachine(t *testing.T) {
|
|
// **The blast radius of a step is its module** (novox/hq ADR 0136). A step exists to make
|
|
// something true before the next thing in its own module needs it — a store seeded before the
|
|
// broker starts, a schema prepared before the version that needs it runs. Stopping the whole
|
|
// apply is what this host's own loop calls holding a machine hostage, and it was already
|
|
// rejected for every other shape (04-ISSUES/011): a module whose database is briefly
|
|
// unreachable must not stop every module declared after it.
|
|
var startedNames []string
|
|
run := func(ctx context.Context, name string, args ...string) (string, error) {
|
|
switch args[0] {
|
|
case "info":
|
|
return "27.0\n", nil
|
|
case "container":
|
|
return "false\t\n", errors.New("no such container")
|
|
case "run":
|
|
startedNames = append(startedNames, nameOf(args))
|
|
if nameOf(args) == "catalogue-prepare" {
|
|
return "", errors.New("exit status 1") // the schema could not be reached
|
|
}
|
|
return "deadbeef\n", nil
|
|
case "rm":
|
|
return "", nil
|
|
}
|
|
return "", nil
|
|
}
|
|
d := parseTrusted(t, `{"declaration":1,"resources":[
|
|
{"id":"mesh-catalog.runtime.prepare","type":"container","name":"catalogue-prepare","image":"`+pinned+`","run-once":true},
|
|
{"id":"mesh-catalog.runtime","type":"container","name":"catalogue","image":"`+pinned+`"},
|
|
{"id":"gitea.server","type":"container","name":"forge","image":"`+pinned+`"}
|
|
]}`)
|
|
|
|
report, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, run, nil, nil)
|
|
if err == nil {
|
|
t.Fatal("a failed step was not reported as a failure")
|
|
}
|
|
started := map[string]bool{}
|
|
for _, n := range startedNames {
|
|
started[n] = true
|
|
}
|
|
if started["catalogue"] {
|
|
t.Error("the module's own workload ran although its step did not complete")
|
|
}
|
|
if !started["forge"] {
|
|
t.Error("another module was not attempted, so one module's step held the machine hostage")
|
|
}
|
|
// And the machine's own account says which was not attempted, rather than leaving it to be
|
|
// inferred from silence.
|
|
var skipped string
|
|
for _, o := range report.Outcomes {
|
|
if o.Action == "skipped" {
|
|
skipped = o.ID
|
|
}
|
|
}
|
|
if skipped != "mesh-catalog.runtime" {
|
|
t.Errorf("the report does not say what was not attempted: %q", skipped)
|
|
}
|
|
}
|