package apply import ( "context" "errors" "os" "path/filepath" "strings" "testing" "github.com/novox/mesh-host/internal/store" ) // novox/hq issue 213: the controller moves from a container to a process on the one machine that // runs it. Every orphan is removed before anything is applied, so without a handover the container // went first and nothing answered the mesh's verbs while the process was fetched, unpacked and // started — and for ever, if it did not start. A process that `replaces` the container is applied // first; the container goes only once the process is running a moment later. // aMachine fakes the service manager and the container runtime: it records every command, answers // `systemctl show` with whether the process is running, and has a container until it is removed. type aMachine struct { commands []string running bool // what `systemctl show` says of the process once it was started started bool container bool timer bool // whether a timer, once started, is up crashing bool // up at the first look, waiting to restart at the next looks int } func (m *aMachine) run(ctx context.Context, name string, args ...string) (string, error) { line := name + " " + strings.Join(args, " ") m.commands = append(m.commands, line) switch { case name == "systemctl" && len(args) > 0 && (args[0] == "restart" || args[0] == "start"): m.started = true case name == "systemctl" && len(args) > 0 && args[0] == "is-active" && strings.HasSuffix(args[len(args)-1], ".timer"): if m.timer { return "active", nil } return "inactive", errors.New("inactive") case name == "systemctl" && len(args) > 0 && args[0] == "is-active": if m.started && m.running { return "active", nil } return "inactive", errors.New("inactive") case name == "systemctl" && len(args) > 0 && args[0] == "show": m.looks++ if m.crashing && m.started { // Up at the first look; waiting to be started again, a new process, at the second. if m.looks == 1 { return "ActiveState=active\nSubState=running\nMainPID=42\nNRestarts=0\n", nil } return "ActiveState=activating\nSubState=auto-restart\nMainPID=0\nNRestarts=1\n", nil } if m.started && m.running { return "ActiveState=active\nSubState=running\nMainPID=42\nNRestarts=0\n", nil } return "ActiveState=inactive\nSubState=dead\nMainPID=0\nNRestarts=0\n", nil case name == "docker" && len(args) > 1 && args[0] == "rm": m.container = false case name == "docker" && len(args) > 1 && args[0] == "container" && args[1] == "inspect": if !m.container { return "", errors.New("no such container") } return "true\t", nil } return "", nil } func (m *aMachine) index(prefix string) int { for i, c := range m.commands { if strings.HasPrefix(c, prefix) { return i } } return -1 } // theController is a machine whose controller ran as a container, recorded, and a declaration that // runs it as a process from a bundle served here instead. func theController(t *testing.T, digest, source, extra string) (store.State, string) { t.Helper() known := store.State{Resources: []store.Applied{{ Origin: store.OriginDeclared, ID: "mesh-controller.server", Type: "container", Target: "mesh-controller", }}} return known, `{"declaration":1,"resources":[ {"id":"mesh-controller.controller","type":"process","name":"mesh-controller","source":"` + source + `","digest":"` + digest + `","run":["./mesh-controller","serve"]` + extra + `}]}` } func onAMachine(t *testing.T) { t.Helper() serviceSettle, handoverSettle = 0, 0 wasUnits, wasBundles := unitDir, daemonRoot unitDir, daemonRoot = t.TempDir(), t.TempDir() t.Cleanup(func() { unitDir, daemonRoot = wasUnits, wasBundles }) } func TestAContainerAProcessReplacesGoesOnlyOnceTheProcessRuns(t *testing.T) { onAMachine(t) body, digest := anArchive(t, map[string]string{"mesh-controller": "#!/bin/sh\n"}) known, raw := theController(t, digest, serving(t, body), `,"replaces":["mesh-controller.server"]`) m := &aMachine{running: true, container: true} report, after, err := Apply(context.Background(), archHost(t), parse(t, raw), known, store.OriginDeclared, m.run, nil, nil) if err != nil { t.Fatalf("the handover failed: %v", err) } started, removed := m.index("systemctl restart mesh-controller.service"), m.index("docker rm -f mesh-controller") if started < 0 || removed < 0 { t.Fatalf("the process was not started or the container not removed: %v", m.commands) } if removed < started { t.Fatalf("the container was removed before its replacement was started — a window with "+ "nothing answering: %v", m.commands) } if looked := m.index("systemctl show mesh-controller.service"); looked < 0 || looked > removed { t.Errorf("the container was removed without looking whether the process runs: %v", m.commands) } if o := outcomeOf(report, "mesh-controller.server"); o.Action != "removed" { t.Errorf("the container's outcome is %+v, want removed", o) } if _, still := after.Find("mesh-controller.server"); still { t.Error("the host still records the container it removed") } if _, has := after.Find("mesh-controller.controller"); !has { t.Error("the process was not recorded") } } // The case the handover exists for: the replacement does not stay up. The container keeps // answering, stays recorded so a later apply hands it over, and the apply says why it failed. func TestAContainerIsKeptWhenItsReplacementDoesNotRun(t *testing.T) { onAMachine(t) body, digest := anArchive(t, map[string]string{"mesh-controller": "#!/bin/sh\n"}) known, raw := theController(t, digest, serving(t, body), `,"replaces":["mesh-controller.server"]`) m := &aMachine{running: false, container: true} report, after, err := Apply(context.Background(), archHost(t), parse(t, raw), known, store.OriginDeclared, m.run, nil, nil) if err == nil { t.Fatal("a replacement that is not running was reported as a clean apply") } if !strings.Contains(err.Error(), "mesh-controller.server") { t.Errorf("the failure does not name what was kept: %v", err) } if m.index("docker rm") >= 0 { t.Fatalf("the container was removed though its replacement is not running: %v", m.commands) } if o := outcomeOf(report, "mesh-controller.server"); o.Action != "kept" { t.Errorf("the container's outcome is %+v, want kept", o) } if _, still := after.Find("mesh-controller.server"); !still { t.Fatal("the container was forgotten, so no later apply would ever remove it") } // The next apply finds the process up and finishes the handover. m.running, m.commands = true, nil report, after, err = Apply(context.Background(), archHost(t), parse(t, raw), after, store.OriginDeclared, m.run, nil, nil) if err != nil { t.Fatalf("the second apply failed: %v", err) } if o := outcomeOf(report, "mesh-controller.server"); o.Action != "removed" { t.Errorf("the second apply did not hand over: %+v (%v)", o, m.commands) } if _, still := after.Find("mesh-controller.server"); still { t.Error("the container is still recorded after the handover") } } // A replacement that never applied — its bundle is not what was declared — touches nothing, and // what it replaces keeps running. func TestAContainerIsKeptWhenItsReplacementFailsToApply(t *testing.T) { onAMachine(t) body, _ := anArchive(t, map[string]string{"mesh-controller": "#!/bin/sh\n"}) known, raw := theController(t, "sha256:"+strings.Repeat("b", 64), serving(t, body), `,"replaces":["mesh-controller.server"]`) m := &aMachine{running: true, container: true} report, after, err := Apply(context.Background(), archHost(t), parse(t, raw), known, store.OriginDeclared, m.run, nil, nil) if err == nil { t.Fatal("a replacement whose bundle did not match was reported applied") } if m.index("docker rm") >= 0 { t.Fatalf("the container was removed though nothing replaced it: %v", m.commands) } if o := outcomeOf(report, "mesh-controller.server"); o.Action != "kept" { t.Errorf("the container's outcome is %+v, want kept", o) } if _, still := after.Find("mesh-controller.server"); !still { t.Fatal("the container was forgotten") } } // Without `replaces` nothing changes: an orphan goes before anything is applied, as it always has. // Kept as a test because it is the window the field exists to close. func TestWithoutReplacesAnOrphanStillGoesFirst(t *testing.T) { onAMachine(t) body, digest := anArchive(t, map[string]string{"mesh-controller": "#!/bin/sh\n"}) known, raw := theController(t, digest, serving(t, body), ``) m := &aMachine{running: true, container: true} if _, _, err := Apply(context.Background(), archHost(t), parse(t, raw), known, store.OriginDeclared, m.run, nil, nil); err != nil { t.Fatal(err) } if removed, started := m.index("docker rm -f mesh-controller"), m.index("systemctl restart mesh-controller.service"); removed < 0 || removed > started { t.Fatalf("an orphan nothing replaces was not removed first: %v", m.commands) } } // novox/hq issue 213, beyond what the oneshot unit (process_step_test.go) already holds: a step // written `./name` runs its own bundle's binary — the controller's preparation is its own binary — // and a step is started, never enabled. func TestAStepRunsItsOwnBundlesBinaryAndIsNotEnabled(t *testing.T) { onAMachine(t) body, digest := anArchive(t, map[string]string{"mesh-controller": "#!/bin/sh\n"}) m := &aMachine{running: true} d := declare(t, `{"id":"mesh-controller.controller-prepare","type":"process","name":"mesh-controller-prepare", "source":"`+serving(t, body)+`","digest":"`+digest+`","run":["./mesh-controller","prepare"],"run-once":true}`) if _, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginDeclared, m.run, nil, nil); err != nil { t.Fatalf("the step failed: %v", err) } for _, c := range m.commands { if strings.HasPrefix(c, "./") || strings.HasPrefix(c, "systemctl enable") { t.Errorf("the step was run directly or enabled: %v", m.commands) } } unit, err := os.ReadFile(filepath.Join(unitDir, "mesh-controller-prepare.service")) if err != nil { t.Fatal(err) } want := "ExecStart=" + filepath.Join(daemonRoot, "mesh-controller-prepare", "mesh-controller") + " prepare" if !strings.Contains(string(unit), want) { t.Errorf("the step does not run its own bundle's binary (%q):\n%s", want, unit) } } func TestAStepThatFailsGatesItsModule(t *testing.T) { onAMachine(t) body, digest := anArchive(t, map[string]string{"mesh-controller": "#!/bin/sh\n"}) src := serving(t, body) run := func(ctx context.Context, name string, args ...string) (string, error) { if name == "systemctl" && len(args) > 1 && args[0] == "start" { return "", errors.New("Job for mesh-controller-prepare.service failed") } if name == "systemctl" && len(args) > 0 && args[0] == "restart" { t.Errorf("the module's process was started after its step failed") } return "", nil } d := declare(t, `{"id":"mesh-controller.controller-prepare","type":"process","name":"mesh-controller-prepare", "source":"`+src+`","digest":"`+digest+`","run":["./mesh-controller","prepare"],"run-once":true}, {"id":"mesh-controller.controller","type":"process","name":"mesh-controller", "source":"`+src+`","digest":"`+digest+`","run":["./mesh-controller","serve"]}`) report, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginDeclared, run, nil, nil) if err == nil { t.Fatal("a failed step was reported as a clean apply") } if o := outcomeOf(report, "mesh-controller.controller"); o.Action != "skipped" { t.Errorf("the process after a failed step was %+v, want skipped", o) } } // A completed step is done: applied again unchanged, it is not run again — its service is never up // between runs, and reading that as "a daemon that stopped" re-ran the controller's preparation on // every apply. Nor is a scheduled run started off its cadence; its timer is what is kept up. func TestACompletedStepIsNotRunAgainAndAScheduleIsItsTimer(t *testing.T) { onAMachine(t) body, digest := anArchive(t, map[string]string{"job": "#!/bin/sh\n"}) src := serving(t, body) for _, mode := range []string{`"run-once":true`, `"schedule":"0 3 * * *"`} { // The service of either is never up between runs; a scheduled one's timer is. m := &aMachine{running: false, timer: true} d := declare(t, `{"id":"m.job","type":"process","name":"m-job","source":"`+src+`","digest":"`+digest+ `","run":["./job"],`+mode+`}`) _, state, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginDeclared, m.run, nil, nil) if err != nil { t.Fatalf("%s: first apply: %v", mode, err) } m.commands = nil report, _, err := Apply(context.Background(), archHost(t), d, state, store.OriginDeclared, m.run, nil, nil) if err != nil { t.Fatalf("%s: second apply: %v", mode, err) } if m.index("systemctl start m-job.service") >= 0 || m.index("systemctl restart m-job.service") >= 0 { t.Errorf("%s: an unchanged apply ran the job again: %v", mode, m.commands) } if o := outcomeOf(report, "m.job"); o.Action != "unchanged" { t.Errorf("%s: an unchanged apply reported %+v", mode, o) } } } // A replacement crash-looping between its restarts is not up, though the service manager calls it // "activating" — the state the host's ordinary second look accepts as running. Removing the // container on that reading would leave nothing answering. func TestAContainerIsKeptWhenItsReplacementIsCrashLooping(t *testing.T) { onAMachine(t) body, digest := anArchive(t, map[string]string{"mesh-controller": "#!/bin/sh\n"}) known, raw := theController(t, digest, serving(t, body), `,"replaces":["mesh-controller.server"]`) m := &aMachine{crashing: true, container: true} _, after, err := Apply(context.Background(), archHost(t), parse(t, raw), known, store.OriginDeclared, m.run, nil, nil) if err == nil || !strings.Contains(err.Error(), "auto-restart") { t.Fatalf("a crash-looping replacement was accepted: %v", err) } if m.index("docker rm") >= 0 { t.Fatalf("the container was removed for a replacement that keeps dying: %v", m.commands) } if _, still := after.Find("mesh-controller.server"); !still { t.Fatal("the container was forgotten") } }