jochen
a018ebd1ca
Third review of mesh-catalog #147. A unit's name can now come from a setting (a template's instance), so every systemctl call that names a unit passes `--` before it, and a name can never be read as an option. On removal, a unit is started or enabled again only when it is exactly the stopped and disabled the mesh held it to. A masked unit, or one somebody started or enabled since, is left as it is, and the report says why. A restore that fails (the start refused, or the unit not running after it) puts back what this attempt changed and is reported as waiting, so no apply is made fatal by it and the next one tries again. The tests' fake service managers read systemctl's arguments in the shape they were written for (oldShape); internal/system's own test holds the `--`.
322 lines
14 KiB
Go
322 lines
14 KiB
Go
package apply
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"testing"
|
|
|
|
"github.com/novox/mesh-host/internal/store"
|
|
)
|
|
|
|
// novox/hq issue 213: the controller moves from a container to a process on the one machine that
|
|
// runs it. Every orphan is removed before anything is applied, so without a handover the container
|
|
// went first and nothing answered the mesh's verbs while the process was fetched, unpacked and
|
|
// started — and for ever, if it did not start. A process that `replaces` the container is applied
|
|
// first; the container goes only once the process is running a moment later.
|
|
|
|
// aMachine fakes the service manager and the container runtime: it records every command, answers
|
|
// `systemctl show` with whether the process is running, and has a container until it is removed.
|
|
type aMachine struct {
|
|
commands []string
|
|
running bool // what `systemctl show` says of the process once it was started
|
|
started bool
|
|
container bool
|
|
timer bool // whether a timer, once started, is up
|
|
crashing bool // up at the first look, waiting to restart at the next
|
|
looks int
|
|
}
|
|
|
|
func (m *aMachine) run(ctx context.Context, name string, args ...string) (string, error) {
|
|
|
|
args = oldShape(name, args)
|
|
line := name + " " + strings.Join(args, " ")
|
|
m.commands = append(m.commands, line)
|
|
switch {
|
|
case name == "systemctl" && len(args) > 0 && (args[0] == "restart" || args[0] == "start"):
|
|
m.started = true
|
|
case name == "systemctl" && len(args) > 0 && args[0] == "is-active" && strings.HasSuffix(args[len(args)-1], ".timer"):
|
|
if m.timer {
|
|
return "active", nil
|
|
}
|
|
return "inactive", errors.New("inactive")
|
|
case name == "systemctl" && len(args) > 0 && args[0] == "is-active":
|
|
if m.started && m.running {
|
|
return "active", nil
|
|
}
|
|
return "inactive", errors.New("inactive")
|
|
case name == "systemctl" && len(args) > 0 && args[0] == "show":
|
|
m.looks++
|
|
if m.crashing && m.started {
|
|
// Up at the first look; waiting to be started again, a new process, at the second.
|
|
if m.looks == 1 {
|
|
return "ActiveState=active\nSubState=running\nMainPID=42\nNRestarts=0\n", nil
|
|
}
|
|
return "ActiveState=activating\nSubState=auto-restart\nMainPID=0\nNRestarts=1\n", nil
|
|
}
|
|
if m.started && m.running {
|
|
return "ActiveState=active\nSubState=running\nMainPID=42\nNRestarts=0\n", nil
|
|
}
|
|
return "ActiveState=inactive\nSubState=dead\nMainPID=0\nNRestarts=0\n", nil
|
|
case name == "docker" && len(args) > 1 && args[0] == "rm":
|
|
m.container = false
|
|
case name == "docker" && len(args) > 1 && args[0] == "container" && args[1] == "inspect":
|
|
if !m.container {
|
|
return "", errors.New("no such container")
|
|
}
|
|
return "true\t", nil
|
|
}
|
|
return "", nil
|
|
}
|
|
|
|
func (m *aMachine) index(prefix string) int {
|
|
for i, c := range m.commands {
|
|
if strings.HasPrefix(c, prefix) {
|
|
return i
|
|
}
|
|
}
|
|
return -1
|
|
}
|
|
|
|
// theController is a machine whose controller ran as a container, recorded, and a declaration that
|
|
// runs it as a process from a bundle served here instead.
|
|
func theController(t *testing.T, digest, source, extra string) (store.State, string) {
|
|
t.Helper()
|
|
known := store.State{Resources: []store.Applied{{
|
|
Origin: store.OriginDeclared, ID: "mesh-controller.server", Type: "container", Target: "mesh-controller",
|
|
}}}
|
|
return known, `{"declaration":1,"resources":[
|
|
{"id":"mesh-controller.controller","type":"process","name":"mesh-controller","source":"` + source +
|
|
`","digest":"` + digest + `","run":["./mesh-controller","serve"]` + extra + `}]}`
|
|
}
|
|
|
|
func onAMachine(t *testing.T) {
|
|
t.Helper()
|
|
serviceSettle, handoverSettle = 0, 0
|
|
wasUnits, wasBundles := unitDir, daemonRoot
|
|
unitDir, daemonRoot = t.TempDir(), t.TempDir()
|
|
t.Cleanup(func() { unitDir, daemonRoot = wasUnits, wasBundles })
|
|
}
|
|
|
|
func TestAContainerAProcessReplacesGoesOnlyOnceTheProcessRuns(t *testing.T) {
|
|
onAMachine(t)
|
|
body, digest := anArchive(t, map[string]string{"mesh-controller": "#!/bin/sh\n"})
|
|
known, raw := theController(t, digest, serving(t, body), `,"replaces":["mesh-controller.server"]`)
|
|
m := &aMachine{running: true, container: true}
|
|
|
|
report, after, err := Apply(context.Background(), archHost(t), parse(t, raw), known,
|
|
store.OriginDeclared, m.run, nil, nil)
|
|
if err != nil {
|
|
t.Fatalf("the handover failed: %v", err)
|
|
}
|
|
started, removed := m.index("systemctl restart mesh-controller.service"), m.index("docker rm -f mesh-controller")
|
|
if started < 0 || removed < 0 {
|
|
t.Fatalf("the process was not started or the container not removed: %v", m.commands)
|
|
}
|
|
if removed < started {
|
|
t.Fatalf("the container was removed before its replacement was started — a window with "+
|
|
"nothing answering: %v", m.commands)
|
|
}
|
|
if looked := m.index("systemctl show mesh-controller.service"); looked < 0 || looked > removed {
|
|
t.Errorf("the container was removed without looking whether the process runs: %v", m.commands)
|
|
}
|
|
if o := outcomeOf(report, "mesh-controller.server"); o.Action != "removed" {
|
|
t.Errorf("the container's outcome is %+v, want removed", o)
|
|
}
|
|
if _, still := after.Find("mesh-controller.server"); still {
|
|
t.Error("the host still records the container it removed")
|
|
}
|
|
if _, has := after.Find("mesh-controller.controller"); !has {
|
|
t.Error("the process was not recorded")
|
|
}
|
|
}
|
|
|
|
// The case the handover exists for: the replacement does not stay up. The container keeps
|
|
// answering, stays recorded so a later apply hands it over, and the apply says why it failed.
|
|
func TestAContainerIsKeptWhenItsReplacementDoesNotRun(t *testing.T) {
|
|
onAMachine(t)
|
|
body, digest := anArchive(t, map[string]string{"mesh-controller": "#!/bin/sh\n"})
|
|
known, raw := theController(t, digest, serving(t, body), `,"replaces":["mesh-controller.server"]`)
|
|
m := &aMachine{running: false, container: true}
|
|
|
|
report, after, err := Apply(context.Background(), archHost(t), parse(t, raw), known,
|
|
store.OriginDeclared, m.run, nil, nil)
|
|
if err == nil {
|
|
t.Fatal("a replacement that is not running was reported as a clean apply")
|
|
}
|
|
if !strings.Contains(err.Error(), "mesh-controller.server") {
|
|
t.Errorf("the failure does not name what was kept: %v", err)
|
|
}
|
|
if m.index("docker rm") >= 0 {
|
|
t.Fatalf("the container was removed though its replacement is not running: %v", m.commands)
|
|
}
|
|
if o := outcomeOf(report, "mesh-controller.server"); o.Action != "kept" {
|
|
t.Errorf("the container's outcome is %+v, want kept", o)
|
|
}
|
|
if _, still := after.Find("mesh-controller.server"); !still {
|
|
t.Fatal("the container was forgotten, so no later apply would ever remove it")
|
|
}
|
|
|
|
// The next apply finds the process up and finishes the handover.
|
|
m.running, m.commands = true, nil
|
|
report, after, err = Apply(context.Background(), archHost(t), parse(t, raw), after,
|
|
store.OriginDeclared, m.run, nil, nil)
|
|
if err != nil {
|
|
t.Fatalf("the second apply failed: %v", err)
|
|
}
|
|
if o := outcomeOf(report, "mesh-controller.server"); o.Action != "removed" {
|
|
t.Errorf("the second apply did not hand over: %+v (%v)", o, m.commands)
|
|
}
|
|
if _, still := after.Find("mesh-controller.server"); still {
|
|
t.Error("the container is still recorded after the handover")
|
|
}
|
|
}
|
|
|
|
// A replacement that never applied — its bundle is not what was declared — touches nothing, and
|
|
// what it replaces keeps running.
|
|
func TestAContainerIsKeptWhenItsReplacementFailsToApply(t *testing.T) {
|
|
onAMachine(t)
|
|
body, _ := anArchive(t, map[string]string{"mesh-controller": "#!/bin/sh\n"})
|
|
known, raw := theController(t, "sha256:"+strings.Repeat("b", 64), serving(t, body),
|
|
`,"replaces":["mesh-controller.server"]`)
|
|
m := &aMachine{running: true, container: true}
|
|
|
|
report, after, err := Apply(context.Background(), archHost(t), parse(t, raw), known,
|
|
store.OriginDeclared, m.run, nil, nil)
|
|
if err == nil {
|
|
t.Fatal("a replacement whose bundle did not match was reported applied")
|
|
}
|
|
if m.index("docker rm") >= 0 {
|
|
t.Fatalf("the container was removed though nothing replaced it: %v", m.commands)
|
|
}
|
|
if o := outcomeOf(report, "mesh-controller.server"); o.Action != "kept" {
|
|
t.Errorf("the container's outcome is %+v, want kept", o)
|
|
}
|
|
if _, still := after.Find("mesh-controller.server"); !still {
|
|
t.Fatal("the container was forgotten")
|
|
}
|
|
}
|
|
|
|
// Without `replaces` nothing changes: an orphan goes before anything is applied, as it always has.
|
|
// Kept as a test because it is the window the field exists to close.
|
|
func TestWithoutReplacesAnOrphanStillGoesFirst(t *testing.T) {
|
|
onAMachine(t)
|
|
body, digest := anArchive(t, map[string]string{"mesh-controller": "#!/bin/sh\n"})
|
|
known, raw := theController(t, digest, serving(t, body), ``)
|
|
m := &aMachine{running: true, container: true}
|
|
if _, _, err := Apply(context.Background(), archHost(t), parse(t, raw), known,
|
|
store.OriginDeclared, m.run, nil, nil); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if removed, started := m.index("docker rm -f mesh-controller"), m.index("systemctl restart mesh-controller.service"); removed < 0 || removed > started {
|
|
t.Fatalf("an orphan nothing replaces was not removed first: %v", m.commands)
|
|
}
|
|
}
|
|
|
|
// novox/hq issue 213, beyond what the oneshot unit (process_step_test.go) already holds: a step
|
|
// written `./name` runs its own bundle's binary — the controller's preparation is its own binary —
|
|
// and a step is started, never enabled.
|
|
func TestAStepRunsItsOwnBundlesBinaryAndIsNotEnabled(t *testing.T) {
|
|
onAMachine(t)
|
|
body, digest := anArchive(t, map[string]string{"mesh-controller": "#!/bin/sh\n"})
|
|
m := &aMachine{running: true}
|
|
d := declare(t, `{"id":"mesh-controller.controller-prepare","type":"process","name":"mesh-controller-prepare",
|
|
"source":"`+serving(t, body)+`","digest":"`+digest+`","run":["./mesh-controller","prepare"],"run-once":true}`)
|
|
if _, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginDeclared, m.run, nil, nil); err != nil {
|
|
t.Fatalf("the step failed: %v", err)
|
|
}
|
|
for _, c := range m.commands {
|
|
if strings.HasPrefix(c, "./") || strings.HasPrefix(c, "systemctl enable") {
|
|
t.Errorf("the step was run directly or enabled: %v", m.commands)
|
|
}
|
|
}
|
|
unit, err := os.ReadFile(filepath.Join(unitDir, "mesh-controller-prepare.service"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
want := "ExecStart=" + filepath.Join(daemonRoot, "mesh-controller-prepare", "mesh-controller") + " prepare"
|
|
if !strings.Contains(string(unit), want) {
|
|
t.Errorf("the step does not run its own bundle's binary (%q):\n%s", want, unit)
|
|
}
|
|
}
|
|
|
|
func TestAStepThatFailsGatesItsModule(t *testing.T) {
|
|
onAMachine(t)
|
|
body, digest := anArchive(t, map[string]string{"mesh-controller": "#!/bin/sh\n"})
|
|
src := serving(t, body)
|
|
run := func(ctx context.Context, name string, args ...string) (string, error) {
|
|
args = oldShape(name, args)
|
|
if name == "systemctl" && len(args) > 1 && args[0] == "start" {
|
|
return "", errors.New("Job for mesh-controller-prepare.service failed")
|
|
}
|
|
if name == "systemctl" && len(args) > 0 && args[0] == "restart" {
|
|
t.Errorf("the module's process was started after its step failed")
|
|
}
|
|
return "", nil
|
|
}
|
|
d := declare(t, `{"id":"mesh-controller.controller-prepare","type":"process","name":"mesh-controller-prepare",
|
|
"source":"`+src+`","digest":"`+digest+`","run":["./mesh-controller","prepare"],"run-once":true},
|
|
{"id":"mesh-controller.controller","type":"process","name":"mesh-controller",
|
|
"source":"`+src+`","digest":"`+digest+`","run":["./mesh-controller","serve"]}`)
|
|
report, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginDeclared, run, nil, nil)
|
|
if err == nil {
|
|
t.Fatal("a failed step was reported as a clean apply")
|
|
}
|
|
if o := outcomeOf(report, "mesh-controller.controller"); o.Action != "skipped" {
|
|
t.Errorf("the process after a failed step was %+v, want skipped", o)
|
|
}
|
|
}
|
|
|
|
// A completed step is done: applied again unchanged, it is not run again — its service is never up
|
|
// between runs, and reading that as "a daemon that stopped" re-ran the controller's preparation on
|
|
// every apply. Nor is a scheduled run started off its cadence; its timer is what is kept up.
|
|
func TestACompletedStepIsNotRunAgainAndAScheduleIsItsTimer(t *testing.T) {
|
|
onAMachine(t)
|
|
body, digest := anArchive(t, map[string]string{"job": "#!/bin/sh\n"})
|
|
src := serving(t, body)
|
|
for _, mode := range []string{`"run-once":true`, `"schedule":"0 3 * * *"`} {
|
|
// The service of either is never up between runs; a scheduled one's timer is.
|
|
m := &aMachine{running: false, timer: true}
|
|
d := declare(t, `{"id":"m.job","type":"process","name":"m-job","source":"`+src+`","digest":"`+digest+
|
|
`","run":["./job"],`+mode+`}`)
|
|
_, state, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginDeclared, m.run, nil, nil)
|
|
if err != nil {
|
|
t.Fatalf("%s: first apply: %v", mode, err)
|
|
}
|
|
m.commands = nil
|
|
report, _, err := Apply(context.Background(), archHost(t), d, state, store.OriginDeclared, m.run, nil, nil)
|
|
if err != nil {
|
|
t.Fatalf("%s: second apply: %v", mode, err)
|
|
}
|
|
if m.index("systemctl start m-job.service") >= 0 || m.index("systemctl restart m-job.service") >= 0 {
|
|
t.Errorf("%s: an unchanged apply ran the job again: %v", mode, m.commands)
|
|
}
|
|
if o := outcomeOf(report, "m.job"); o.Action != "unchanged" {
|
|
t.Errorf("%s: an unchanged apply reported %+v", mode, o)
|
|
}
|
|
}
|
|
}
|
|
|
|
// A replacement crash-looping between its restarts is not up, though the service manager calls it
|
|
// "activating" — the state the host's ordinary second look accepts as running. Removing the
|
|
// container on that reading would leave nothing answering.
|
|
func TestAContainerIsKeptWhenItsReplacementIsCrashLooping(t *testing.T) {
|
|
onAMachine(t)
|
|
body, digest := anArchive(t, map[string]string{"mesh-controller": "#!/bin/sh\n"})
|
|
known, raw := theController(t, digest, serving(t, body), `,"replaces":["mesh-controller.server"]`)
|
|
m := &aMachine{crashing: true, container: true}
|
|
_, after, err := Apply(context.Background(), archHost(t), parse(t, raw), known,
|
|
store.OriginDeclared, m.run, nil, nil)
|
|
if err == nil || !strings.Contains(err.Error(), "auto-restart") {
|
|
t.Fatalf("a crash-looping replacement was accepted: %v", err)
|
|
}
|
|
if m.index("docker rm") >= 0 {
|
|
t.Fatalf("the container was removed for a replacement that keeps dying: %v", m.commands)
|
|
}
|
|
if _, still := after.Find("mesh-controller.server"); !still {
|
|
t.Fatal("the container was forgotten")
|
|
}
|
|
}
|