Hand a replaced resource over to the process that replaces it (hq issue 213)
The controller moves from a container to a process on the one machine that runs it (novox/hq issue 213). Every orphan is removed before anything is applied, so the container would go first and nothing would answer the mesh's verbs while the process was fetched, unpacked and started — and never again, if it did not start. - a process may say what it `replaces`: resources the declaration no longer declares. Such an orphan is kept through the up-front sweep and removed right after the process applied and is up: active and running at two looks ten seconds apart, the same main process, no restart in between (stricter than ADR 0184's second look, which reads a unit waiting to restart as running). If the process failed, was skipped behind its module's step, or is not running, the orphan stays running and recorded, reported kept, and the next apply hands it over. Refused: naming something still declared, itself, an empty id, one thing named by two processes, and `replaces` on a step or a schedule. - a run-once process is run by a oneshot unit, so a step has its process's user, working directory and environment and `./name` is its own bundle's binary; it was run directly by the host, as root, with no environment. It is not enabled. - a run-once process that fails gates its module, as a run-once container already did, so a version whose preparation failed is not started. - an unchanged run-once process is not run again, and an unchanged scheduled one is kept up by its timer: both were "a daemon that had stopped" and were started on every apply.
This commit is contained in:
jochen
@@ -0,0 +1,336 @@
|
||||
package apply
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-host/internal/store"
|
||||
)
|
||||
|
||||
// novox/hq issue 213: the controller moves from a container to a process on the one machine that
|
||||
// runs it. Every orphan is removed before anything is applied, so without a handover the container
|
||||
// went first and nothing answered the mesh's verbs while the process was fetched, unpacked and
|
||||
// started — and for ever, if it did not start. A process that `replaces` the container is applied
|
||||
// first; the container goes only once the process is running a moment later.
|
||||
|
||||
// aMachine fakes the service manager and the container runtime: it records every command, answers
|
||||
// `systemctl show` with whether the process is running, and has a container until it is removed.
|
||||
type aMachine struct {
|
||||
commands []string
|
||||
running bool // what `systemctl show` says of the process once it was started
|
||||
started bool
|
||||
container bool
|
||||
timer bool // whether a timer, once started, is up
|
||||
crashing bool // up at the first look, waiting to restart at the next
|
||||
looks int
|
||||
}
|
||||
|
||||
func (m *aMachine) run(ctx context.Context, name string, args ...string) (string, error) {
|
||||
line := name + " " + strings.Join(args, " ")
|
||||
m.commands = append(m.commands, line)
|
||||
switch {
|
||||
case name == "systemctl" && len(args) > 0 && (args[0] == "restart" || args[0] == "start"):
|
||||
m.started = true
|
||||
case name == "systemctl" && len(args) > 0 && args[0] == "is-active" && strings.HasSuffix(args[len(args)-1], ".timer"):
|
||||
if m.timer {
|
||||
return "active", nil
|
||||
}
|
||||
return "inactive", errors.New("inactive")
|
||||
case name == "systemctl" && len(args) > 0 && args[0] == "is-active":
|
||||
if m.started && m.running {
|
||||
return "active", nil
|
||||
}
|
||||
return "inactive", errors.New("inactive")
|
||||
case name == "systemctl" && len(args) > 0 && args[0] == "show":
|
||||
m.looks++
|
||||
if m.crashing && m.started {
|
||||
// Up at the first look; waiting to be started again, a new process, at the second.
|
||||
if m.looks == 1 {
|
||||
return "ActiveState=active\nSubState=running\nMainPID=42\nNRestarts=0\n", nil
|
||||
}
|
||||
return "ActiveState=activating\nSubState=auto-restart\nMainPID=0\nNRestarts=1\n", nil
|
||||
}
|
||||
if m.started && m.running {
|
||||
return "ActiveState=active\nSubState=running\nMainPID=42\nNRestarts=0\n", nil
|
||||
}
|
||||
return "ActiveState=inactive\nSubState=dead\nMainPID=0\nNRestarts=0\n", nil
|
||||
case name == "docker" && len(args) > 1 && args[0] == "rm":
|
||||
m.container = false
|
||||
case name == "docker" && len(args) > 1 && args[0] == "container" && args[1] == "inspect":
|
||||
if !m.container {
|
||||
return "", errors.New("no such container")
|
||||
}
|
||||
return "true\t", nil
|
||||
}
|
||||
return "", nil
|
||||
}
|
||||
|
||||
func (m *aMachine) index(prefix string) int {
|
||||
for i, c := range m.commands {
|
||||
if strings.HasPrefix(c, prefix) {
|
||||
return i
|
||||
}
|
||||
}
|
||||
return -1
|
||||
}
|
||||
|
||||
// theController is a machine whose controller ran as a container, recorded, and a declaration that
|
||||
// runs it as a process from a bundle served here instead.
|
||||
func theController(t *testing.T, digest, source, extra string) (store.State, string) {
|
||||
t.Helper()
|
||||
known := store.State{Resources: []store.Applied{{
|
||||
Origin: store.OriginDeclared, ID: "mesh-controller.server", Type: "container", Target: "mesh-controller",
|
||||
}}}
|
||||
return known, `{"declaration":1,"resources":[
|
||||
{"id":"mesh-controller.controller","type":"process","name":"mesh-controller","source":"` + source +
|
||||
`","digest":"` + digest + `","run":["./mesh-controller","serve"]` + extra + `}]}`
|
||||
}
|
||||
|
||||
func onAMachine(t *testing.T) {
|
||||
t.Helper()
|
||||
serviceSettle, handoverSettle = 0, 0
|
||||
wasUnits, wasBundles := unitDir, daemonRoot
|
||||
unitDir, daemonRoot = t.TempDir(), t.TempDir()
|
||||
t.Cleanup(func() { unitDir, daemonRoot = wasUnits, wasBundles })
|
||||
}
|
||||
|
||||
func TestAContainerAProcessReplacesGoesOnlyOnceTheProcessRuns(t *testing.T) {
|
||||
onAMachine(t)
|
||||
body, digest := anArchive(t, map[string]string{"mesh-controller": "#!/bin/sh\n"})
|
||||
known, raw := theController(t, digest, serving(t, body), `,"replaces":["mesh-controller.server"]`)
|
||||
m := &aMachine{running: true, container: true}
|
||||
|
||||
report, after, err := Apply(context.Background(), archHost(t), parse(t, raw), known,
|
||||
store.OriginDeclared, m.run, nil, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("the handover failed: %v", err)
|
||||
}
|
||||
started, removed := m.index("systemctl restart mesh-controller.service"), m.index("docker rm -f mesh-controller")
|
||||
if started < 0 || removed < 0 {
|
||||
t.Fatalf("the process was not started or the container not removed: %v", m.commands)
|
||||
}
|
||||
if removed < started {
|
||||
t.Fatalf("the container was removed before its replacement was started — a window with "+
|
||||
"nothing answering: %v", m.commands)
|
||||
}
|
||||
if looked := m.index("systemctl show mesh-controller.service"); looked < 0 || looked > removed {
|
||||
t.Errorf("the container was removed without looking whether the process runs: %v", m.commands)
|
||||
}
|
||||
if o := outcomeOf(report, "mesh-controller.server"); o.Action != "removed" {
|
||||
t.Errorf("the container's outcome is %+v, want removed", o)
|
||||
}
|
||||
if _, still := after.Find("mesh-controller.server"); still {
|
||||
t.Error("the host still records the container it removed")
|
||||
}
|
||||
if _, has := after.Find("mesh-controller.controller"); !has {
|
||||
t.Error("the process was not recorded")
|
||||
}
|
||||
}
|
||||
|
||||
// The case the handover exists for: the replacement does not stay up. The container keeps
|
||||
// answering, stays recorded so a later apply hands it over, and the apply says why it failed.
|
||||
func TestAContainerIsKeptWhenItsReplacementDoesNotRun(t *testing.T) {
|
||||
onAMachine(t)
|
||||
body, digest := anArchive(t, map[string]string{"mesh-controller": "#!/bin/sh\n"})
|
||||
known, raw := theController(t, digest, serving(t, body), `,"replaces":["mesh-controller.server"]`)
|
||||
m := &aMachine{running: false, container: true}
|
||||
|
||||
report, after, err := Apply(context.Background(), archHost(t), parse(t, raw), known,
|
||||
store.OriginDeclared, m.run, nil, nil)
|
||||
if err == nil {
|
||||
t.Fatal("a replacement that is not running was reported as a clean apply")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "mesh-controller.server") {
|
||||
t.Errorf("the failure does not name what was kept: %v", err)
|
||||
}
|
||||
if m.index("docker rm") >= 0 {
|
||||
t.Fatalf("the container was removed though its replacement is not running: %v", m.commands)
|
||||
}
|
||||
if o := outcomeOf(report, "mesh-controller.server"); o.Action != "kept" {
|
||||
t.Errorf("the container's outcome is %+v, want kept", o)
|
||||
}
|
||||
if _, still := after.Find("mesh-controller.server"); !still {
|
||||
t.Fatal("the container was forgotten, so no later apply would ever remove it")
|
||||
}
|
||||
|
||||
// The next apply finds the process up and finishes the handover.
|
||||
m.running, m.commands = true, nil
|
||||
report, after, err = Apply(context.Background(), archHost(t), parse(t, raw), after,
|
||||
store.OriginDeclared, m.run, nil, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("the second apply failed: %v", err)
|
||||
}
|
||||
if o := outcomeOf(report, "mesh-controller.server"); o.Action != "removed" {
|
||||
t.Errorf("the second apply did not hand over: %+v (%v)", o, m.commands)
|
||||
}
|
||||
if _, still := after.Find("mesh-controller.server"); still {
|
||||
t.Error("the container is still recorded after the handover")
|
||||
}
|
||||
}
|
||||
|
||||
// A replacement that never applied — its bundle is not what was declared — touches nothing, and
|
||||
// what it replaces keeps running.
|
||||
func TestAContainerIsKeptWhenItsReplacementFailsToApply(t *testing.T) {
|
||||
onAMachine(t)
|
||||
body, _ := anArchive(t, map[string]string{"mesh-controller": "#!/bin/sh\n"})
|
||||
known, raw := theController(t, "sha256:"+strings.Repeat("b", 64), serving(t, body),
|
||||
`,"replaces":["mesh-controller.server"]`)
|
||||
m := &aMachine{running: true, container: true}
|
||||
|
||||
report, after, err := Apply(context.Background(), archHost(t), parse(t, raw), known,
|
||||
store.OriginDeclared, m.run, nil, nil)
|
||||
if err == nil {
|
||||
t.Fatal("a replacement whose bundle did not match was reported applied")
|
||||
}
|
||||
if m.index("docker rm") >= 0 {
|
||||
t.Fatalf("the container was removed though nothing replaced it: %v", m.commands)
|
||||
}
|
||||
if o := outcomeOf(report, "mesh-controller.server"); o.Action != "kept" {
|
||||
t.Errorf("the container's outcome is %+v, want kept", o)
|
||||
}
|
||||
if _, still := after.Find("mesh-controller.server"); !still {
|
||||
t.Fatal("the container was forgotten")
|
||||
}
|
||||
}
|
||||
|
||||
// Without `replaces` nothing changes: an orphan goes before anything is applied, as it always has.
|
||||
// Kept as a test because it is the window the field exists to close.
|
||||
func TestWithoutReplacesAnOrphanStillGoesFirst(t *testing.T) {
|
||||
onAMachine(t)
|
||||
body, digest := anArchive(t, map[string]string{"mesh-controller": "#!/bin/sh\n"})
|
||||
known, raw := theController(t, digest, serving(t, body), ``)
|
||||
m := &aMachine{running: true, container: true}
|
||||
if _, _, err := Apply(context.Background(), archHost(t), parse(t, raw), known,
|
||||
store.OriginDeclared, m.run, nil, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if removed, started := m.index("docker rm -f mesh-controller"), m.index("systemctl restart mesh-controller.service"); removed < 0 || removed > started {
|
||||
t.Fatalf("an orphan nothing replaces was not removed first: %v", m.commands)
|
||||
}
|
||||
}
|
||||
|
||||
// novox/hq issue 213: a step is run by its unit — as the process it prepares for is — so it has
|
||||
// that process's user, directory and environment, and `./name` is its own bundle's binary. It used
|
||||
// to be run directly by the host, as root, in the host's directory, with no environment.
|
||||
func TestAStepIsRunByItsUnitWithItsUserAndEnvironment(t *testing.T) {
|
||||
onAMachine(t)
|
||||
body, digest := anArchive(t, map[string]string{"mesh-controller": "#!/bin/sh\n"})
|
||||
m := &aMachine{running: true}
|
||||
d := declare(t, `{"id":"mesh-controller.controller-prepare","type":"process","name":"mesh-controller-prepare",
|
||||
"source":"`+serving(t, body)+`","digest":"`+digest+`","run":["./mesh-controller","prepare"],"run-once":true,
|
||||
"env":{"MESH_STORE_INVENTORY_FILE":"/var/lib/mesh/x/inventory"}}`)
|
||||
report, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginDeclared, m.run, nil, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("the step failed: %v", err)
|
||||
}
|
||||
if m.index("systemctl start mesh-controller-prepare.service") < 0 {
|
||||
t.Fatalf("the step was not started through its unit: %v", m.commands)
|
||||
}
|
||||
for _, c := range m.commands {
|
||||
if strings.HasPrefix(c, "./") || strings.HasPrefix(c, "systemctl enable") {
|
||||
t.Errorf("the step was run directly or enabled: %v", m.commands)
|
||||
}
|
||||
}
|
||||
unit, err := os.ReadFile(filepath.Join(unitDir, "mesh-controller-prepare.service"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, want := range []string{
|
||||
"Type=oneshot",
|
||||
"ExecStart=" + filepath.Join(daemonRoot, "mesh-controller-prepare", "mesh-controller") + " prepare",
|
||||
`Environment="MESH_STORE_INVENTORY_FILE=/var/lib/mesh/x/inventory"`,
|
||||
} {
|
||||
if !strings.Contains(string(unit), want) {
|
||||
t.Errorf("the step's unit does not say %q:\n%s", want, unit)
|
||||
}
|
||||
}
|
||||
for _, not := range []string{"Restart=always", "WantedBy="} {
|
||||
if strings.Contains(string(unit), not) {
|
||||
t.Errorf("the step's unit says %q, which makes it a service:\n%s", not, unit)
|
||||
}
|
||||
}
|
||||
if o := outcomeOf(report, "mesh-controller.controller-prepare"); o.Action != "created" {
|
||||
t.Errorf("the step's outcome is %+v", o)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAStepThatFailsGatesItsModule(t *testing.T) {
|
||||
onAMachine(t)
|
||||
body, digest := anArchive(t, map[string]string{"mesh-controller": "#!/bin/sh\n"})
|
||||
src := serving(t, body)
|
||||
run := func(ctx context.Context, name string, args ...string) (string, error) {
|
||||
if name == "systemctl" && len(args) > 1 && args[0] == "start" {
|
||||
return "", errors.New("Job for mesh-controller-prepare.service failed")
|
||||
}
|
||||
if name == "systemctl" && len(args) > 0 && args[0] == "restart" {
|
||||
t.Errorf("the module's process was started after its step failed")
|
||||
}
|
||||
return "", nil
|
||||
}
|
||||
d := declare(t, `{"id":"mesh-controller.controller-prepare","type":"process","name":"mesh-controller-prepare",
|
||||
"source":"`+src+`","digest":"`+digest+`","run":["./mesh-controller","prepare"],"run-once":true},
|
||||
{"id":"mesh-controller.controller","type":"process","name":"mesh-controller",
|
||||
"source":"`+src+`","digest":"`+digest+`","run":["./mesh-controller","serve"]}`)
|
||||
report, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginDeclared, run, nil, nil)
|
||||
if err == nil {
|
||||
t.Fatal("a failed step was reported as a clean apply")
|
||||
}
|
||||
if o := outcomeOf(report, "mesh-controller.controller"); o.Action != "skipped" {
|
||||
t.Errorf("the process after a failed step was %+v, want skipped", o)
|
||||
}
|
||||
}
|
||||
|
||||
// A completed step is done: applied again unchanged, it is not run again — its service is never up
|
||||
// between runs, and reading that as "a daemon that stopped" re-ran the controller's preparation on
|
||||
// every apply. Nor is a scheduled run started off its cadence; its timer is what is kept up.
|
||||
func TestACompletedStepIsNotRunAgainAndAScheduleIsItsTimer(t *testing.T) {
|
||||
onAMachine(t)
|
||||
body, digest := anArchive(t, map[string]string{"job": "#!/bin/sh\n"})
|
||||
src := serving(t, body)
|
||||
for _, mode := range []string{`"run-once":true`, `"schedule":"0 3 * * *"`} {
|
||||
// The service of either is never up between runs; a scheduled one's timer is.
|
||||
m := &aMachine{running: false, timer: true}
|
||||
d := declare(t, `{"id":"m.job","type":"process","name":"m-job","source":"`+src+`","digest":"`+digest+
|
||||
`","run":["./job"],`+mode+`}`)
|
||||
_, state, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginDeclared, m.run, nil, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("%s: first apply: %v", mode, err)
|
||||
}
|
||||
m.commands = nil
|
||||
report, _, err := Apply(context.Background(), archHost(t), d, state, store.OriginDeclared, m.run, nil, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("%s: second apply: %v", mode, err)
|
||||
}
|
||||
if m.index("systemctl start m-job.service") >= 0 || m.index("systemctl restart m-job.service") >= 0 {
|
||||
t.Errorf("%s: an unchanged apply ran the job again: %v", mode, m.commands)
|
||||
}
|
||||
if o := outcomeOf(report, "m.job"); o.Action != "unchanged" {
|
||||
t.Errorf("%s: an unchanged apply reported %+v", mode, o)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A replacement crash-looping between its restarts is not up, though the service manager calls it
|
||||
// "activating" — the state the host's ordinary second look accepts as running. Removing the
|
||||
// container on that reading would leave nothing answering.
|
||||
func TestAContainerIsKeptWhenItsReplacementIsCrashLooping(t *testing.T) {
|
||||
onAMachine(t)
|
||||
body, digest := anArchive(t, map[string]string{"mesh-controller": "#!/bin/sh\n"})
|
||||
known, raw := theController(t, digest, serving(t, body), `,"replaces":["mesh-controller.server"]`)
|
||||
m := &aMachine{crashing: true, container: true}
|
||||
_, after, err := Apply(context.Background(), archHost(t), parse(t, raw), known,
|
||||
store.OriginDeclared, m.run, nil, nil)
|
||||
if err == nil || !strings.Contains(err.Error(), "auto-restart") {
|
||||
t.Fatalf("a crash-looping replacement was accepted: %v", err)
|
||||
}
|
||||
if m.index("docker rm") >= 0 {
|
||||
t.Fatalf("the container was removed for a replacement that keeps dying: %v", m.commands)
|
||||
}
|
||||
if _, still := after.Find("mesh-controller.server"); !still {
|
||||
t.Fatal("the container was forgotten")
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user