317 lines
15 KiB
Go
317 lines
15 KiB
Go
package apply
|
|
|
|
import (
|
|
"context"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"testing"
|
|
|
|
"git.novox.be/novox/mesh-sdk/go/declaration"
|
|
"github.com/novox/mesh-host/internal/store"
|
|
)
|
|
|
|
// Defends novox/hq 04-ISSUES/103: a container is recreated when the CONTENT of a file it reads at
|
|
// creation changes, not only when its path does. A container takes its env-file and its mounted
|
|
// files in once, when it is created; `docker restart` hands it the same environment again, so
|
|
// only a recreate carries a rewritten file into the process.
|
|
//
|
|
// The runtime here is the `machine` fake: it keeps the spec label the host gave a container and
|
|
// hands it back on inspect, so the comparison under test is the one the host really makes,
|
|
// against what it really wrote — not against a spec a test imagined.
|
|
|
|
func applyCarried(t *testing.T, d *declaration.Declaration, known store.State, m *machine,
|
|
log func(string)) (Report, store.State) {
|
|
t.Helper()
|
|
report, state, err := Apply(context.Background(), archHost(t), d, known, store.OriginCarried, m.run, log, nil)
|
|
if err != nil {
|
|
t.Fatalf("apply failed: %v", err)
|
|
}
|
|
return report, state
|
|
}
|
|
|
|
func TestAContainerIsRecreatedWhenItsEnvFileChanged(t *testing.T) {
|
|
// The night of the issue: the store was given a new port, the host rewrote the forge's
|
|
// environment file with it — and left the forge running with the old one.
|
|
dir := t.TempDir()
|
|
env := filepath.Join(dir, "forge.env")
|
|
declare := func(port string) *declaration.Declaration {
|
|
return parseTrusted(t, `{"declaration":1,"resources":[
|
|
{"id":"forge.env","type":"file","path":"`+env+`","content":"DATABASE_PORT=`+port+`\n","mode":"0600"},
|
|
{"id":"forge.server","type":"container","name":"forge","image":"`+pinned+`","env-file":["`+env+`"]}
|
|
]}`)
|
|
}
|
|
m := &machine{containers: map[string]*fakeContainer{}}
|
|
var logged []string
|
|
log := func(line string) { logged = append(logged, line) }
|
|
|
|
report, state := applyCarried(t, declare("5432"), store.State{}, m, log)
|
|
if o := outcomeOf(report, "forge.server"); o.Action != "created" {
|
|
t.Fatalf("the container was not created: %+v", report.Outcomes)
|
|
}
|
|
|
|
// The store moved. The file is rewritten in this apply, before the container is reached, and
|
|
// the container must follow it in the same pass.
|
|
m.asked, logged = nil, nil
|
|
report, state = applyCarried(t, declare("5433"), state, m, log)
|
|
if !m.removed("forge") || !m.did("docker run") {
|
|
t.Fatalf("the container kept running with the old environment after its env-file changed: %v", m.asked)
|
|
}
|
|
o := outcomeOf(report, "forge.server")
|
|
if o.Action != "updated" || o.Detail != "recreated: "+env+" changed" {
|
|
t.Errorf("the recreate did not say which file changed: %+v", o)
|
|
}
|
|
var said bool
|
|
for _, line := range logged {
|
|
if strings.Contains(line, "updated forge.server") && strings.Contains(line, "recreated: "+env+" changed") {
|
|
said = true
|
|
}
|
|
}
|
|
if !said {
|
|
t.Errorf("the log did not say which file made the container recreate: %q", logged)
|
|
}
|
|
|
|
// And with nothing moved, it is left alone: content is part of the identity, not a tripwire.
|
|
m.asked = nil
|
|
report, _ = applyCarried(t, declare("5433"), state, m, log)
|
|
if m.did("docker rm") || m.did("docker run") || report.Changed() {
|
|
t.Errorf("a container whose env-file did not change was recreated: %v %+v", m.asked, report.Outcomes)
|
|
}
|
|
}
|
|
|
|
func TestAnEnvFileTheHostDidNotWriteIsStillReadForWhatItHolds(t *testing.T) {
|
|
// The host has no record of this file — a predecessor left it, or something else on the
|
|
// machine maintains it — and the container still reads it once. Its content is read from the
|
|
// disk, so a change is a recreate exactly as for a file the host wrote.
|
|
dir := t.TempDir()
|
|
env := filepath.Join(dir, "app.env")
|
|
if err := os.WriteFile(env, []byte("TOKEN=old\n"), 0o600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
d := parseTrusted(t, `{"declaration":1,"resources":[
|
|
{"id":"app.server","type":"container","name":"app","image":"`+pinned+`","env-file":["`+env+`"]}
|
|
]}`)
|
|
m := &machine{containers: map[string]*fakeContainer{}}
|
|
_, state := applyCarried(t, d, store.State{}, m, nil)
|
|
|
|
if err := os.WriteFile(env, []byte("TOKEN=new\n"), 0o600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
m.asked = nil
|
|
report, _ := applyCarried(t, d, state, m, nil)
|
|
if !m.removed("app") || !m.did("docker run") {
|
|
t.Fatalf("a container reading an env-file the host did not write was not recreated when it changed: %v", m.asked)
|
|
}
|
|
if o := outcomeOf(report, "app.server"); o.Detail != "recreated: "+env+" changed" {
|
|
t.Errorf("the recreate did not name the file: %+v", o)
|
|
}
|
|
}
|
|
|
|
func TestAContainerIsRecreatedWhenAMountedSecretChanged(t *testing.T) {
|
|
// A rotated credential has the same shape as a moved port: the host writes the file the
|
|
// container mounts, and the process holds the value it was created with.
|
|
dir := t.TempDir()
|
|
secret := filepath.Join(dir, "db.secret")
|
|
declare := func(value string) *declaration.Declaration {
|
|
return parseTrusted(t, `{"declaration":1,"resources":[
|
|
{"id":"app.secret","type":"file","path":"`+secret+`","content":"`+value+`","mode":"0600"},
|
|
{"id":"app.server","type":"container","name":"app","image":"`+pinned+`",
|
|
"volumes":["`+secret+`:/run/secrets/db:ro"]}
|
|
]}`)
|
|
}
|
|
m := &machine{containers: map[string]*fakeContainer{}}
|
|
_, state := applyCarried(t, declare("hunter2"), store.State{}, m, nil)
|
|
|
|
m.asked = nil
|
|
report, _ := applyCarried(t, declare("correct-horse-battery-staple"), state, m, nil)
|
|
if !m.removed("app") || !m.did("docker run") {
|
|
t.Fatalf("the container kept the secret it was created with after the mounted file changed: %v", m.asked)
|
|
}
|
|
if o := outcomeOf(report, "app.server"); o.Action != "updated" || o.Detail != "recreated: "+secret+" changed" {
|
|
t.Errorf("the recreate did not say which file changed: %+v", o)
|
|
}
|
|
}
|
|
|
|
func TestAMountedDirectoryIsNotLookedInside(t *testing.T) {
|
|
// A bind-mounted directory is not part of what a container is — not the data the service
|
|
// grows in it, and not the files the host itself writes there either. Whether a service reads
|
|
// a file under its directory once at start or watches it live is the service's business: the
|
|
// route proxy re-reads its routes live, a provisioner sidecar polls what it receives every few
|
|
// seconds, and recreating either for a file the host rewrote would kill them for nothing. A
|
|
// module whose container does read such a file once says so with restart-on, which stays the
|
|
// opt-in.
|
|
dir := t.TempDir()
|
|
state := filepath.Join(dir, "state")
|
|
config := filepath.Join(state, "config.toml")
|
|
declare := func(level, restartOn string) *declaration.Declaration {
|
|
return parseTrusted(t, `{"declaration":1,"resources":[
|
|
{"id":"app.state","type":"directory","path":"`+state+`"},
|
|
{"id":"app.config","type":"file","path":"`+config+`","content":"level = \"`+level+`\"\n"},
|
|
{"id":"app.server","type":"container","name":"app","image":"`+pinned+`",
|
|
"volumes":["`+state+`:/var/lib/app"]`+restartOn+`}
|
|
]}`)
|
|
}
|
|
m := &machine{containers: map[string]*fakeContainer{}}
|
|
_, known := applyCarried(t, declare("info", ""), store.State{}, m, nil)
|
|
|
|
// The service grows its data in the directory it was given.
|
|
if err := os.WriteFile(filepath.Join(state, "app.db"), []byte("rows"), 0o600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := os.MkdirAll(filepath.Join(state, "cache"), 0o700); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := os.WriteFile(filepath.Join(state, "cache", "index"), []byte("entries"), 0o600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
m.asked = nil
|
|
report, known := applyCarried(t, declare("info", ""), known, m, nil)
|
|
if m.did("docker rm") || m.did("docker run") || report.Changed() {
|
|
t.Errorf("a container was recreated for data its service wrote in a mounted directory: %v %+v",
|
|
m.asked, report.Outcomes)
|
|
}
|
|
|
|
// The host rewrites its own file under the same directory: still not a reason. The container
|
|
// did not name it.
|
|
m.asked = nil
|
|
report, known = applyCarried(t, declare("debug", ""), known, m, nil)
|
|
if m.did("docker rm") || m.did("docker run") {
|
|
t.Errorf("a container was recreated for a file under a mounted directory it did not name: %v", m.asked)
|
|
}
|
|
if o := outcomeOf(report, "app.config"); o.Action != "updated" {
|
|
t.Fatalf("the config was not rewritten: %+v", o)
|
|
}
|
|
|
|
// Naming it is what makes it a reason, as before this change.
|
|
m.asked = nil
|
|
report, _ = applyCarried(t, declare("trace", `,"restart-on":["app.config"]`), known, m, nil)
|
|
if !m.removed("app") || !m.did("docker run") {
|
|
t.Fatalf("a container naming a rewritten file under its mount was not recreated: %v", m.asked)
|
|
}
|
|
if o := outcomeOf(report, "app.server"); !strings.Contains(o.Detail, "app.config") {
|
|
t.Errorf("the recreate did not name why: %+v", o)
|
|
}
|
|
}
|
|
|
|
func TestAContainerLabelledBeforeTheHostReadItsFilesIsAcceptedNotRecreated(t *testing.T) {
|
|
// The first apply after the host upgrades finds every container carrying a label computed
|
|
// without the file lines. Recreating them all would be a restart storm across the mesh in
|
|
// declaration order, the store first. A label that matches the spec as it used to be computed
|
|
// is accepted: what the container reads is recorded now, and from then on a change is caught.
|
|
dir := t.TempDir()
|
|
env := filepath.Join(dir, "forge.env")
|
|
declare := func(port string) *declaration.Declaration {
|
|
return parseTrusted(t, `{"declaration":1,"resources":[
|
|
{"id":"forge.env","type":"file","path":"`+env+`","content":"DATABASE_PORT=`+port+`\n","mode":"0600"},
|
|
{"id":"forge.server","type":"container","name":"forge","image":"`+pinned+`","env-file":["`+env+`"]}
|
|
]}`)
|
|
}
|
|
// The machine as the previous host left it: the file written and recorded, the container up
|
|
// under the label that host computed — the spec with nothing about the file's content.
|
|
if err := os.WriteFile(env, []byte("DATABASE_PORT=5432\n"), 0o600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
d := declare("5432")
|
|
legacy := containerSpecReading(d.Resources[1].(*declaration.Container), nil, nil)
|
|
known := store.State{}
|
|
known.Record(store.Applied{ID: "forge.env", Type: "file", Origin: store.OriginCarried, Target: env,
|
|
Wrote: digestOf("DATABASE_PORT=5432\n")})
|
|
known.Record(store.Applied{ID: "forge.server", Type: "container", Origin: store.OriginCarried, Target: "forge"})
|
|
m := &machine{containers: map[string]*fakeContainer{"forge": {id: "made-by-host", running: true, spec: legacy}}}
|
|
|
|
report, known := applyCarried(t, d, known, m, nil)
|
|
if m.did("docker rm") || m.did("docker run") || report.Changed() {
|
|
t.Fatalf("a container labelled by the previous host was recreated on upgrade: %v %+v", m.asked, report.Outcomes)
|
|
}
|
|
if got, _ := known.Find("forge.server"); got.Reads[env] != digestOf("DATABASE_PORT=5432\n") {
|
|
t.Fatalf("what the accepted container reads was not recorded: %+v", got)
|
|
}
|
|
// Accepted stays accepted: the next pass with nothing moved is quiet too.
|
|
m.asked = nil
|
|
report, known = applyCarried(t, d, known, m, nil)
|
|
if m.did("docker rm") || m.did("docker run") || report.Changed() {
|
|
t.Fatalf("an accepted container was recreated on the pass after: %v", m.asked)
|
|
}
|
|
|
|
// And a change to the file is caught from the record, and the label is renewed.
|
|
m.asked = nil
|
|
report, _ = applyCarried(t, declare("5433"), known, m, nil)
|
|
if !m.removed("forge") || !m.did("docker run") {
|
|
t.Fatalf("an accepted container was not recreated when its env-file changed: %v", m.asked)
|
|
}
|
|
if o := outcomeOf(report, "forge.server"); o.Detail != "recreated: "+env+" changed" {
|
|
t.Errorf("the recreate did not name the file: %+v", o)
|
|
}
|
|
if m.containers["forge"].spec == legacy {
|
|
t.Error("the recreated container still carries the legacy label")
|
|
}
|
|
}
|
|
|
|
func TestAContainerAdoptedUnderANewIdStillSaysWhichFileChanged(t *testing.T) {
|
|
// The bundle's `store` becomes the postgres module's `postgres.server`: the same container by
|
|
// name, under a new id with no record of its own. A file change on that day is a real change,
|
|
// and the record of what it read is under the old id — by name, it is found.
|
|
dir := t.TempDir()
|
|
env := filepath.Join(dir, "store.env")
|
|
m := &machine{containers: map[string]*fakeContainer{}}
|
|
raised := parseTrusted(t, `{"declaration":1,"resources":[
|
|
{"id":"env","type":"file","path":"`+env+`","content":"PORT=5432\n","mode":"0600"},
|
|
{"id":"store","type":"container","name":"mesh-store","image":"`+pinned+`","env-file":["`+env+`"]}
|
|
]}`)
|
|
_, known := applyCarried(t, raised, store.State{}, m, nil)
|
|
|
|
adopted := parseTrusted(t, `{"declaration":1,"resources":[
|
|
{"id":"postgres.env","type":"file","path":"`+env+`","content":"PORT=5433\n","mode":"0600"},
|
|
{"id":"postgres.server","type":"container","name":"mesh-store","image":"`+pinned+`","env-file":["`+env+`"]}
|
|
]}`)
|
|
m.asked = nil
|
|
report, _, err := Apply(context.Background(), archHost(t), adopted, known, store.OriginDeclared, m.run, nil, nil)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if o := outcomeOf(report, "postgres.server"); o.Action != "updated" || o.Detail != "recreated: "+env+" changed" {
|
|
t.Errorf("a container adopted under a new id did not say which file changed: %+v", o)
|
|
}
|
|
}
|
|
|
|
func TestAHeldContainerIsNotRecreatedByAChangedHeldFile(t *testing.T) {
|
|
// On an adopted node the predecessor's container and the file it reads are both held as
|
|
// found (novox/hq ADR 0100). The predecessor rewriting its own file is reported on the file
|
|
// — and is nothing to recreate the container for: it is not the host's to recreate.
|
|
dir := t.TempDir()
|
|
env := filepath.Join(dir, "hello.env")
|
|
if err := os.WriteFile(env, []byte("PORT=5432\n"), 0o600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
m := &machine{containers: map[string]*fakeContainer{
|
|
"hello-web": {id: "predecessor-id", running: true},
|
|
}}
|
|
d := adopted(t, untaken("hello-web.env", "hello-web.server"),
|
|
`{"id":"hello-web.env","type":"file","path":"`+env+`","content":"PORT=5433\n","mode":"0600"},
|
|
{"id":"hello-web.server","type":"container","name":"hello-web","image":"`+pinned+`","env-file":["`+env+`"]}`)
|
|
report, state := applyAdopted(t, d, store.State{}, m, dir)
|
|
if outcomeOf(report, "hello-web.env").Action != "held" || outcomeOf(report, "hello-web.server").Action != "held" {
|
|
t.Fatalf("the predecessor's file and container were not held: %+v", report.Outcomes)
|
|
}
|
|
|
|
if err := os.WriteFile(env, []byte("PORT=5434\n"), 0o600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
m.asked = nil
|
|
report, state = applyAdopted(t, d, state, m, dir)
|
|
for _, a := range m.asked {
|
|
if strings.HasPrefix(a, "docker run") || strings.HasPrefix(a, "docker rm") {
|
|
t.Fatalf("a held container was acted on because a held file changed: %s", a)
|
|
}
|
|
}
|
|
if o := outcomeOf(report, "hello-web.server"); o.Action != "held" {
|
|
t.Errorf("the container is no longer held: %+v", o)
|
|
}
|
|
if h, _ := state.HeldAt("hello-web.env"); h.Changed != "rewritten" {
|
|
t.Errorf("the predecessor's rewrite was not reported on the file: %+v", h)
|
|
}
|
|
if h, _ := state.HeldAt("hello-web.server"); h.Changed != "" {
|
|
t.Errorf("a file change was charged to the container: %+v", h)
|
|
}
|
|
}
|