jschoubben
982b84310e
Review of the first cut found four things. A directory mounted into a container is no longer looked inside, not even for the files this host wrote there. The controller records every provider's received and contributions file as a plain file under a mounted directory, so folding those in would have recreated the route proxy — which re-reads its routes live, by design — on every route change, and killed every provisioner sidecar, which polls what it receives, mid-reconcile on every grant. Whether a service reads a file under its directory once or watches it is the service's; restart-on is how a module says "once", and it stays the opt-in. Env-files and files mounted directly remain by content. Genesis wrote the superuser secret as `value\n`; `secret accept` strips the line ending by design, so the postgres module declared `value` — and with a mounted file's content in the spec, phase three would have recreated the store it meant to adopt in place, with the temporary control plane connected to it. Genesis now writes the value alone. readCredentialFile tolerated both endings already. Pinned with the bytes the genesis code path writes, then the module's declaration of the same container: it must reconcile. A container carrying a label from before the host folded in what it reads is accepted rather than recreated, when that label matches the spec as it used to be computed: what it reads is recorded then, a change is caught from that record from the next apply on, and the label is renewed at the next genuine recreate. Recreating them all would have been a restart storm across the mesh in declaration order, the store first. The trade-off is stated in the code: a container already stale at upgrade time is not caught, and could not have been either way. The record of what a container read is looked up by its name when its declared id has none — the bundle's `store` becomes `postgres.server` for the same container — so a change on the day it is adopted still names the file. The by-target lookup takes the most recently applied record, since the bundle's record for the same target is never removed by the mesh's. novox/hq 04-ISSUES/103
317 lines
15 KiB
Go
317 lines
15 KiB
Go
package apply
|
|
|
|
import (
|
|
"context"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"testing"
|
|
|
|
"github.com/novox/mesh-host/internal/declaration"
|
|
"github.com/novox/mesh-host/internal/store"
|
|
)
|
|
|
|
// Defends novox/hq 04-ISSUES/103: a container is recreated when the CONTENT of a file it reads at
|
|
// creation changes, not only when its path does. A container takes its env-file and its mounted
|
|
// files in once, when it is created; `docker restart` hands it the same environment again, so
|
|
// only a recreate carries a rewritten file into the process.
|
|
//
|
|
// The runtime here is the `machine` fake: it keeps the spec label the host gave a container and
|
|
// hands it back on inspect, so the comparison under test is the one the host really makes,
|
|
// against what it really wrote — not against a spec a test imagined.
|
|
|
|
func applyCarried(t *testing.T, d *declaration.Declaration, known store.State, m *machine,
|
|
log func(string)) (Report, store.State) {
|
|
t.Helper()
|
|
report, state, err := Apply(context.Background(), archHost(t), d, known, store.OriginCarried, m.run, log, nil)
|
|
if err != nil {
|
|
t.Fatalf("apply failed: %v", err)
|
|
}
|
|
return report, state
|
|
}
|
|
|
|
func TestAContainerIsRecreatedWhenItsEnvFileChanged(t *testing.T) {
|
|
// The night of the issue: the store was given a new port, the host rewrote the forge's
|
|
// environment file with it — and left the forge running with the old one.
|
|
dir := t.TempDir()
|
|
env := filepath.Join(dir, "forge.env")
|
|
declare := func(port string) *declaration.Declaration {
|
|
return parseTrusted(t, `{"declaration":1,"resources":[
|
|
{"id":"forge.env","type":"file","path":"`+env+`","content":"DATABASE_PORT=`+port+`\n","mode":"0600"},
|
|
{"id":"forge.server","type":"container","name":"forge","image":"`+pinned+`","env-file":["`+env+`"]}
|
|
]}`)
|
|
}
|
|
m := &machine{containers: map[string]*fakeContainer{}}
|
|
var logged []string
|
|
log := func(line string) { logged = append(logged, line) }
|
|
|
|
report, state := applyCarried(t, declare("5432"), store.State{}, m, log)
|
|
if o := outcomeOf(report, "forge.server"); o.Action != "created" {
|
|
t.Fatalf("the container was not created: %+v", report.Outcomes)
|
|
}
|
|
|
|
// The store moved. The file is rewritten in this apply, before the container is reached, and
|
|
// the container must follow it in the same pass.
|
|
m.asked, logged = nil, nil
|
|
report, state = applyCarried(t, declare("5433"), state, m, log)
|
|
if !m.removed("forge") || !m.did("docker run") {
|
|
t.Fatalf("the container kept running with the old environment after its env-file changed: %v", m.asked)
|
|
}
|
|
o := outcomeOf(report, "forge.server")
|
|
if o.Action != "updated" || o.Detail != "recreated: "+env+" changed" {
|
|
t.Errorf("the recreate did not say which file changed: %+v", o)
|
|
}
|
|
var said bool
|
|
for _, line := range logged {
|
|
if strings.Contains(line, "updated forge.server") && strings.Contains(line, "recreated: "+env+" changed") {
|
|
said = true
|
|
}
|
|
}
|
|
if !said {
|
|
t.Errorf("the log did not say which file made the container recreate: %q", logged)
|
|
}
|
|
|
|
// And with nothing moved, it is left alone: content is part of the identity, not a tripwire.
|
|
m.asked = nil
|
|
report, _ = applyCarried(t, declare("5433"), state, m, log)
|
|
if m.did("docker rm") || m.did("docker run") || report.Changed() {
|
|
t.Errorf("a container whose env-file did not change was recreated: %v %+v", m.asked, report.Outcomes)
|
|
}
|
|
}
|
|
|
|
func TestAnEnvFileTheHostDidNotWriteIsStillReadForWhatItHolds(t *testing.T) {
|
|
// The host has no record of this file — a predecessor left it, or something else on the
|
|
// machine maintains it — and the container still reads it once. Its content is read from the
|
|
// disk, so a change is a recreate exactly as for a file the host wrote.
|
|
dir := t.TempDir()
|
|
env := filepath.Join(dir, "app.env")
|
|
if err := os.WriteFile(env, []byte("TOKEN=old\n"), 0o600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
d := parseTrusted(t, `{"declaration":1,"resources":[
|
|
{"id":"app.server","type":"container","name":"app","image":"`+pinned+`","env-file":["`+env+`"]}
|
|
]}`)
|
|
m := &machine{containers: map[string]*fakeContainer{}}
|
|
_, state := applyCarried(t, d, store.State{}, m, nil)
|
|
|
|
if err := os.WriteFile(env, []byte("TOKEN=new\n"), 0o600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
m.asked = nil
|
|
report, _ := applyCarried(t, d, state, m, nil)
|
|
if !m.removed("app") || !m.did("docker run") {
|
|
t.Fatalf("a container reading an env-file the host did not write was not recreated when it changed: %v", m.asked)
|
|
}
|
|
if o := outcomeOf(report, "app.server"); o.Detail != "recreated: "+env+" changed" {
|
|
t.Errorf("the recreate did not name the file: %+v", o)
|
|
}
|
|
}
|
|
|
|
func TestAContainerIsRecreatedWhenAMountedSecretChanged(t *testing.T) {
|
|
// A rotated credential has the same shape as a moved port: the host writes the file the
|
|
// container mounts, and the process holds the value it was created with.
|
|
dir := t.TempDir()
|
|
secret := filepath.Join(dir, "db.secret")
|
|
declare := func(value string) *declaration.Declaration {
|
|
return parseTrusted(t, `{"declaration":1,"resources":[
|
|
{"id":"app.secret","type":"file","path":"`+secret+`","content":"`+value+`","mode":"0600"},
|
|
{"id":"app.server","type":"container","name":"app","image":"`+pinned+`",
|
|
"volumes":["`+secret+`:/run/secrets/db:ro"]}
|
|
]}`)
|
|
}
|
|
m := &machine{containers: map[string]*fakeContainer{}}
|
|
_, state := applyCarried(t, declare("hunter2"), store.State{}, m, nil)
|
|
|
|
m.asked = nil
|
|
report, _ := applyCarried(t, declare("correct-horse-battery-staple"), state, m, nil)
|
|
if !m.removed("app") || !m.did("docker run") {
|
|
t.Fatalf("the container kept the secret it was created with after the mounted file changed: %v", m.asked)
|
|
}
|
|
if o := outcomeOf(report, "app.server"); o.Action != "updated" || o.Detail != "recreated: "+secret+" changed" {
|
|
t.Errorf("the recreate did not say which file changed: %+v", o)
|
|
}
|
|
}
|
|
|
|
func TestAMountedDirectoryIsNotLookedInside(t *testing.T) {
|
|
// A bind-mounted directory is not part of what a container is — not the data the service
|
|
// grows in it, and not the files the host itself writes there either. Whether a service reads
|
|
// a file under its directory once at start or watches it live is the service's business: the
|
|
// route proxy re-reads its routes live, a provisioner sidecar polls what it receives every few
|
|
// seconds, and recreating either for a file the host rewrote would kill them for nothing. A
|
|
// module whose container does read such a file once says so with restart-on, which stays the
|
|
// opt-in.
|
|
dir := t.TempDir()
|
|
state := filepath.Join(dir, "state")
|
|
config := filepath.Join(state, "config.toml")
|
|
declare := func(level, restartOn string) *declaration.Declaration {
|
|
return parseTrusted(t, `{"declaration":1,"resources":[
|
|
{"id":"app.state","type":"directory","path":"`+state+`"},
|
|
{"id":"app.config","type":"file","path":"`+config+`","content":"level = \"`+level+`\"\n"},
|
|
{"id":"app.server","type":"container","name":"app","image":"`+pinned+`",
|
|
"volumes":["`+state+`:/var/lib/app"]`+restartOn+`}
|
|
]}`)
|
|
}
|
|
m := &machine{containers: map[string]*fakeContainer{}}
|
|
_, known := applyCarried(t, declare("info", ""), store.State{}, m, nil)
|
|
|
|
// The service grows its data in the directory it was given.
|
|
if err := os.WriteFile(filepath.Join(state, "app.db"), []byte("rows"), 0o600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := os.MkdirAll(filepath.Join(state, "cache"), 0o700); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := os.WriteFile(filepath.Join(state, "cache", "index"), []byte("entries"), 0o600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
m.asked = nil
|
|
report, known := applyCarried(t, declare("info", ""), known, m, nil)
|
|
if m.did("docker rm") || m.did("docker run") || report.Changed() {
|
|
t.Errorf("a container was recreated for data its service wrote in a mounted directory: %v %+v",
|
|
m.asked, report.Outcomes)
|
|
}
|
|
|
|
// The host rewrites its own file under the same directory: still not a reason. The container
|
|
// did not name it.
|
|
m.asked = nil
|
|
report, known = applyCarried(t, declare("debug", ""), known, m, nil)
|
|
if m.did("docker rm") || m.did("docker run") {
|
|
t.Errorf("a container was recreated for a file under a mounted directory it did not name: %v", m.asked)
|
|
}
|
|
if o := outcomeOf(report, "app.config"); o.Action != "updated" {
|
|
t.Fatalf("the config was not rewritten: %+v", o)
|
|
}
|
|
|
|
// Naming it is what makes it a reason, as before this change.
|
|
m.asked = nil
|
|
report, _ = applyCarried(t, declare("trace", `,"restart-on":["app.config"]`), known, m, nil)
|
|
if !m.removed("app") || !m.did("docker run") {
|
|
t.Fatalf("a container naming a rewritten file under its mount was not recreated: %v", m.asked)
|
|
}
|
|
if o := outcomeOf(report, "app.server"); !strings.Contains(o.Detail, "app.config") {
|
|
t.Errorf("the recreate did not name why: %+v", o)
|
|
}
|
|
}
|
|
|
|
func TestAContainerLabelledBeforeTheHostReadItsFilesIsAcceptedNotRecreated(t *testing.T) {
|
|
// The first apply after the host upgrades finds every container carrying a label computed
|
|
// without the file lines. Recreating them all would be a restart storm across the mesh in
|
|
// declaration order, the store first. A label that matches the spec as it used to be computed
|
|
// is accepted: what the container reads is recorded now, and from then on a change is caught.
|
|
dir := t.TempDir()
|
|
env := filepath.Join(dir, "forge.env")
|
|
declare := func(port string) *declaration.Declaration {
|
|
return parseTrusted(t, `{"declaration":1,"resources":[
|
|
{"id":"forge.env","type":"file","path":"`+env+`","content":"DATABASE_PORT=`+port+`\n","mode":"0600"},
|
|
{"id":"forge.server","type":"container","name":"forge","image":"`+pinned+`","env-file":["`+env+`"]}
|
|
]}`)
|
|
}
|
|
// The machine as the previous host left it: the file written and recorded, the container up
|
|
// under the label that host computed — the spec with nothing about the file's content.
|
|
if err := os.WriteFile(env, []byte("DATABASE_PORT=5432\n"), 0o600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
d := declare("5432")
|
|
legacy := containerSpecReading(d.Resources[1].(*declaration.Container), nil, nil)
|
|
known := store.State{}
|
|
known.Record(store.Applied{ID: "forge.env", Type: "file", Origin: store.OriginCarried, Target: env,
|
|
Wrote: digestOf("DATABASE_PORT=5432\n")})
|
|
known.Record(store.Applied{ID: "forge.server", Type: "container", Origin: store.OriginCarried, Target: "forge"})
|
|
m := &machine{containers: map[string]*fakeContainer{"forge": {id: "made-by-host", running: true, spec: legacy}}}
|
|
|
|
report, known := applyCarried(t, d, known, m, nil)
|
|
if m.did("docker rm") || m.did("docker run") || report.Changed() {
|
|
t.Fatalf("a container labelled by the previous host was recreated on upgrade: %v %+v", m.asked, report.Outcomes)
|
|
}
|
|
if got, _ := known.Find("forge.server"); got.Reads[env] != digestOf("DATABASE_PORT=5432\n") {
|
|
t.Fatalf("what the accepted container reads was not recorded: %+v", got)
|
|
}
|
|
// Accepted stays accepted: the next pass with nothing moved is quiet too.
|
|
m.asked = nil
|
|
report, known = applyCarried(t, d, known, m, nil)
|
|
if m.did("docker rm") || m.did("docker run") || report.Changed() {
|
|
t.Fatalf("an accepted container was recreated on the pass after: %v", m.asked)
|
|
}
|
|
|
|
// And a change to the file is caught from the record, and the label is renewed.
|
|
m.asked = nil
|
|
report, _ = applyCarried(t, declare("5433"), known, m, nil)
|
|
if !m.removed("forge") || !m.did("docker run") {
|
|
t.Fatalf("an accepted container was not recreated when its env-file changed: %v", m.asked)
|
|
}
|
|
if o := outcomeOf(report, "forge.server"); o.Detail != "recreated: "+env+" changed" {
|
|
t.Errorf("the recreate did not name the file: %+v", o)
|
|
}
|
|
if m.containers["forge"].spec == legacy {
|
|
t.Error("the recreated container still carries the legacy label")
|
|
}
|
|
}
|
|
|
|
func TestAContainerAdoptedUnderANewIdStillSaysWhichFileChanged(t *testing.T) {
|
|
// The bundle's `store` becomes the postgres module's `postgres.server`: the same container by
|
|
// name, under a new id with no record of its own. A file change on that day is a real change,
|
|
// and the record of what it read is under the old id — by name, it is found.
|
|
dir := t.TempDir()
|
|
env := filepath.Join(dir, "store.env")
|
|
m := &machine{containers: map[string]*fakeContainer{}}
|
|
raised := parseTrusted(t, `{"declaration":1,"resources":[
|
|
{"id":"env","type":"file","path":"`+env+`","content":"PORT=5432\n","mode":"0600"},
|
|
{"id":"store","type":"container","name":"mesh-store","image":"`+pinned+`","env-file":["`+env+`"]}
|
|
]}`)
|
|
_, known := applyCarried(t, raised, store.State{}, m, nil)
|
|
|
|
adopted := parseTrusted(t, `{"declaration":1,"resources":[
|
|
{"id":"postgres.env","type":"file","path":"`+env+`","content":"PORT=5433\n","mode":"0600"},
|
|
{"id":"postgres.server","type":"container","name":"mesh-store","image":"`+pinned+`","env-file":["`+env+`"]}
|
|
]}`)
|
|
m.asked = nil
|
|
report, _, err := Apply(context.Background(), archHost(t), adopted, known, store.OriginDeclared, m.run, nil, nil)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if o := outcomeOf(report, "postgres.server"); o.Action != "updated" || o.Detail != "recreated: "+env+" changed" {
|
|
t.Errorf("a container adopted under a new id did not say which file changed: %+v", o)
|
|
}
|
|
}
|
|
|
|
func TestAHeldContainerIsNotRecreatedByAChangedHeldFile(t *testing.T) {
|
|
// On an adopted node the predecessor's container and the file it reads are both held as
|
|
// found (novox/hq ADR 0100). The predecessor rewriting its own file is reported on the file
|
|
// — and is nothing to recreate the container for: it is not the host's to recreate.
|
|
dir := t.TempDir()
|
|
env := filepath.Join(dir, "hello.env")
|
|
if err := os.WriteFile(env, []byte("PORT=5432\n"), 0o600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
m := &machine{containers: map[string]*fakeContainer{
|
|
"hello-web": {id: "predecessor-id", running: true},
|
|
}}
|
|
d := adopted(t, untaken("hello-web.env", "hello-web.server"),
|
|
`{"id":"hello-web.env","type":"file","path":"`+env+`","content":"PORT=5433\n","mode":"0600"},
|
|
{"id":"hello-web.server","type":"container","name":"hello-web","image":"`+pinned+`","env-file":["`+env+`"]}`)
|
|
report, state := applyAdopted(t, d, store.State{}, m, dir)
|
|
if outcomeOf(report, "hello-web.env").Action != "held" || outcomeOf(report, "hello-web.server").Action != "held" {
|
|
t.Fatalf("the predecessor's file and container were not held: %+v", report.Outcomes)
|
|
}
|
|
|
|
if err := os.WriteFile(env, []byte("PORT=5434\n"), 0o600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
m.asked = nil
|
|
report, state = applyAdopted(t, d, state, m, dir)
|
|
for _, a := range m.asked {
|
|
if strings.HasPrefix(a, "docker run") || strings.HasPrefix(a, "docker rm") {
|
|
t.Fatalf("a held container was acted on because a held file changed: %s", a)
|
|
}
|
|
}
|
|
if o := outcomeOf(report, "hello-web.server"); o.Action != "held" {
|
|
t.Errorf("the container is no longer held: %+v", o)
|
|
}
|
|
if h, _ := state.HeldAt("hello-web.env"); h.Changed != "rewritten" {
|
|
t.Errorf("the predecessor's rewrite was not reported on the file: %+v", h)
|
|
}
|
|
if h, _ := state.HeldAt("hello-web.server"); h.Changed != "" {
|
|
t.Errorf("a file change was charged to the container: %+v", h)
|
|
}
|
|
}
|