package apply import ( "context" "os" "path/filepath" "strings" "testing" "github.com/novox/mesh-host/internal/declaration" "github.com/novox/mesh-host/internal/store" ) // Defends novox/hq 04-ISSUES/103: a container is recreated when the CONTENT of a file it reads at // creation changes, not only when its path does. A container takes its env-file and its mounted // files in once, when it is created; `docker restart` hands it the same environment again, so // only a recreate carries a rewritten file into the process. // // The runtime here is the `machine` fake: it keeps the spec label the host gave a container and // hands it back on inspect, so the comparison under test is the one the host really makes, // against what it really wrote — not against a spec a test imagined. func applyCarried(t *testing.T, d *declaration.Declaration, known store.State, m *machine, log func(string)) (Report, store.State) { t.Helper() report, state, err := Apply(context.Background(), archHost(t), d, known, store.OriginCarried, m.run, log, nil) if err != nil { t.Fatalf("apply failed: %v", err) } return report, state } func TestAContainerIsRecreatedWhenItsEnvFileChanged(t *testing.T) { // The night of the issue: the store was given a new port, the host rewrote the forge's // environment file with it — and left the forge running with the old one. dir := t.TempDir() env := filepath.Join(dir, "forge.env") declare := func(port string) *declaration.Declaration { return parseTrusted(t, `{"declaration":1,"resources":[ {"id":"forge.env","type":"file","path":"`+env+`","content":"DATABASE_PORT=`+port+`\n","mode":"0600"}, {"id":"forge.server","type":"container","name":"forge","image":"`+pinned+`","env-file":["`+env+`"]} ]}`) } m := &machine{containers: map[string]*fakeContainer{}} var logged []string log := func(line string) { logged = append(logged, line) } report, state := applyCarried(t, declare("5432"), store.State{}, m, log) if o := outcomeOf(report, "forge.server"); o.Action != "created" { t.Fatalf("the container was not created: %+v", report.Outcomes) } // The store moved. The file is rewritten in this apply, before the container is reached, and // the container must follow it in the same pass. m.asked, logged = nil, nil report, state = applyCarried(t, declare("5433"), state, m, log) if !m.removed("forge") || !m.did("docker run") { t.Fatalf("the container kept running with the old environment after its env-file changed: %v", m.asked) } o := outcomeOf(report, "forge.server") if o.Action != "updated" || o.Detail != "recreated: "+env+" changed" { t.Errorf("the recreate did not say which file changed: %+v", o) } var said bool for _, line := range logged { if strings.Contains(line, "updated forge.server") && strings.Contains(line, "recreated: "+env+" changed") { said = true } } if !said { t.Errorf("the log did not say which file made the container recreate: %q", logged) } // And with nothing moved, it is left alone: content is part of the identity, not a tripwire. m.asked = nil report, _ = applyCarried(t, declare("5433"), state, m, log) if m.did("docker rm") || m.did("docker run") || report.Changed() { t.Errorf("a container whose env-file did not change was recreated: %v %+v", m.asked, report.Outcomes) } } func TestAnEnvFileTheHostDidNotWriteIsStillReadForWhatItHolds(t *testing.T) { // The host has no record of this file — a predecessor left it, or something else on the // machine maintains it — and the container still reads it once. Its content is read from the // disk, so a change is a recreate exactly as for a file the host wrote. dir := t.TempDir() env := filepath.Join(dir, "app.env") if err := os.WriteFile(env, []byte("TOKEN=old\n"), 0o600); err != nil { t.Fatal(err) } d := parseTrusted(t, `{"declaration":1,"resources":[ {"id":"app.server","type":"container","name":"app","image":"`+pinned+`","env-file":["`+env+`"]} ]}`) m := &machine{containers: map[string]*fakeContainer{}} _, state := applyCarried(t, d, store.State{}, m, nil) if err := os.WriteFile(env, []byte("TOKEN=new\n"), 0o600); err != nil { t.Fatal(err) } m.asked = nil report, _ := applyCarried(t, d, state, m, nil) if !m.removed("app") || !m.did("docker run") { t.Fatalf("a container reading an env-file the host did not write was not recreated when it changed: %v", m.asked) } if o := outcomeOf(report, "app.server"); o.Detail != "recreated: "+env+" changed" { t.Errorf("the recreate did not name the file: %+v", o) } } func TestAContainerIsRecreatedWhenAMountedSecretChanged(t *testing.T) { // A rotated credential has the same shape as a moved port: the host writes the file the // container mounts, and the process holds the value it was created with. dir := t.TempDir() secret := filepath.Join(dir, "db.secret") declare := func(value string) *declaration.Declaration { return parseTrusted(t, `{"declaration":1,"resources":[ {"id":"app.secret","type":"file","path":"`+secret+`","content":"`+value+`","mode":"0600"}, {"id":"app.server","type":"container","name":"app","image":"`+pinned+`", "volumes":["`+secret+`:/run/secrets/db:ro"]} ]}`) } m := &machine{containers: map[string]*fakeContainer{}} _, state := applyCarried(t, declare("hunter2"), store.State{}, m, nil) m.asked = nil report, _ := applyCarried(t, declare("correct-horse-battery-staple"), state, m, nil) if !m.removed("app") || !m.did("docker run") { t.Fatalf("the container kept the secret it was created with after the mounted file changed: %v", m.asked) } if o := outcomeOf(report, "app.server"); o.Action != "updated" || o.Detail != "recreated: "+secret+" changed" { t.Errorf("the recreate did not say which file changed: %+v", o) } } func TestAMountedDirectoryIsNotLookedInside(t *testing.T) { // A bind-mounted directory is not part of what a container is — not the data the service // grows in it, and not the files the host itself writes there either. Whether a service reads // a file under its directory once at start or watches it live is the service's business: the // route proxy re-reads its routes live, a provisioner sidecar polls what it receives every few // seconds, and recreating either for a file the host rewrote would kill them for nothing. A // module whose container does read such a file once says so with restart-on, which stays the // opt-in. dir := t.TempDir() state := filepath.Join(dir, "state") config := filepath.Join(state, "config.toml") declare := func(level, restartOn string) *declaration.Declaration { return parseTrusted(t, `{"declaration":1,"resources":[ {"id":"app.state","type":"directory","path":"`+state+`"}, {"id":"app.config","type":"file","path":"`+config+`","content":"level = \"`+level+`\"\n"}, {"id":"app.server","type":"container","name":"app","image":"`+pinned+`", "volumes":["`+state+`:/var/lib/app"]`+restartOn+`} ]}`) } m := &machine{containers: map[string]*fakeContainer{}} _, known := applyCarried(t, declare("info", ""), store.State{}, m, nil) // The service grows its data in the directory it was given. if err := os.WriteFile(filepath.Join(state, "app.db"), []byte("rows"), 0o600); err != nil { t.Fatal(err) } if err := os.MkdirAll(filepath.Join(state, "cache"), 0o700); err != nil { t.Fatal(err) } if err := os.WriteFile(filepath.Join(state, "cache", "index"), []byte("entries"), 0o600); err != nil { t.Fatal(err) } m.asked = nil report, known := applyCarried(t, declare("info", ""), known, m, nil) if m.did("docker rm") || m.did("docker run") || report.Changed() { t.Errorf("a container was recreated for data its service wrote in a mounted directory: %v %+v", m.asked, report.Outcomes) } // The host rewrites its own file under the same directory: still not a reason. The container // did not name it. m.asked = nil report, known = applyCarried(t, declare("debug", ""), known, m, nil) if m.did("docker rm") || m.did("docker run") { t.Errorf("a container was recreated for a file under a mounted directory it did not name: %v", m.asked) } if o := outcomeOf(report, "app.config"); o.Action != "updated" { t.Fatalf("the config was not rewritten: %+v", o) } // Naming it is what makes it a reason, as before this change. m.asked = nil report, _ = applyCarried(t, declare("trace", `,"restart-on":["app.config"]`), known, m, nil) if !m.removed("app") || !m.did("docker run") { t.Fatalf("a container naming a rewritten file under its mount was not recreated: %v", m.asked) } if o := outcomeOf(report, "app.server"); !strings.Contains(o.Detail, "app.config") { t.Errorf("the recreate did not name why: %+v", o) } } func TestAContainerLabelledBeforeTheHostReadItsFilesIsAcceptedNotRecreated(t *testing.T) { // The first apply after the host upgrades finds every container carrying a label computed // without the file lines. Recreating them all would be a restart storm across the mesh in // declaration order, the store first. A label that matches the spec as it used to be computed // is accepted: what the container reads is recorded now, and from then on a change is caught. dir := t.TempDir() env := filepath.Join(dir, "forge.env") declare := func(port string) *declaration.Declaration { return parseTrusted(t, `{"declaration":1,"resources":[ {"id":"forge.env","type":"file","path":"`+env+`","content":"DATABASE_PORT=`+port+`\n","mode":"0600"}, {"id":"forge.server","type":"container","name":"forge","image":"`+pinned+`","env-file":["`+env+`"]} ]}`) } // The machine as the previous host left it: the file written and recorded, the container up // under the label that host computed — the spec with nothing about the file's content. if err := os.WriteFile(env, []byte("DATABASE_PORT=5432\n"), 0o600); err != nil { t.Fatal(err) } d := declare("5432") legacy := containerSpecReading(d.Resources[1].(*declaration.Container), nil, nil) known := store.State{} known.Record(store.Applied{ID: "forge.env", Type: "file", Origin: store.OriginCarried, Target: env, Wrote: digestOf("DATABASE_PORT=5432\n")}) known.Record(store.Applied{ID: "forge.server", Type: "container", Origin: store.OriginCarried, Target: "forge"}) m := &machine{containers: map[string]*fakeContainer{"forge": {id: "made-by-host", running: true, spec: legacy}}} report, known := applyCarried(t, d, known, m, nil) if m.did("docker rm") || m.did("docker run") || report.Changed() { t.Fatalf("a container labelled by the previous host was recreated on upgrade: %v %+v", m.asked, report.Outcomes) } if got, _ := known.Find("forge.server"); got.Reads[env] != digestOf("DATABASE_PORT=5432\n") { t.Fatalf("what the accepted container reads was not recorded: %+v", got) } // Accepted stays accepted: the next pass with nothing moved is quiet too. m.asked = nil report, known = applyCarried(t, d, known, m, nil) if m.did("docker rm") || m.did("docker run") || report.Changed() { t.Fatalf("an accepted container was recreated on the pass after: %v", m.asked) } // And a change to the file is caught from the record, and the label is renewed. m.asked = nil report, _ = applyCarried(t, declare("5433"), known, m, nil) if !m.removed("forge") || !m.did("docker run") { t.Fatalf("an accepted container was not recreated when its env-file changed: %v", m.asked) } if o := outcomeOf(report, "forge.server"); o.Detail != "recreated: "+env+" changed" { t.Errorf("the recreate did not name the file: %+v", o) } if m.containers["forge"].spec == legacy { t.Error("the recreated container still carries the legacy label") } } func TestAContainerAdoptedUnderANewIdStillSaysWhichFileChanged(t *testing.T) { // The bundle's `store` becomes the postgres module's `postgres.server`: the same container by // name, under a new id with no record of its own. A file change on that day is a real change, // and the record of what it read is under the old id — by name, it is found. dir := t.TempDir() env := filepath.Join(dir, "store.env") m := &machine{containers: map[string]*fakeContainer{}} raised := parseTrusted(t, `{"declaration":1,"resources":[ {"id":"env","type":"file","path":"`+env+`","content":"PORT=5432\n","mode":"0600"}, {"id":"store","type":"container","name":"mesh-store","image":"`+pinned+`","env-file":["`+env+`"]} ]}`) _, known := applyCarried(t, raised, store.State{}, m, nil) adopted := parseTrusted(t, `{"declaration":1,"resources":[ {"id":"postgres.env","type":"file","path":"`+env+`","content":"PORT=5433\n","mode":"0600"}, {"id":"postgres.server","type":"container","name":"mesh-store","image":"`+pinned+`","env-file":["`+env+`"]} ]}`) m.asked = nil report, _, err := Apply(context.Background(), archHost(t), adopted, known, store.OriginDeclared, m.run, nil, nil) if err != nil { t.Fatal(err) } if o := outcomeOf(report, "postgres.server"); o.Action != "updated" || o.Detail != "recreated: "+env+" changed" { t.Errorf("a container adopted under a new id did not say which file changed: %+v", o) } } func TestAHeldContainerIsNotRecreatedByAChangedHeldFile(t *testing.T) { // On an adopted node the predecessor's container and the file it reads are both held as // found (novox/hq ADR 0100). The predecessor rewriting its own file is reported on the file // — and is nothing to recreate the container for: it is not the host's to recreate. dir := t.TempDir() env := filepath.Join(dir, "hello.env") if err := os.WriteFile(env, []byte("PORT=5432\n"), 0o600); err != nil { t.Fatal(err) } m := &machine{containers: map[string]*fakeContainer{ "hello-web": {id: "predecessor-id", running: true}, }} d := adopted(t, untaken("hello-web.env", "hello-web.server"), `{"id":"hello-web.env","type":"file","path":"`+env+`","content":"PORT=5433\n","mode":"0600"}, {"id":"hello-web.server","type":"container","name":"hello-web","image":"`+pinned+`","env-file":["`+env+`"]}`) report, state := applyAdopted(t, d, store.State{}, m, dir) if outcomeOf(report, "hello-web.env").Action != "held" || outcomeOf(report, "hello-web.server").Action != "held" { t.Fatalf("the predecessor's file and container were not held: %+v", report.Outcomes) } if err := os.WriteFile(env, []byte("PORT=5434\n"), 0o600); err != nil { t.Fatal(err) } m.asked = nil report, state = applyAdopted(t, d, state, m, dir) for _, a := range m.asked { if strings.HasPrefix(a, "docker run") || strings.HasPrefix(a, "docker rm") { t.Fatalf("a held container was acted on because a held file changed: %s", a) } } if o := outcomeOf(report, "hello-web.server"); o.Action != "held" { t.Errorf("the container is no longer held: %+v", o) } if h, _ := state.HeldAt("hello-web.env"); h.Changed != "rewritten" { t.Errorf("the predecessor's rewrite was not reported on the file: %+v", h) } if h, _ := state.HeldAt("hello-web.server"); h.Changed != "" { t.Errorf("a file change was charged to the container: %+v", h) } }