A one-shot service that finished is not stopped, and a container is what it reads

Two faults that both reported success while being wrong, found while proving
the firewall module actually delivers.

A unit whose job is to apply something and exit — load a rule set, set a
sysctl — is inactive the instant it succeeds. Reading that as stopped made it
permanently unsatisfiable: the host started it, it worked, the host read back
stopped and reported the machine as not doing what it was told, on every apply,
for ever, with the rules correctly in place the whole time. That is what the
firewall has been doing on every machine it was assigned to, and why the
four-machine bed was red.

And a container took its identity from its own fields, not from the files it
reads. A file written in an earlier apply — or before the container declared it
as a dependency — left a process holding a credential the mesh had already
replaced, with everything reporting success (novox/hq 04-ISSUES/045). What a
container reads is now part of what it is, so the comparison is a standing one
rather than a tripwire that fires during one apply and never again.

internal/apply/apply_test.go

@@ -666,7 +666,7 @@ func TestAContainerWhoseDeclarationChangedIsReplaced(t *testing.T) {
 	d := parseTrusted(t, `{"declaration":1,"resources":[
 	  {"id":"store","type":"container","name":"store","image":"`+pinned+`","env":{"PGDATA":"/data"}}
 	]}`)
-	want := containerSpec(d.Resources[0].(*declaration.Container))
+	want := containerSpec(d.Resources[0].(*declaration.Container), nil)
 
 	var removed, created bool
 	run := func(ctx context.Context, name string, args ...string) (string, error) {
@@ -704,7 +704,7 @@ func TestAContainerThatMatchesIsLeftAlone(t *testing.T) {
 	d := parseTrusted(t, `{"declaration":1,"resources":[
 	  {"id":"store","type":"container","name":"store","image":"`+pinned+`","env":{"PGDATA":"/data"}}
 	]}`)
-	spec := containerSpec(d.Resources[0].(*declaration.Container))
+	spec := containerSpec(d.Resources[0].(*declaration.Container), nil)
 
 	var touched bool
 	run := func(ctx context.Context, name string, args ...string) (string, error) {
@@ -743,7 +743,13 @@ func TestAContainerIsRecreatedWhenARestartOnResourceChanged(t *testing.T) {
 	  {"id":"config","type":"file","path":"`+conf+`","content":"{\"token\":\"new\"}\n"},
 	  {"id":"app","type":"container","name":"app","image":"`+pinned+`","restart-on":["config"]}
 	]}`)
-	spec := containerSpec(d.Resources[1].(*declaration.Container))
+	// The spec the container was created with, back when the file said something else. Built the
+	// way the host builds it, so this is the real comparison rather than a hand-written string:
+	// what a container reads is part of what it is, so the old content yields a different spec.
+	was := map[string]string{"config": declaredDigest(&declaration.File{Content: "{\"token\":\"old\"}\n"})}
+	now := map[string]string{"config": declaredDigest(d.Resources[0].(*declaration.File))}
+	stale := containerSpec(d.Resources[1].(*declaration.Container), was)
+	fresh := containerSpec(d.Resources[1].(*declaration.Container), now)
 
 	var removed, created bool
 	run := func(ctx context.Context, name string, args ...string) (string, error) {
@@ -751,9 +757,12 @@ func TestAContainerIsRecreatedWhenARestartOnResourceChanged(t *testing.T) {
 		case "info":
 			return "27.0\n", nil
 		case "inspect":
-			// The container already exists, running, with exactly the spec it is declared with —
-			// only the mounted file changed.
-			return "true\t" + spec, nil
+			// Already there and running, created against the file as it was. After the host
+			// recreates it, the runtime holds the one it just made — as a real one would.
+			if created {
+				return "true\t" + fresh, nil
+			}
+			return "true\t" + stale, nil
 		case "rm":
 			removed = true
 			return "", nil
@@ -1492,3 +1501,60 @@ func TestAnEmptyDirectoryIsStillRemoved(t *testing.T) {
 		t.Fatal("an empty directory the mesh made was left behind, so nothing is ever cleaned up")
 	}
 }
+
+// A container holding values from before is replaced, even when nothing changed this pass.
+//
+// **This is the fault the restart-on tripwire could not catch** (novox/hq 04-ISSUES/045). That
+// mechanism fires while a resource is being changed, so it covers the apply where the file moved
+// and nothing afterwards. A file written in an earlier apply — or written before the container
+// declared it as a dependency — leaves a process holding a credential the mesh has already
+// replaced, and every check passes: the container is up, the spec matched, the machine reported
+// success. Making what a container reads part of what it is turns that from a tripwire into a
+// standing comparison.
+func TestAContainerStaleFromAnEarlierApplyIsReplaced(t *testing.T) {
+	dir := t.TempDir()
+	conf := filepath.Join(dir, "db.env")
+	// Already on disk with the current content, so this apply changes nothing at all.
+	if err := os.WriteFile(conf, []byte("PASSWORD=new\n"), 0o600); err != nil {
+		t.Fatal(err)
+	}
+	d := parseTrusted(t, `{"declaration":1,"resources":[
+	  {"id":"env","type":"file","path":"`+conf+`","content":"PASSWORD=new\n","mode":"0600"},
+	  {"id":"app","type":"container","name":"app","image":"`+pinned+`","restart-on":["env"]}
+	]}`)
+
+	// The container was created when the file said something else.
+	was := map[string]string{"env": declaredDigest(&declaration.File{Content: "PASSWORD=old\n"})}
+	now := map[string]string{"env": declaredDigest(d.Resources[0].(*declaration.File))}
+	stale := containerSpec(d.Resources[1].(*declaration.Container), was)
+	fresh := containerSpec(d.Resources[1].(*declaration.Container), now)
+
+	var removed, created bool
+	run := func(ctx context.Context, name string, args ...string) (string, error) {
+		switch args[0] {
+		case "info":
+			return "27.0\n", nil
+		case "inspect":
+			if created {
+				return "true\t" + fresh, nil
+			}
+			return "true\t" + stale, nil
+		case "rm":
+			removed = true
+			return "", nil
+		case "run":
+			created = true
+			return "deadbeef\n", nil
+		}
+		return "", nil
+	}
+
+	_, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, run, nil, nil)
+	if err != nil {
+		t.Fatalf("apply failed: %v", err)
+	}
+	if !removed || !created {
+		t.Fatalf("a container running values the machine no longer holds was left alone "+
+			"(removed=%v created=%v)", removed, created)
+	}
+}