Take a container left set aside as the next apply's, and bound an apply to 30 minutes (issue 1945)

internal/apply/apply.go

@@ -2965,6 +2965,13 @@ func applyContainer(ctx context.Context, r *declaration.Container, run Runner,
 	}
 
 	aside := ""
+	if !existed {
+		// A container an engine set aside and then stopped before it ran the replacement (it died between the two):
+		// the set-aside one of this apply, put back if the replacement fails and removed once it runs (issue 1945).
+		if _, err := containerState(ctx, r.Name+asideSuffix, run); err == nil {
+			aside = r.Name + asideSuffix
+		}
+	}
 	switch {
 	case existed && (before.Spec == want || legacy) && before.Running && len(reasons) == 0:
 		out.Action = "unchanged"

internal/apply/containers_kept_test.go

@@ -20,6 +20,9 @@ func TestAContainerJoinsTheNetworksItKeeps(t *testing.T) {
 		if name != "docker" {
 			return "", errors.New("not installed")
 		}
+		if len(args) > 0 && args[0] == "container" && strings.HasSuffix(args[len(args)-1], asideSuffix) {
+			return "", errors.New("no such container") // nothing was left set aside
+		}
 		ran = append(ran, strings.Join(args, " "))
 		switch args[0] {
 		case "info":

internal/apply/replace_image_test.go

@@ -162,3 +162,58 @@ func TestAReplacementThatRunsRemovesTheContainerSetAside(t *testing.T) {
 		t.Fatalf("the steps were %q: the set-aside container is removed after the replacement runs", got)
 	}
 }
+
+// A container left set aside by an engine that stopped between setting it aside and running its replacement is the
+// set-aside one of the next apply: put back when the replacement fails, removed once it runs (the review of issue
+// 1945).
+func TestAContainerLeftAsideIsPutBackOrRemovedByTheNextApply(t *testing.T) {
+	d := parseTrusted(t, `{"declaration":1,"resources":[
+	  {"id":"store","type":"container","name":"store","image":"`+pinned+`","env":{"PGDATA":"/data"}}
+	]}`)
+	want := containerSpec(d.Resources[0].(*declaration.Container), inputs{})
+	for _, failRun := range []bool{true, false} {
+		var order []string
+		created := false
+		run := func(ctx context.Context, name string, args ...string) (string, error) {
+			switch args[0] {
+			case "info":
+				return "27.0\n", nil
+			case "container":
+				switch args[len(args)-1] {
+				case "store":
+					if created {
+						return "true\t" + want, nil
+					}
+					return "", errors.New("no such container")
+				case "store.replaced":
+					return "false\tsome-older-spec", nil
+				}
+				return "", errors.New("no such container")
+			case "image":
+				return "", nil
+			case "rm", "rename", "stop", "start":
+				order = append(order, strings.Join(args, " "))
+				return "", nil
+			case "run":
+				order = append(order, "run")
+				if failRun {
+					return "", errors.New("port is already allocated")
+				}
+				created = true
+				return "deadbeef\n", nil
+			}
+			return "", nil
+		}
+		_, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, run, nil, nil)
+		got := strings.Join(order, "; ")
+		if failRun {
+			if err == nil || !strings.Contains(got, "rename store.replaced store; start store") {
+				t.Fatalf("a failed run beside a container left aside: %v; steps %q", err, got)
+			}
+			continue
+		}
+		if err != nil || !strings.Contains(got, "run; rm -f store.replaced") {
+			t.Fatalf("a run beside a container left aside: %v; steps %q", err, got)
+		}
+	}
+}