Fail a resource when the container runtime cannot answer, instead of reading silence as nothing there (hq ADR 0100)

internal/apply/hold_test.go

@@ -825,3 +825,82 @@ func TestMountingTheMachinesOwnPlumbingIsNotFoundData(t *testing.T) {
 		t.Errorf("held: %+v", state.Held)
 	}
 }
+
+// Defends novox/hq ADR 0100: a runtime that cannot answer is not a machine with nothing there.
+
+// unreachable is a machine whose container runtime answers everything with a daemon that is down.
+type unreachable struct{ asked []string }
+
+func (u *unreachable) run(_ context.Context, name string, args ...string) (string, error) {
+	u.asked = append(u.asked, name+" "+strings.Join(args, " "))
+	if name == "docker" && args[0] == "info" {
+		return "27.0\n", nil
+	}
+	if name == "docker" {
+		return "", errors.New("docker exited 1: Cannot connect to the Docker daemon at unix:///var/run/docker.sock")
+	}
+	return "", nil
+}
+
+func TestARuntimeThatCannotAnswerNeverLetsTheMeshCreateOverAFoundContainer(t *testing.T) {
+	dir := t.TempDir()
+	u := &unreachable{}
+	d := adopted(t, untaken("hello-web.server"),
+		`{"id":"hello-web.server","type":"container","name":"hello-web","image":"`+pinned+`"}`)
+	_, state, err := ApplyKeeping(context.Background(), archHost(t), d, store.State{}, store.OriginDeclared,
+		u.run, nil, nil, KeepIn(dir))
+	if err == nil || !strings.Contains(err.Error(), "not safe to make one") {
+		t.Fatalf("a runtime that could not answer was read as nothing there: %v", err)
+	}
+	for _, a := range u.asked {
+		if strings.HasPrefix(a, "docker run") || strings.HasPrefix(a, "docker rm") {
+			t.Errorf("the mesh acted on a container it could not ask about: %s", a)
+		}
+	}
+	if len(state.Held) != 0 {
+		t.Errorf("something was held on an answer the machine never gave: %+v", state.Held)
+	}
+}
+
+func TestAHeldContainerStaysHeldWhenTheRuntimeCannotAnswer(t *testing.T) {
+	dir, page, m := predecessor(t)
+	d := adopted(t, untaken("hello-web.page", "hello-web.server"), webResources(page))
+	_, state := applyAdopted(t, d, store.State{}, m, dir)
+	if _, held := state.HeldAt("hello-web.server"); !held {
+		t.Fatal("the found container was not held to begin with")
+	}
+	u := &unreachable{}
+	_, state, err := ApplyKeeping(context.Background(), archHost(t), d, state, store.OriginDeclared,
+		u.run, nil, nil, KeepIn(dir))
+	if err == nil {
+		t.Fatal("a runtime that could not answer reported success")
+	}
+	if h, held := state.HeldAt("hello-web.server"); !held || h.Changed != "" {
+		t.Errorf("a hold was let go or called changed on an answer the machine never gave: %+v", h)
+	}
+}
+
+func TestAVolumeTheRuntimeCannotBeAskedAboutStopsTheContainer(t *testing.T) {
+	dir := t.TempDir()
+	run := func(_ context.Context, name string, args ...string) (string, error) {
+		switch {
+		case name == "docker" && args[0] == "info":
+			return "27.0\n", nil
+		case name == "docker" && args[0] == "volume":
+			return "", errors.New("docker exited 1: Cannot connect to the Docker daemon")
+		case name == "docker" && args[0] == "inspect":
+			return "", errors.New("Error: No such object: hello-web")
+		case name == "docker":
+			return "", errors.New("docker run must not happen")
+		}
+		return "", nil
+	}
+	d := adopted(t, untaken("hello-web.server"),
+		`{"id":"hello-web.server","type":"container","name":"hello-web","image":"`+pinned+`",
+		  "volumes":["predecessor-data:/data"]}`)
+	_, _, err := ApplyKeeping(context.Background(), archHost(t), d, store.State{}, store.OriginDeclared,
+		run, nil, nil, KeepIn(dir))
+	if err == nil || !strings.Contains(err.Error(), "could not say whether the volume") {
+		t.Fatalf("a volume the runtime could not be asked about did not stop the container: %v", err)
+	}
+}