Skip the docker tests where the daemon is unreachable, not only where the binary is missing

The checker's integration level runs shut off, so docker is found but its daemon
is not, and TestTheDeclaredDumpRunsInTheBussImage failed the group check on g14
(issue 1823). MESH_REQUIRE_DOCKER=1 still fails.

modules/nats/cmd/mesh-nats-snapshot/dockerguard_test.go

@@ -0,0 +1,53 @@
+package main
+
+import (
+	"fmt"
+	"testing"
+)
+
+// dockerVerdict decides what a test that starts containers does: run, skip or fail. found is the result of looking
+// the docker program up, probeErr the result of asking the daemon for its version (nil when it answered). The checker's
+// integration level runs shut off (no network, no /run), so on a build node the program is found and the daemon is not
+// reachable; that is a skip, not a failure, except where docker is required (MESH_REQUIRE_DOCKER=1).
+func dockerVerdict(found bool, lookErr, probeErr error, required bool) (action, reason string) {
+	var why string
+	switch {
+	case !found:
+		why = fmt.Sprintf("there is no docker in this toolchain (%v)", lookErr)
+	case probeErr != nil:
+		why = fmt.Sprintf("no docker daemon is reachable from here (%v)", probeErr)
+	default:
+		return "run", ""
+	}
+	if required {
+		return "fail", "MESH_REQUIRE_DOCKER=1 and " + why
+	}
+	return "skip", why
+}
+
+func TestDockerVerdict(t *testing.T) {
+	boom := fmt.Errorf("cannot connect")
+	cases := []struct {
+		name        string
+		found       bool
+		look, probe error
+		required    bool
+		want        string
+	}{
+		{"answers", true, nil, nil, false, "run"},
+		{"answers, required", true, nil, nil, true, "run"},
+		{"no program", false, boom, nil, false, "skip"},
+		{"no program, required", false, boom, nil, true, "fail"},
+		{"program but no daemon", true, nil, boom, false, "skip"},
+		{"program but no daemon, required", true, nil, boom, true, "fail"},
+	}
+	for _, c := range cases {
+		got, why := dockerVerdict(c.found, c.look, c.probe, c.required)
+		if got != c.want {
+			t.Errorf("%s: got %s, want %s", c.name, got, c.want)
+		}
+		if got != "run" && why == "" {
+			t.Errorf("%s: no reason given", c.name)
+		}
+	}
+}

modules/nats/cmd/mesh-nats-snapshot/live_test.go

@@ -60,14 +60,24 @@ var snapshotGrants = struct{ publish, subscribe []string }{
 
 func needDocker(t *testing.T) {
 	t.Helper()
-	if _, err := exec.LookPath("docker"); err != nil {
-		// The Go toolchain container of the merge check holds no docker. Where docker is expected
-		// (MESH_REQUIRE_DOCKER=1, which merge-check.sh sets for its docker step) its absence is a failure.
-		if os.Getenv("MESH_REQUIRE_DOCKER") == "1" {
-			t.Fatalf("MESH_REQUIRE_DOCKER=1 and there is no docker here: %v", err)
+	_, lookErr := exec.LookPath("docker")
+	var probeErr error
+	if lookErr == nil {
+		// The binary alone proves nothing: the checker's integration level runs shut off (no network, no /run), so on
+		// a build node docker is found and its daemon is not reachable (issue 1823).
+		ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+		defer cancel()
+		if out, err := exec.CommandContext(ctx, "docker", "version", "--format", "{{.Server.Version}}").CombinedOutput(); err != nil {
+			probeErr = fmt.Errorf("%v: %s", err, strings.TrimSpace(string(out)))
 		}
-		t.Skipf("NOT RUN: this test starts a throwaway nats container and there is no docker in this toolchain (%v); "+
-			"it is proof only where merge-check.sh's docker step runs it (MESH_CHECK_DOCKER=1)", err)
+	}
+	// Where docker is expected (MESH_REQUIRE_DOCKER=1, which merge-check.sh sets for its docker step) its absence is a failure.
+	switch action, why := dockerVerdict(lookErr == nil, lookErr, probeErr, os.Getenv("MESH_REQUIRE_DOCKER") == "1"); action {
+	case "fail":
+		t.Fatal(why)
+	case "skip":
+		t.Skipf("NOT RUN: this test starts a throwaway nats container and %s; "+
+			"it is proof only where merge-check.sh's docker step runs it (MESH_CHECK_DOCKER=1)", why)
 	}
 }