Review of 082: a store killed mid-conversation is an outage and a wrong password is not; one report holds the queue at most two minutes, then is let go loudly; shutdown does not wait out the pause

internal/inventory/reachable_test.go

@@ -1,28 +1,76 @@
 package inventory
 
 import (
+	"context"
 	"errors"
 	"fmt"
+	"io"
+	"os"
+	"strings"
 	"testing"
+	"time"
 
 	"github.com/jackc/pgx/v5/pgconn"
 )
 
-// The store restarting is "not now"; a constraint the store enforced is an answer (issue 082).
-func TestAStoreThatIsStartingIsUnreachableAndARefusalIsNot(t *testing.T) {
+// The store restarting or dying is "not now"; an answer the store gave is not (issue 082).
+func TestAStoreThatIsGoneForNowIsUnreachableAndAnAnswerIsNot(t *testing.T) {
 	for _, c := range []struct {
+		what string
 		err  error
 		want bool
 	}{
-		{&pgconn.PgError{Code: "57P03", Message: "the database system is starting up"}, true},
-		{fmt.Errorf("recording: %w", &pgconn.PgError{Code: "57P01"}), true},
-		{&pgconn.PgError{Code: "08006"}, true},
-		{&pgconn.PgError{Code: "23503", Message: "violates foreign key constraint"}, false},
-		{errors.New("a report named no node"), false},
-		{nil, false},
+		{"starting up", &pgconn.PgError{Code: "57P03"}, true},
+		{"stopped by its administrator, wrapped", fmt.Errorf("recording: %w", &pgconn.PgError{Code: "57P01"}), true},
+		{"crashed", &pgconn.PgError{Code: "57P02"}, true},
+		{"a connection exception", &pgconn.PgError{Code: "08006"}, true},
+		{"killed under a live connection", io.ErrUnexpectedEOF, true},
+		{"killed, wrapped", fmt.Errorf("recording: %w", io.ErrUnexpectedEOF), true},
+		{"a statement cancelled", &pgconn.PgError{Code: "57014"}, false},
+		{"the database dropped", &pgconn.PgError{Code: "57P04"}, false},
+		{"a wrong password", &pgconn.PgError{Code: "28P01"}, false},
+		{"a constraint", &pgconn.PgError{Code: "23503"}, false},
+		{"a plain error", errors.New("a report named no node"), false},
+		{"nothing", nil, false},
 	} {
 		if got := Unreachable(c.err); got != c.want {
-			t.Errorf("Unreachable(%v) = %v, want %v", c.err, got, c.want)
+			t.Errorf("%s: Unreachable(%v) = %v, want %v", c.what, c.err, got, c.want)
 		}
 	}
 }
+
+// A connection refused is the shape a store that is down gives — a real one, to a port on
+// loopback nothing listens on.
+func TestAStoreThatRefusesTheConnectionIsUnreachable(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+	_, err := pgconn.Connect(ctx, "postgres://nobody@127.0.0.1:1/nothing?sslmode=disable&connect_timeout=2")
+	if err == nil {
+		t.Skip("something listens on port 1")
+	}
+	if !Unreachable(err) {
+		t.Fatalf("a refused connection is not unreachable: %T %v", err, err)
+	}
+}
+
+// A connection the store refused on a wrong password is an answer, even though it arrives as a
+// failed connection — asked again for ever, it would hold every message behind it.
+func TestAWrongPasswordIsAnAnswerNotAnOutage(t *testing.T) {
+	dsn := os.Getenv("MESH_TEST_POSTGRES")
+	if dsn == "" {
+		t.Skip("MESH_TEST_POSTGRES is not set")
+	}
+	wrong := strings.Replace(dsn, "postgres:check@", "postgres:not-the-password@", 1)
+	if wrong == dsn {
+		t.Skip("the test store's address does not carry the expected credential")
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+	_, err := pgconn.Connect(ctx, wrong)
+	if err == nil {
+		t.Fatal("a wrong password connected")
+	}
+	if Unreachable(err) {
+		t.Fatalf("a wrong password was taken for an outage and would be retried for ever: %v", err)
+	}
+}