mesh-catalog/modules/postgres/cmd/postgres-provider/standing_test.go

package main

// A provider that keeps failing a consumer says so on the bus (novox/hq ADR 0224): not on the first
// failure, which may be a restart; after FailingAfter of failures with no success between; again
// every SayAgainEvery while it lasts; and recovered on the first success, or when the consumer goes.

import (
	"errors"
	"os"
	"strings"
	"testing"
	"time"
)

type announced struct {
	event string
	body  map[string]any
}

func standingWorld(t *testing.T) (*world, *[]announced) {
	w := newWorld(t)
	var said []announced
	w.h.Announce = func(e string, b map[string]any) {
		// The first success since start is its own test's; every other test reads past it.
		if b["why"] != "first-success" {
			said = append(said, announced{e, b})
		}
	}
	w.h.Node = "anchor"
	return w, &said
}

// passes reconciles every five seconds for d, as Run would.
func (w *world) passes(d time.Duration) {
	for end := w.now.Add(d); w.now.Before(end); w.now = w.now.Add(5 * time.Second) {
		w.h.Reconcile(ctx)
	}
}

func events(said []announced) string {
	var out []string
	for _, a := range said {
		out = append(out, a.event)
	}
	return strings.Join(out, ",")
}

func TestAConsumerFailedForMinutesIsAnnouncedNamingItAndTheClass(t *testing.T) {
	w, said := standingWorld(t)
	w.a.failing = errors.New(`token request failed: 401 {"error":"invalid_grant","error_description":"Invalid user credentials"}`)
	w.give(map[string]any{"as": "mesh_home_grafana", "node": "home-server"})

	w.passes(4 * time.Minute)
	if len(*said) != 0 {
		t.Fatalf("announced before FailingAfter: %v", events(*said))
	}
	w.passes(2 * time.Minute)
	if events(*said) != EventFailing {
		t.Fatalf("want one %s, got %q", EventFailing, events(*said))
	}
	b := (*said)[0].body
	if b["consumer"] != "mesh_home_grafana" || b["node"] != "home-server" || b["class"] != ClassCredentials ||
		b["provider"] != "postgres-database" || b["provider-node"] != "anchor" || b["attempts"].(int) < 60 {
		t.Fatalf("%v", b)
	}

	// Said again while it lasts, not every pass.
	w.passes(14 * time.Minute)
	if events(*said) != EventFailing {
		t.Fatalf("repeated too soon: %q", events(*said))
	}
	w.passes(2 * time.Minute)
	if events(*said) != EventFailing+","+EventFailing {
		t.Fatalf("not repeated: %q", events(*said))
	}

	// The first success takes it back.
	w.a.failing = nil
	w.passes(5 * time.Second)
	if events(*said) != EventFailing+","+EventFailing+","+EventRecovered {
		t.Fatalf("no recovery: %q", events(*said))
	}
	if (*said)[2].body["consumer"] != "mesh_home_grafana" {
		t.Fatal((*said)[2].body)
	}
}

func TestOneSuccessBetweenFailuresStartsTheRunAgain(t *testing.T) {
	w, said := standingWorld(t)
	w.a.failing = errors.New("connection refused")
	w.give(map[string]any{"as": "a"})
	w.passes(4 * time.Minute)
	w.a.failing = nil
	w.passes(5 * time.Second)
	w.give(map[string]any{"as": "a", "values": map[string]any{"name": "changed"}})
	w.a.failing = errors.New("connection refused")
	w.passes(4 * time.Minute)
	if len(*said) != 0 {
		t.Fatalf("two runs of four minutes are not one of eight: %q", events(*said))
	}
}

// The check that failed for a day on 2026-10-05: clients already made, every minute's check refused
// at the token. A check that cannot be asked is a failure too.
func TestACheckThatKeepsFailingIsAFailureToo(t *testing.T) {
	w, said := standingWorld(t)
	w.give(map[string]any{"as": "a"})
	w.h.Reconcile(ctx)
	w.a.holdsErr = errors.New("401 invalid_grant")
	w.passes(7 * time.Minute)
	if events(*said) != EventFailing || (*said)[0].body["class"] != ClassCredentials {
		t.Fatalf("%q %v", events(*said), *said)
	}
	w.a.holdsErr = nil
	w.passes(time.Minute + 5*time.Second)
	if events(*said) != EventFailing+","+EventRecovered {
		t.Fatalf("%q", events(*said))
	}
}

func TestAnUnreadableSecretIsAnnouncedAsSuch(t *testing.T) {
	w, said := standingWorld(t)
	w.give(map[string]any{"as": "a"})
	os.Remove(w.dir + "/a.secret")
	w.passes(6 * time.Minute)
	if events(*said) != EventFailing || (*said)[0].body["class"] != ClassSecret {
		t.Fatalf("%q %v", events(*said), *said)
	}
}

func TestAWithdrawnConsumerIsNoLongerFailing(t *testing.T) {
	w, said := standingWorld(t)
	w.a.failing = errors.New("boom")
	w.give(map[string]any{"as": "a"}, map[string]any{"as": "b"})
	w.passes(6 * time.Minute)
	if events(*said) != EventFailing+","+EventFailing {
		t.Fatalf("%q", events(*said))
	}
	w.give(map[string]any{"as": "a"})
	w.passes(5 * time.Second)
	last := (*said)[len(*said)-1]
	if last.event != EventRecovered || last.body["consumer"] != "b" || last.body["why"] != "withdrawn" {
		t.Fatalf("%v", *said)
	}
}

func TestAnErrorIsClassedByItsWords(t *testing.T) {
	for text, want := range map[string]string{
		`Keycloak token request failed: 401 {"error":"invalid_grant"}`: ClassCredentials,
		`FATAL: password authentication failed for user "postgres"`:    ClassCredentials,
		`dial tcp 127.0.0.1:5432: connect: connection refused`:         ClassUnreachable,
		`context deadline exceeded`:                                    ClassUnreachable,
		`extension "nope" is not available`:                            ClassRefused,
	} {
		if got := ClassOf(text); got != want {
			t.Errorf("%s: %s, want %s", text, got, want)
		}
	}
}

func TestAnAdapterThatClassesItsOwnErrorsIsBelieved(t *testing.T) {
	w, said := standingWorld(t)
	w.h.Adapter = classing{w.a}
	w.a.failing = errors.New("anything")
	w.give(map[string]any{"as": "a"})
	w.passes(6 * time.Minute)
	if (*said)[0].body["class"] != "its-own" {
		t.Fatal((*said)[0].body)
	}
}

type classing struct{ *recorder }

func (classing) Class(error) string { return "its-own" }

// A provider restarted after announcing a failure has forgotten it; its first success for each
// consumer is announced, so the controller clears what it kept rather than naming it for ever.
func TestTheFirstSuccessSinceStartIsAnnouncedOnce(t *testing.T) {
	w := newWorld(t)
	var said []announced
	w.h.Announce = func(e string, b map[string]any) { said = append(said, announced{e, b}) }
	w.give(map[string]any{"as": "a"})
	w.passes(3 * time.Minute)
	if events(said) != EventRecovered || said[0].body["why"] != "first-success" || said[0].body["consumer"] != "a" {
		t.Fatalf("%v", said)
	}
}