package main // A provider that keeps failing a consumer says so on the bus (novox/hq ADR 0224): not on the first // failure, which may be a restart; after FailingAfter of failures with no success between; again // every SayAgainEvery while it lasts; and recovered on the first success, or when the consumer goes. import ( "errors" "os" "strings" "testing" "time" ) type announced struct { event string body map[string]any } func standingWorld(t *testing.T) (*world, *[]announced) { w := newWorld(t) var said []announced w.h.Announce = func(e string, b map[string]any) { // The first success since start is its own test's; every other test reads past it. if b["why"] != "first-success" { said = append(said, announced{e, b}) } } w.h.Node = "anchor" return w, &said } // passes reconciles every five seconds for d, as Run would. func (w *world) passes(d time.Duration) { for end := w.now.Add(d); w.now.Before(end); w.now = w.now.Add(5 * time.Second) { w.h.Reconcile(ctx) } } func events(said []announced) string { var out []string for _, a := range said { out = append(out, a.event) } return strings.Join(out, ",") } func TestAConsumerFailedForMinutesIsAnnouncedNamingItAndTheClass(t *testing.T) { w, said := standingWorld(t) w.a.failing = errors.New(`token request failed: 401 {"error":"invalid_grant","error_description":"Invalid user credentials"}`) w.give(map[string]any{"as": "mesh_home_grafana", "node": "home-server"}) w.passes(4 * time.Minute) if len(*said) != 0 { t.Fatalf("announced before FailingAfter: %v", events(*said)) } w.passes(2 * time.Minute) if events(*said) != EventFailing { t.Fatalf("want one %s, got %q", EventFailing, events(*said)) } b := (*said)[0].body if b["consumer"] != "mesh_home_grafana" || b["node"] != "home-server" || b["class"] != ClassCredentials || b["provider"] != "postgres-database" || b["provider-node"] != "anchor" || b["attempts"].(int) < 60 { t.Fatalf("%v", b) } // Said again while it lasts, not every pass. w.passes(14 * time.Minute) if events(*said) != EventFailing { t.Fatalf("repeated too soon: %q", events(*said)) } w.passes(2 * time.Minute) if events(*said) != EventFailing+","+EventFailing { t.Fatalf("not repeated: %q", events(*said)) } // The first success takes it back. w.a.failing = nil w.passes(5 * time.Second) if events(*said) != EventFailing+","+EventFailing+","+EventRecovered { t.Fatalf("no recovery: %q", events(*said)) } if (*said)[2].body["consumer"] != "mesh_home_grafana" { t.Fatal((*said)[2].body) } } func TestOneSuccessBetweenFailuresStartsTheRunAgain(t *testing.T) { w, said := standingWorld(t) w.a.failing = errors.New("connection refused") w.give(map[string]any{"as": "a"}) w.passes(4 * time.Minute) w.a.failing = nil w.passes(5 * time.Second) w.give(map[string]any{"as": "a", "values": map[string]any{"name": "changed"}}) w.a.failing = errors.New("connection refused") w.passes(4 * time.Minute) if len(*said) != 0 { t.Fatalf("two runs of four minutes are not one of eight: %q", events(*said)) } } // The check that failed for a day on 2026-10-05: clients already made, every minute's check refused // at the token. A check that cannot be asked is a failure too. func TestACheckThatKeepsFailingIsAFailureToo(t *testing.T) { w, said := standingWorld(t) w.give(map[string]any{"as": "a"}) w.h.Reconcile(ctx) w.a.holdsErr = errors.New("401 invalid_grant") w.passes(7 * time.Minute) if events(*said) != EventFailing || (*said)[0].body["class"] != ClassCredentials { t.Fatalf("%q %v", events(*said), *said) } w.a.holdsErr = nil w.passes(time.Minute + 5*time.Second) if events(*said) != EventFailing+","+EventRecovered { t.Fatalf("%q", events(*said)) } } func TestAnUnreadableSecretIsAnnouncedAsSuch(t *testing.T) { w, said := standingWorld(t) w.give(map[string]any{"as": "a"}) os.Remove(w.dir + "/a.secret") w.passes(6 * time.Minute) if events(*said) != EventFailing || (*said)[0].body["class"] != ClassSecret { t.Fatalf("%q %v", events(*said), *said) } } func TestAWithdrawnConsumerIsNoLongerFailing(t *testing.T) { w, said := standingWorld(t) w.a.failing = errors.New("boom") w.give(map[string]any{"as": "a"}, map[string]any{"as": "b"}) w.passes(6 * time.Minute) if events(*said) != EventFailing+","+EventFailing { t.Fatalf("%q", events(*said)) } w.give(map[string]any{"as": "a"}) w.passes(5 * time.Second) last := (*said)[len(*said)-1] if last.event != EventRecovered || last.body["consumer"] != "b" || last.body["why"] != "withdrawn" { t.Fatalf("%v", *said) } } func TestAnErrorIsClassedByItsWords(t *testing.T) { for text, want := range map[string]string{ `Keycloak token request failed: 401 {"error":"invalid_grant"}`: ClassCredentials, `FATAL: password authentication failed for user "postgres"`: ClassCredentials, `dial tcp 127.0.0.1:5432: connect: connection refused`: ClassUnreachable, `context deadline exceeded`: ClassUnreachable, `extension "nope" is not available`: ClassRefused, } { if got := ClassOf(text); got != want { t.Errorf("%s: %s, want %s", text, got, want) } } } func TestAnAdapterThatClassesItsOwnErrorsIsBelieved(t *testing.T) { w, said := standingWorld(t) w.h.Adapter = classing{w.a} w.a.failing = errors.New("anything") w.give(map[string]any{"as": "a"}) w.passes(6 * time.Minute) if (*said)[0].body["class"] != "its-own" { t.Fatal((*said)[0].body) } } type classing struct{ *recorder } func (classing) Class(error) string { return "its-own" } // A provider restarted after announcing a failure has forgotten it; its first success for each // consumer is announced, so the controller clears what it kept rather than naming it for ever. func TestTheFirstSuccessSinceStartIsAnnouncedOnce(t *testing.T) { w := newWorld(t) var said []announced w.h.Announce = func(e string, b map[string]any) { said = append(said, announced{e, b}) } w.give(map[string]any{"as": "a"}) w.passes(3 * time.Minute) if events(said) != EventRecovered || said[0].body["why"] != "first-success" || said[0].body["consumer"] != "a" { t.Fatalf("%v", said) } }