Report a provider that keeps failing a consumer in status (hq ADR 0224)

The identity provider failed every consumer for a day and status called the
mesh well (hq issue 179). The controller now follows every provider's
provisioner.failing/recovered, keeps the newest failing word per provider,
machine and consumer (migration 0065), and status, its JSON and node show
name it until it recovers. Every module that receives contributions is
granted the two events, so no manifest can forget them.

cmd/mesh-controller/standing_test.go

@@ -0,0 +1,115 @@
+package main
+
+import (
+	"encoding/json"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/novox/mesh-controller/internal/catalogue"
+	"github.com/novox/mesh-controller/internal/inventory"
+	"github.com/novox/mesh-controller/internal/link"
+)
+
+// A provider that keeps failing a consumer is a problem `status` names (novox/hq ADR 0224). On
+// 2026-10-05 the identity provider refused every consumer for a day and status called the mesh well
+// (04-ISSUES/179): this is that day, told to the controller the way the provider now tells it.
+func TestAProviderFailingAConsumerBreaksAllWellUntilItRecovers(t *testing.T) {
+	open := aMesh(t)
+	ctx := t.Context()
+	register(t, open, catalogue.Manifest{Module: "idp", Version: "1",
+		Receives: map[string]string{"oidc-client": "/var/lib/mesh/idp/mesh.json"}})
+	if _, err := assign(ctx, open, "anchor", "idp"); err != nil {
+		t.Fatal(err)
+	}
+	kept := standings{open.inventory}
+	since := time.Now().Add(-23 * time.Hour)
+	failing := link.Standing{Module: "idp", Failing: true, Provider: "oidc-client", ProviderNode: "anchor",
+		Consumer: "mesh_laptop_dashboard", Node: "laptop", Class: "credentials-rejected",
+		Error: `Keycloak token request failed: 401 {"error":"invalid_grant"}`, Since: since, Attempts: 31000}
+	if _, err := kept.Stood(ctx, failing); err != nil {
+		t.Fatal(err)
+	}
+
+	asked, err := theThreeQuestions(ctx, open)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if asked.well() {
+		t.Fatal("a mesh whose identity provider fails a consumer reads as well")
+	}
+	said := printed(t, func() error { return printStatus(asked) })
+	for _, want := range []string{"1 consumer(s) a provider keeps failing", "idp on anchor",
+		"mesh_laptop_dashboard (laptop)", "credentials-rejected", "31000 attempts", "invalid_grant"} {
+		if !strings.Contains(said, want) {
+			t.Fatalf("status does not say %q:\n%s", want, said)
+		}
+	}
+	if strings.Contains(said, "all doing what they were told") {
+		t.Fatalf("status said all well beside a failing provider:\n%s", said)
+	}
+	body, err := statusAsJSON(asked)
+	if err != nil {
+		t.Fatal(err)
+	}
+	var doc struct {
+		Failing []inventory.ProviderStanding `json:"failing"`
+	}
+	if err := json.Unmarshal(body, &doc); err != nil || len(doc.Failing) != 1 || doc.Failing[0].Consumer != "mesh_laptop_dashboard" {
+		t.Fatalf("the document does not carry it: %v\n%s", err, body)
+	}
+	// Both machines' `node show` name it: where the provider runs, and where the consumer is.
+	for _, node := range []string{"anchor", "laptop"} {
+		shown := printed(t, func() error { return showNode(ctx, open.inventory, node) })
+		if !strings.Contains(shown, "a provider keeps failing") || !strings.Contains(shown, "mesh_laptop_dashboard") {
+			t.Fatalf("node show %s does not name it:\n%s", node, shown)
+		}
+	}
+
+	// Recovered: gone, and the mesh may be well again as far as this is concerned.
+	failing.Failing = false
+	if cleared, err := kept.Stood(ctx, failing); err != nil || !cleared {
+		t.Fatalf("%v %v", cleared, err)
+	}
+	asked, err = theThreeQuestions(ctx, open)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(asked.failing) != 0 {
+		t.Fatalf("a recovered consumer is still named: %+v", asked.failing)
+	}
+}
+
+// A provider no longer assigned where it ran has nothing running to fail anybody: its last word is
+// not a problem.
+func TestAnUnassignedProvidersLastWordIsNotAProblem(t *testing.T) {
+	open := aMesh(t)
+	ctx := t.Context()
+	if _, err := (standings{open.inventory}).Stood(ctx, link.Standing{Module: "gone", Failing: true,
+		ProviderNode: "anchor", Consumer: "x", Since: time.Now()}); err != nil {
+		t.Fatal(err)
+	}
+	asked, err := theThreeQuestions(ctx, open)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(asked.failing) != 0 {
+		t.Fatalf("%+v", asked.failing)
+	}
+}
+
+func TestAProviderThatStoppedRepeatingItselfIsSaidToHaveGoneQuiet(t *testing.T) {
+	now := time.Now()
+	lines := strings.Join(failingLines([]inventory.ProviderStanding{{
+		Module: "idp", ProviderNode: "anchor", Consumer: "c", Class: "unreachable", Error: "connection refused\nmore",
+		Since: now.Add(-3 * time.Hour), SaidAt: now.Add(-2 * time.Hour), Attempts: 9,
+	}}, now), "\n")
+	for _, want := range []string{"unreachable, for 3h", "connection refused", "not said again for 2h"} {
+		if !strings.Contains(lines, want) {
+			t.Fatalf("%q not in:\n%s", want, lines)
+		}
+	}
+	if strings.Contains(lines, "more") {
+		t.Fatalf("more than the first line of an error:\n%s", lines)
+	}
+}