Report a provider that keeps failing a consumer in status (hq ADR 0224)

The identity provider failed every consumer for a day and status called the
mesh well (hq issue 179). The controller now follows every provider's
provisioner.failing/recovered, keeps the newest failing word per provider,
machine and consumer (migration 0065), and status, its JSON and node show
name it until it recovers. Every module that receives contributions is
granted the two events, so no manifest can forget them.

cmd/mesh-controller/build.go

@@ -676,6 +676,9 @@ type answers struct {
 	// refused, until the switch — and while there is any, the mesh is not all well: the order the
 	// machines' modules are built in is the mesh's to keep, and this is where it says it is not kept.
 	unheld []catalogue.Unheld
+	// failing is every consumer a provider says it keeps failing (novox/hq ADR 0224): a provider's
+	// journal was the only place that said so for a day (04-ISSUES/179).
+	failing []inventory.ProviderStanding
 }
 
 // heldBy is every artifact this mesh has built, for a build that may need one as its base.

cmd/mesh-controller/modules.go

@@ -661,7 +661,7 @@ func issueWith(ctx context.Context, inv *inventory.Inventory, m catalogue.Manife
 	// durable subscription nobody reads.
 	if consumer, needed := broker.ConsumerFor(broker.Principal{
 		Kind: broker.KindModule, Node: node, Module: m.Module,
-		Emits: m.Emits, Consumes: m.Consumes, Serves: m.Tools,
+		Emits: m.EmitsAll(), Consumes: m.Consumes, Serves: m.Tools,
 	}); needed {
 		if busAddress == "" {
 			fmt.Printf("  %s consumes; its consumer is created when the bus is reachable (`push`, then "+

cmd/mesh-controller/nodes.go

@@ -505,6 +505,19 @@ func showNode(ctx context.Context, inv *inventory.Inventory, name string) error
 		fmt.Printf("  public domain     %s\n", domain)
 	}
 
+	// A provider here failing a consumer, or a consumer here failed (novox/hq ADR 0224). Before the
+	// capabilities, because it is something not working now and they are a description.
+	failing, err := failingProviders(ctx, inv)
+	if err != nil {
+		return err
+	}
+	if here := failingOn(failing, name); len(here) > 0 {
+		fmt.Printf("\n  %d consumer(s) a provider keeps failing, here or for a module here:\n", len(here))
+		for _, line := range failingLines(here, time.Now()) {
+			fmt.Printf("  %s\n", line)
+		}
+	}
+
 	held, err := inv.Profile(ctx, name)
 	if err != nil {
 		return err

cmd/mesh-controller/push.go

@@ -132,6 +132,11 @@ func serve(ctx context.Context) error {
 	if err := server.Answers(following{open}); err != nil {
 		return err
 	}
+	// And what providers say about consumers they keep failing, kept for `status` (novox/hq ADR
+	// 0224): a provider's journal must not be the only place that says so.
+	if err := server.Watches(standings{inv}); err != nil {
+		return err
+	}
 
 	// And the mesh's own verbs, as the seat this control plane holds (novox/hq ADR 0154). Served
 	// from the store's row, so what the seat declares is what is answered.

cmd/mesh-controller/readable.go

@@ -78,6 +78,11 @@ type meshStatus struct {
 	// that machine holds, with the modules that could hold it (novox/hq ADR 0207). Absent when every
 	// dependency is met. Reported, not refused, until the switch.
 	Unheld []catalogue.Unheld `json:"unheld,omitempty"`
+	// Failing is every consumer a provider says it keeps failing, with the class of error, since
+	// when, and when it was last said (novox/hq ADR 0224). Absent when no provider says so. A
+	// document without this called the mesh well while the identity provider refused every consumer
+	// for a day (04-ISSUES/179).
+	Failing []inventory.ProviderStanding `json:"failing,omitempty"`
 }
 
 // machineFiltered is one rule set on a converged machine that the mesh did not write and that
@@ -210,6 +215,7 @@ func statusAsJSON(asked answers) ([]byte, error) {
 		}
 	}
 	out.Unheld = asked.unheld
+	out.Failing = asked.failing
 	for name := range asked.refused {
 		out.Unresolved = append(out.Unresolved, machineUnresolved{
 			Node: name, Problem: asked.refused[name]})

cmd/mesh-controller/rollout.go

@@ -189,7 +189,7 @@ func readinessOf(ctx context.Context, inv *inventory.Inventory) (broker.Readines
 // A third of the catalogue never does (novox/hq ADR 0120), and counting those as missing a credential
 // would bury the ones that matter under a list nobody can act on.
 func speaksOnTheBus(m catalogue.Manifest) bool {
-	return len(m.Emits) > 0 || len(m.Consumes) > 0 || len(m.Tools) > 0 ||
+	return len(m.EmitsAll()) > 0 || len(m.Consumes) > 0 || len(m.Tools) > 0 ||
 		len(m.DefinesSeats) > 0 || len(m.Uses) > 0 || len(m.Claims) > 0
 }
 

cmd/mesh-controller/standing.go

@@ -0,0 +1,120 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"strings"
+	"time"
+
+	"github.com/novox/mesh-controller/internal/inventory"
+	"github.com/novox/mesh-controller/internal/link"
+)
+
+// A provider that keeps failing a consumer is a problem the controller reports (novox/hq ADR 0224).
+//
+// On 2026-10-05 the identity provider's provisioner failed every consumer from shortly after midnight
+// until it was fixed by hand that night — 31,000 refused logins after its database was moved and its
+// admin kept an older password — and `status` called the mesh well all day (novox/hq issue 179). A
+// provider now announces a consumer it has failed for minutes; the controller keeps it until the
+// provider says it recovered; and `status`, its JSON and `node show` name it, breaking "all well".
+
+// standings keeps what providers say, in the inventory.
+type standings struct{ inv *inventory.Inventory }
+
+func (s standings) Stood(ctx context.Context, st link.Standing) (bool, error) {
+	return s.inv.KeepStanding(ctx, st.Failing, inventory.ProviderStanding{
+		Module: st.Module, ProviderNode: st.ProviderNode, Provision: st.Provider,
+		Consumer: st.Consumer, ConsumerNode: st.Node,
+		Class: st.Class, Error: st.Error, Since: st.Since, Attempts: st.Attempts,
+	})
+}
+
+// failingProviders is every consumer a provider still assigned where it ran says it keeps failing.
+//
+// **A provider no longer assigned is not asked about.** Its last word stays in the store, and is
+// not a problem: nothing runs there to fail anybody. Assigned again, its first success for each
+// consumer clears it.
+func failingProviders(ctx context.Context, inv *inventory.Inventory) ([]inventory.ProviderStanding, error) {
+	all, err := inv.FailingProviders(ctx)
+	if err != nil {
+		return nil, fmt.Errorf("what providers say they keep failing cannot be read: %w", err)
+	}
+	assigned := map[string]map[string]bool{}
+	var out []inventory.ProviderStanding
+	for _, s := range all {
+		on, asked := assigned[s.ProviderNode]
+		if !asked {
+			modules, err := inv.Assigned(ctx, s.ProviderNode)
+			if err != nil {
+				// A provider on a machine the mesh no longer knows has nothing running to fail anybody.
+				modules = nil
+			}
+			on = map[string]bool{}
+			for _, m := range modules {
+				on[m] = true
+			}
+			assigned[s.ProviderNode] = on
+		}
+		if on[s.Module] {
+			out = append(out, s)
+		}
+	}
+	return out, nil
+}
+
+// failingLines is how status says them: one consumer per entry, the error under it, and a provider
+// that stopped repeating itself said so.
+func failingLines(list []inventory.ProviderStanding, now time.Time) []string {
+	var out []string
+	for _, s := range list {
+		where := s.Module
+		if s.ProviderNode != "" {
+			where += " on " + s.ProviderNode
+		}
+		whom := s.Consumer
+		if s.ConsumerNode != "" {
+			whom += " (" + s.ConsumerNode + ")"
+		}
+		out = append(out, fmt.Sprintf("  %-24s fails %s: %s, for %s (%d attempts since %s)",
+			where, whom, orUnclassed(s.Class), roughly(now.Sub(s.Since)), s.Attempts,
+			s.Since.Local().Format("2006-01-02 15:04")))
+		if e := strings.TrimSpace(s.Error); e != "" {
+			out = append(out, fmt.Sprintf("  %-24s %s", "", firstLine(e)))
+		}
+		if s.Quiet(now) {
+			out = append(out, fmt.Sprintf("  %-24s not said again for %s — the provider has stopped "+
+				"saying anything, so this is its last word", "", roughly(now.Sub(s.SaidAt))))
+		}
+	}
+	return out
+}
+
+func orUnclassed(class string) string {
+	if class == "" {
+		return "failing"
+	}
+	return class
+}
+
+// printFailing is the status section, said when there is anything to say.
+func printFailing(list []inventory.ProviderStanding, now time.Time) {
+	if len(list) == 0 {
+		return
+	}
+	fmt.Printf("%d consumer(s) a provider keeps failing (ADR 0224):\n\n", len(list))
+	for _, line := range failingLines(list, now) {
+		fmt.Println(line)
+	}
+	fmt.Printf("\n  the provider's journal has every attempt; it says recovered on its next success\n\n")
+}
+
+// failingOn is the standings that concern one machine: a provider running there, or a consumer.
+func failingOn(list []inventory.ProviderStanding, node string) []inventory.ProviderStanding {
+	var out []inventory.ProviderStanding
+	for _, s := range list {
+		if s.ProviderNode == node || s.ConsumerNode == node {
+			out = append(out, s)
+		}
+	}
+	return out
+}

cmd/mesh-controller/standing_test.go

@@ -0,0 +1,115 @@
+package main
+
+import (
+	"encoding/json"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/novox/mesh-controller/internal/catalogue"
+	"github.com/novox/mesh-controller/internal/inventory"
+	"github.com/novox/mesh-controller/internal/link"
+)
+
+// A provider that keeps failing a consumer is a problem `status` names (novox/hq ADR 0224). On
+// 2026-10-05 the identity provider refused every consumer for a day and status called the mesh well
+// (04-ISSUES/179): this is that day, told to the controller the way the provider now tells it.
+func TestAProviderFailingAConsumerBreaksAllWellUntilItRecovers(t *testing.T) {
+	open := aMesh(t)
+	ctx := t.Context()
+	register(t, open, catalogue.Manifest{Module: "idp", Version: "1",
+		Receives: map[string]string{"oidc-client": "/var/lib/mesh/idp/mesh.json"}})
+	if _, err := assign(ctx, open, "anchor", "idp"); err != nil {
+		t.Fatal(err)
+	}
+	kept := standings{open.inventory}
+	since := time.Now().Add(-23 * time.Hour)
+	failing := link.Standing{Module: "idp", Failing: true, Provider: "oidc-client", ProviderNode: "anchor",
+		Consumer: "mesh_laptop_dashboard", Node: "laptop", Class: "credentials-rejected",
+		Error: `Keycloak token request failed: 401 {"error":"invalid_grant"}`, Since: since, Attempts: 31000}
+	if _, err := kept.Stood(ctx, failing); err != nil {
+		t.Fatal(err)
+	}
+
+	asked, err := theThreeQuestions(ctx, open)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if asked.well() {
+		t.Fatal("a mesh whose identity provider fails a consumer reads as well")
+	}
+	said := printed(t, func() error { return printStatus(asked) })
+	for _, want := range []string{"1 consumer(s) a provider keeps failing", "idp on anchor",
+		"mesh_laptop_dashboard (laptop)", "credentials-rejected", "31000 attempts", "invalid_grant"} {
+		if !strings.Contains(said, want) {
+			t.Fatalf("status does not say %q:\n%s", want, said)
+		}
+	}
+	if strings.Contains(said, "all doing what they were told") {
+		t.Fatalf("status said all well beside a failing provider:\n%s", said)
+	}
+	body, err := statusAsJSON(asked)
+	if err != nil {
+		t.Fatal(err)
+	}
+	var doc struct {
+		Failing []inventory.ProviderStanding `json:"failing"`
+	}
+	if err := json.Unmarshal(body, &doc); err != nil || len(doc.Failing) != 1 || doc.Failing[0].Consumer != "mesh_laptop_dashboard" {
+		t.Fatalf("the document does not carry it: %v\n%s", err, body)
+	}
+	// Both machines' `node show` name it: where the provider runs, and where the consumer is.
+	for _, node := range []string{"anchor", "laptop"} {
+		shown := printed(t, func() error { return showNode(ctx, open.inventory, node) })
+		if !strings.Contains(shown, "a provider keeps failing") || !strings.Contains(shown, "mesh_laptop_dashboard") {
+			t.Fatalf("node show %s does not name it:\n%s", node, shown)
+		}
+	}
+
+	// Recovered: gone, and the mesh may be well again as far as this is concerned.
+	failing.Failing = false
+	if cleared, err := kept.Stood(ctx, failing); err != nil || !cleared {
+		t.Fatalf("%v %v", cleared, err)
+	}
+	asked, err = theThreeQuestions(ctx, open)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(asked.failing) != 0 {
+		t.Fatalf("a recovered consumer is still named: %+v", asked.failing)
+	}
+}
+
+// A provider no longer assigned where it ran has nothing running to fail anybody: its last word is
+// not a problem.
+func TestAnUnassignedProvidersLastWordIsNotAProblem(t *testing.T) {
+	open := aMesh(t)
+	ctx := t.Context()
+	if _, err := (standings{open.inventory}).Stood(ctx, link.Standing{Module: "gone", Failing: true,
+		ProviderNode: "anchor", Consumer: "x", Since: time.Now()}); err != nil {
+		t.Fatal(err)
+	}
+	asked, err := theThreeQuestions(ctx, open)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(asked.failing) != 0 {
+		t.Fatalf("%+v", asked.failing)
+	}
+}
+
+func TestAProviderThatStoppedRepeatingItselfIsSaidToHaveGoneQuiet(t *testing.T) {
+	now := time.Now()
+	lines := strings.Join(failingLines([]inventory.ProviderStanding{{
+		Module: "idp", ProviderNode: "anchor", Consumer: "c", Class: "unreachable", Error: "connection refused\nmore",
+		Since: now.Add(-3 * time.Hour), SaidAt: now.Add(-2 * time.Hour), Attempts: 9,
+	}}, now), "\n")
+	for _, want := range []string{"unreachable, for 3h", "connection refused", "not said again for 2h"} {
+		if !strings.Contains(lines, want) {
+			t.Fatalf("%q not in:\n%s", want, lines)
+		}
+	}
+	if strings.Contains(lines, "more") {
+		t.Fatalf("more than the first line of an error:\n%s", lines)
+	}
+}

cmd/mesh-controller/status.go

@@ -129,6 +129,10 @@ func printStatus(asked answers) error {
 		fmt.Println()
 	}
 
+	// A provider failing a consumer, beside machines failing what they were told: both are something
+	// not working now (novox/hq ADR 0224).
+	printFailing(asked.failing, time.Now())
+
 	if len(quiet) > 0 {
 		var said []string
 		for _, n := range quiet {
@@ -416,6 +420,12 @@ func theThreeQuestions(ctx context.Context, open *stores) (answers, error) {
 		}
 		out.unheld = append(out.unheld, plan.Unheld...)
 	}
+	// And every consumer a provider says it keeps failing (novox/hq ADR 0224). Read from what the
+	// providers announced: nothing else in the mesh knows whether a provision is being made.
+	out.failing, err = failingProviders(ctx, inv)
+	if err != nil {
+		return answers{}, err
+	}
 	out.plans, err = inv.RecentPlans(ctx, 5)
 	if err != nil {
 		return answers{}, err
@@ -528,7 +538,7 @@ func untakenModules(ctx context.Context, inv *inventory.Inventory, nodes []inven
 func (a answers) well() bool {
 	return len(a.wrong) == 0 && len(a.quiet) == 0 && len(a.behind) == 0 &&
 		len(a.waiting) == 0 && len(a.refused) == 0 && a.network == "" && len(a.untaken) == 0 &&
-		len(a.filtered) == 0 && len(a.unheld) == 0
+		len(a.filtered) == 0 && len(a.unheld) == 0 && len(a.failing) == 0
 }
 
 // hostSplit is which machines report which host version, for every version more than one machine