Report a provider that keeps failing a consumer in status (hq ADR 0224)

The identity provider failed every consumer for a day and status called the
mesh well (hq issue 179). The controller now follows every provider's
provisioner.failing/recovered, keeps the newest failing word per provider,
machine and consumer (migration 0065), and status, its JSON and node show
name it until it recovers. Every module that receives contributions is
granted the two events, so no manifest can forget them.

internal/inventory/busrecords.go

@@ -124,7 +124,7 @@ func declaredFor(m catalogue.Manifest, seats map[string]catalogue.SeatDeclaratio
 
 	d := broker.Declared{
 		Module:   m.Module,
-		Emits:    m.Emits,
+		Emits:    m.EmitsAll(),
 		Consumes: fromModules,
 		Watches:  watches,
 		// The tools it answers, which is `tools` and not `serves`: the manifest's `serves` is the

internal/inventory/migrations/0065-a-provider-says-which-consumer-it-keeps-failing.sql

@@ -0,0 +1,20 @@
+-- A provider says which consumer it keeps failing (novox/hq ADR 0224).
+--
+-- On 2026-10-05 the identity provider's provisioner failed every consumer 31,000 times in a day and
+-- only its journal said so (issue 179). A provider now announces a consumer it has failed for minutes
+-- without one success, and the consumer recovering; the controller keeps the newest failing word per
+-- provider module, the machine it runs on and the consumer, and removes it on recovery. `status` and
+-- `node show` read this table: a row is a problem until it is gone.
+create table provider_standing (
+    module        text        not null,
+    provider_node text        not null,
+    consumer      text        not null,
+    consumer_node text        not null default '',
+    provision     text        not null default '',
+    class         text        not null default '',
+    error         text        not null default '',
+    since         timestamptz not null,
+    attempts      integer     not null default 0,
+    said_at       timestamptz not null default now(),
+    primary key (module, provider_node, consumer)
+);

internal/inventory/standing.go

@@ -0,0 +1,76 @@
+package inventory
+
+import (
+	"context"
+	"time"
+)
+
+// ProviderStanding is a consumer a provider says it keeps failing (novox/hq ADR 0224).
+type ProviderStanding struct {
+	// Module is the provider's module, and ProviderNode the machine it runs on.
+	Module       string `json:"module"`
+	ProviderNode string `json:"provider-node"`
+	// Provision is the interface it provides, e.g. `oidc-client`.
+	Provision string `json:"provision"`
+	// Consumer is the identity the mesh derived for the consumer, ConsumerNode its machine.
+	Consumer     string `json:"consumer"`
+	ConsumerNode string `json:"consumer-node"`
+	// Class is what kind of failure: credentials-rejected, unreachable, secret-unreadable, refused.
+	Class string `json:"class"`
+	Error string `json:"error"`
+	// Since is when the unbroken run of failures began; Attempts how many it has been.
+	Since    time.Time `json:"since"`
+	Attempts int       `json:"attempts"`
+	// SaidAt is when the controller last heard it. A provider says it again every quarter of an hour
+	// while it lasts, so an old one is a provider that stopped saying anything.
+	SaidAt time.Time `json:"said-at"`
+}
+
+// SayAgainWithin is how long a failing standing stays current without being said again: twice the
+// quarter of an hour a provider repeats it at. Older, and status says the provider has gone quiet.
+const SayAgainWithin = 30 * time.Minute
+
+// Quiet says the provider has not repeated this standing for longer than it would while it lasts.
+func (s ProviderStanding) Quiet(now time.Time) bool { return now.Sub(s.SaidAt) > SayAgainWithin }
+
+// KeepStanding records a provider's newest word: failing keeps it, recovered removes it, and says
+// whether a recovery removed anything.
+func (i *Inventory) KeepStanding(ctx context.Context, failing bool, s ProviderStanding) (bool, error) {
+	if !failing {
+		tag, err := i.store.Pool().Exec(ctx,
+			`delete from provider_standing where module = $1 and provider_node = $2 and consumer = $3`,
+			s.Module, s.ProviderNode, s.Consumer)
+		return err == nil && tag.RowsAffected() > 0, err
+	}
+	_, err := i.store.Pool().Exec(ctx, `
+		insert into provider_standing
+			(module, provider_node, consumer, consumer_node, provision, class, error, since, attempts, said_at)
+		values ($1, $2, $3, $4, $5, $6, $7, $8, $9, now())
+		on conflict (module, provider_node, consumer) do update set
+			consumer_node = excluded.consumer_node, provision = excluded.provision,
+			class = excluded.class, error = excluded.error, since = excluded.since,
+			attempts = excluded.attempts, said_at = excluded.said_at`,
+		s.Module, s.ProviderNode, s.Consumer, s.ConsumerNode, s.Provision, s.Class, s.Error, s.Since, s.Attempts)
+	return false, err
+}
+
+// FailingProviders is every consumer a provider last said it keeps failing, oldest run first.
+func (i *Inventory) FailingProviders(ctx context.Context) ([]ProviderStanding, error) {
+	rows, err := i.store.Pool().Query(ctx, `
+		select module, provider_node, provision, consumer, consumer_node, class, error, since, attempts, said_at
+		from provider_standing order by since, module, consumer`)
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+	var out []ProviderStanding
+	for rows.Next() {
+		var s ProviderStanding
+		if err := rows.Scan(&s.Module, &s.ProviderNode, &s.Provision, &s.Consumer, &s.ConsumerNode,
+			&s.Class, &s.Error, &s.Since, &s.Attempts, &s.SaidAt); err != nil {
+			return nil, err
+		}
+		out = append(out, s)
+	}
+	return out, rows.Err()
+}

internal/inventory/standing_test.go

@@ -0,0 +1,130 @@
+package inventory
+
+import (
+	"slices"
+	"testing"
+	"time"
+
+	"github.com/novox/mesh-controller/internal/broker"
+	"github.com/novox/mesh-controller/internal/catalogue"
+)
+
+// A provider's standing (novox/hq ADR 0224), from the grant that lets it say so to the row status
+// reads.
+
+// The broker spells the events itself because it cannot import the catalogue; the two agree.
+func TestTheBrokerAndTheCatalogueNameTheSameStandingEvents(t *testing.T) {
+	if broker.ProvisionerFailing != catalogue.ProvisionerFailing ||
+		broker.ProvisionerRecovered != catalogue.ProvisionerRecovered {
+		t.Fatal("the broker and the catalogue disagree about what a provider's standing is called")
+	}
+}
+
+// **Every provider may say it, whatever its manifest lists**: a provider whose manifest forgot the
+// events would have its announcement refused by the bus, and fail its consumers as silently as on
+// 2026-10-05 (issue 179). A module that receives no contributions provides nothing and is given
+// nothing.
+func TestEveryProviderIsGrantedItsStandingAndNothingElseIs(t *testing.T) {
+	provider := catalogue.Manifest{Module: "keycloak", Version: "1",
+		Emits: []string{"client.created"}, Receives: map[string]string{"oidc-client": "/x/mesh.json"}}
+	consumer := catalogue.Manifest{Module: "grafana", Version: "1", Emits: []string{"dashboard.saved"}}
+
+	d := declaredFor(provider, nil)
+	for _, e := range []string{"client.created", catalogue.ProvisionerFailing, catalogue.ProvisionerRecovered} {
+		if !slices.Contains(d.Emits, e) {
+			t.Fatalf("a provider is not granted %s: %v", e, d.Emits)
+		}
+	}
+	perms, err := broker.PermissionsFor(broker.Principal{Kind: broker.KindModule, Node: "anchor",
+		Module: "keycloak", Emits: d.Emits})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !slices.Contains(perms.Publish, "mesh.mod.keycloak.event.provisioner.failing") {
+		t.Fatalf("the bus would refuse a provider's standing: %v", perms.Publish)
+	}
+
+	if got := declaredFor(consumer, nil).Emits; slices.Contains(got, catalogue.ProvisionerFailing) {
+		t.Fatalf("a module that provides nothing was granted a provider's standing: %v", got)
+	}
+	// Declared by hand as well: said once.
+	provider.Emits = append(provider.Emits, catalogue.ProvisionerFailing)
+	n := 0
+	for _, e := range provider.EmitsAll() {
+		if e == catalogue.ProvisionerFailing {
+			n++
+		}
+	}
+	if n != 1 {
+		t.Fatalf("%v", provider.EmitsAll())
+	}
+}
+
+// And the controller may hear it from every provider, and only those two events.
+func TestTheControllerHearsEveryProvidersStanding(t *testing.T) {
+	perms, err := broker.PermissionsFor(broker.Principal{Kind: broker.KindController})
+	if err != nil {
+		t.Fatal(err)
+	}
+	for _, want := range []string{"mesh.mod.*.event.provisioner.failing", "mesh.mod.*.event.provisioner.recovered"} {
+		if !slices.Contains(perms.Subscribe, want) {
+			t.Fatalf("the controller may not hear %s: %v", want, perms.Subscribe)
+		}
+	}
+	if slices.Contains(perms.Subscribe, "mesh.mod.*.event.>") {
+		t.Fatal("the controller hears every event in the mesh")
+	}
+}
+
+func TestAFailingStandingIsKeptUntilItRecovers(t *testing.T) {
+	inv := ForTest(t)
+	ctx := t.Context()
+	since := time.Date(2026, 10, 5, 0, 49, 0, 0, time.UTC)
+	s := ProviderStanding{Module: "keycloak", ProviderNode: "anchor", Provision: "oidc-client",
+		Consumer: "mesh_home_grafana", ConsumerNode: "home-server", Class: "credentials-rejected",
+		Error: "401 invalid_grant", Since: since, Attempts: 60}
+	if _, err := inv.KeepStanding(ctx, true, s); err != nil {
+		t.Fatal(err)
+	}
+	// Said again: one row, the newest word.
+	s.Attempts = 31000
+	if _, err := inv.KeepStanding(ctx, true, s); err != nil {
+		t.Fatal(err)
+	}
+	got, err := inv.FailingProviders(ctx)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(got) != 1 || got[0].Attempts != 31000 || !got[0].Since.Equal(since) || got[0].ConsumerNode != "home-server" ||
+		got[0].Class != "credentials-rejected" || got[0].SaidAt.IsZero() {
+		t.Fatalf("%+v", got)
+	}
+	if got[0].Quiet(time.Now()) {
+		t.Fatal("a standing just said reads as quiet")
+	}
+	if !got[0].Quiet(time.Now().Add(SayAgainWithin + time.Minute)) {
+		t.Fatal("a standing not said again for longer than a provider repeats it does not read as quiet")
+	}
+
+	// The same consumer from another machine's provider is its own row.
+	other := s
+	other.ProviderNode = "laptop"
+	if _, err := inv.KeepStanding(ctx, true, other); err != nil {
+		t.Fatal(err)
+	}
+	cleared, err := inv.KeepStanding(ctx, false, s)
+	if err != nil || !cleared {
+		t.Fatalf("recovered cleared nothing: %v %v", cleared, err)
+	}
+	cleared, err = inv.KeepStanding(ctx, false, s)
+	if err != nil || cleared {
+		t.Fatalf("a recovery for nothing kept said it cleared something: %v %v", cleared, err)
+	}
+	got, err = inv.FailingProviders(ctx)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(got) != 1 || got[0].ProviderNode != "laptop" {
+		t.Fatalf("%+v", got)
+	}
+}