Report a provider that keeps failing a consumer in status (hq ADR 0224)

The identity provider failed every consumer for a day and status called the
mesh well (hq issue 179). The controller now follows every provider's
provisioner.failing/recovered, keeps the newest failing word per provider,
machine and consumer (migration 0065), and status, its JSON and node show
name it until it recovers. Every module that receives contributions is
granted the two events, so no manifest can forget them.

internal/link/receive.go

@@ -34,6 +34,9 @@ const (
 	// built without anybody telling the mesh (novox/hq 04-ISSUES/131).
 	KindSourceMoved = "source-moved"
 	KindCatchUp     = "catch-up"
+	// KindProvisioner is a provider saying a consumer has failed for minutes, or recovered
+	// (novox/hq ADR 0224).
+	KindProvisioner = "provisioner"
 )
 
 // Control is one thing a node or a module said, as the controller must act on it.
@@ -54,6 +57,11 @@ type Control interface {
 	// Body is the message itself — the payload alone, never the envelope.
 	Body() []byte
 
+	// Subject is where it was published. For a module's event it names the emitter, which the bus
+	// enforces (only a module may publish into its own namespace), so who said it is read from here
+	// and never from the body.
+	Subject() string
+
 	// Redelivered says the bus has handed this message over before. An enrolment cares and
 	// nothing else does: one already spent is not finished a second time.
 	Redelivered() bool

internal/link/receive_fake_test.go

@@ -61,6 +61,7 @@ func (c *fakeInbound) retries(ctx context.Context, s *Server) {
 
 type fakeControl struct {
 	kind        string
+	subject     string
 	body        []byte
 	tag         uint64
 	to          *settled
@@ -71,6 +72,7 @@ type fakeControl struct {
 
 func (m *fakeControl) Kind() string                         { return m.kind }
 func (m *fakeControl) Body() []byte                         { return m.body }
+func (m *fakeControl) Subject() string                      { return m.subject }
 func (m *fakeControl) Redelivered() bool                    { return m.redelivered }
 func (m *fakeControl) About(string)                         {}
 func (m *fakeControl) Answer(context.Context, []byte) error { return nil }

internal/link/receive_nats.go

@@ -53,7 +53,7 @@ func Nats(js *broker.JetStream) Inbound {
 // whatever was asked for — and not at all when nothing was.
 func (n *natsInbound) Also(kind string) error {
 	switch kind {
-	case KindModuleMoved, KindCatchUp, KindSourceMoved:
+	case KindModuleMoved, KindCatchUp, KindSourceMoved, KindProvisioner:
 		n.follows[kind] = true
 		return nil
 	default:
@@ -241,9 +241,30 @@ func kindOfSubject(subject string) (string, bool) {
 		// runs (ADR 0190): the old builder still answers on the retired seat until it is unassigned.
 		return KindBuilt, true
 	}
+	if _, ok := ProvisionerEmitter(subject); ok {
+		return KindProvisioner, true
+	}
 	return "", false
 }
 
+// ProvisionerEmitter is the module a provider's standing event came from, read from its subject
+// (`mesh.mod.<module>.event.provisioner.<failing|recovered>`); false for any other subject. The
+// controller's own follow pattern, with `*` for the module, decodes too.
+func ProvisionerEmitter(subject string) (string, bool) {
+	rest, ok := strings.CutPrefix(subject, "mesh.mod.")
+	if !ok {
+		return "", false
+	}
+	module, event, ok := strings.Cut(rest, ".event.")
+	if !ok || module == "" || strings.Contains(module, ".") {
+		return "", false
+	}
+	if event != broker.ProvisionerFailing && event != broker.ProvisionerRecovered {
+		return "", false
+	}
+	return module, true
+}
+
 // natsControl is one message from the bus being built, as the controller reads it.
 type natsControl struct {
 	kind string
@@ -256,8 +277,9 @@ type natsControl struct {
 	delivered uint64
 }
 
-func (m *natsControl) Kind() string { return m.kind }
-func (m *natsControl) Body() []byte { return m.msg.Data }
+func (m *natsControl) Kind() string    { return m.kind }
+func (m *natsControl) Body() []byte    { return m.msg.Data }
+func (m *natsControl) Subject() string { return m.msg.Subject }
 
 // Redelivered is what the server counted, not what the controller remembers. Which is the answer to
 // a question the AMQP side could only guess at across a restart: an enrolment redelivered because

internal/link/serve.go

@@ -79,6 +79,8 @@ type Server struct {
 	recorder Recorder
 	upgrader Upgrader
 	replayer Replayer
+	// standings keeps what providers say about their consumers (novox/hq ADR 0224).
+	standings Standings
 
 	log *log.Logger
 	// giveUp is how long one message is held for the store; zero means GiveUpAfter.
@@ -153,6 +155,9 @@ func (s *Server) Serve(ctx context.Context) error {
 	if s.replayer != nil {
 		s.log.Printf("answering %s", KindCatchUp)
 	}
+	if s.standings != nil {
+		s.log.Printf("keeping every provider's %s", KindProvisioner)
+	}
 	return s.inbound.Receive(ctx, s.act)
 }
 
@@ -173,6 +178,8 @@ func (s *Server) act(ctx context.Context, m Control) {
 		s.sourceMoved(ctx, m)
 	case KindCatchUp:
 		s.catchingUp(ctx, m)
+	case KindProvisioner:
+		s.provisioner(ctx, m)
 	default:
 		// Dropped: a message nothing understands will not be understood on the next attempt
 		// either, and asking for it again would spin.

internal/link/standing.go

@@ -0,0 +1,114 @@
+package link
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"strings"
+	"time"
+
+	"github.com/novox/mesh-controller/internal/broker"
+)
+
+// A provider's standing (novox/hq ADR 0224).
+//
+// **A provider that keeps failing a consumer is a problem the controller reports**, not a line in a
+// journal. On 2026-10-05 the identity provider's provisioner failed every consumer 31,000 times in a
+// day — its admin no longer took the mesh's secret once its database was moved — and every surface
+// the mesh has called the mesh well (novox/hq issue 179). A provider now says, as an event, a
+// consumer it has failed for minutes without one success, and the consumer recovering; the
+// controller keeps the newest word per provider, machine and consumer, and `status` names each one
+// still failing.
+
+// Standing is one provider's word about one consumer.
+type Standing struct {
+	// Module is the emitter, read from the subject the bus let it publish on — never from the body.
+	Module string `json:"-"`
+	// Failing is which of the two it said: failing, or recovered.
+	Failing bool `json:"-"`
+
+	Provider     string    `json:"provider"`
+	ProviderNode string    `json:"provider-node"`
+	Consumer     string    `json:"consumer"`
+	Node         string    `json:"node"`
+	Class        string    `json:"class,omitempty"`
+	Error        string    `json:"error,omitempty"`
+	Since        time.Time `json:"since"`
+	Attempts     int       `json:"attempts"`
+	// Why is said with a recovery that is not a success: `withdrawn`, a consumer no longer asked for.
+	Why string `json:"why,omitempty"`
+}
+
+// Standings keeps what providers say about their consumers.
+type Standings interface {
+	// Stood records a provider's newest word about a consumer: a failing one kept, a recovered one
+	// cleared — and says whether a recovery cleared anything, since a provider announces its first
+	// success for every consumer after it starts. An error the store is away for is held and asked
+	// again, like a report.
+	Stood(ctx context.Context, s Standing) (cleared bool, err error)
+}
+
+// Watches says where providers' standings are kept, and asks for them to be delivered.
+func (s *Server) Watches(st Standings) error {
+	if err := s.inbound.Also(KindProvisioner); err != nil {
+		return err
+	}
+	s.standings = st
+	return nil
+}
+
+// ReadStanding is one standing event as the controller understands it, from its subject and body.
+func ReadStanding(subject string, body []byte) (Standing, error) {
+	module, ok := ProvisionerEmitter(subject)
+	if !ok {
+		return Standing{}, fmt.Errorf("%s is not a provider's standing", subject)
+	}
+	var st Standing
+	if err := json.Unmarshal(body, &st); err != nil {
+		return Standing{}, fmt.Errorf("%s's standing could not be read: %w", module, err)
+	}
+	if st.Consumer == "" {
+		return Standing{}, fmt.Errorf("%s's standing named no consumer", module)
+	}
+	st.Module = module
+	st.Failing = strings.HasSuffix(subject, "."+broker.ProvisionerFailing)
+	return st, nil
+}
+
+// provisioner acts on one standing event.
+//
+// **A recovery must not be lost.** A failing standing is said again every quarter of an hour while
+// it lasts, so one dropped is replaced; a recovery is said once, and dropping it would leave status
+// naming a consumer that is fine. So a store that is away holds the message, as a report is held.
+func (s *Server) provisioner(ctx context.Context, m Control) {
+	if s.standings == nil {
+		// Delivered because the consumer's filter names it, with nothing here keeping it: taken,
+		// because handing it back would not give it anywhere to go.
+		_ = m.Took()
+		return
+	}
+	st, err := ReadStanding(m.Subject(), m.Body())
+	if err != nil {
+		s.log.Printf("%v; ignored", err)
+		_ = m.Took()
+		return
+	}
+	cleared, err := s.standings.Stood(ctx, st)
+	what := fmt.Sprintf("%s's standing for %s", st.Module, st.Consumer)
+	switch s.decide(ctx, m, what, "", "", err) {
+	case Hold:
+		return
+	case Stale, GiveUp:
+		_ = m.Took()
+		return
+	}
+	if err != nil {
+		s.log.Printf("%s could not be kept: %v", what, err)
+	} else if st.Failing {
+		s.log.Printf("%s on %s is FAILING %s on %s (%s, %d attempts since %s): %s", st.Module,
+			st.ProviderNode, st.Consumer, st.Node, st.Class, st.Attempts, st.Since.Format(time.RFC3339), st.Error)
+	} else if cleared {
+		s.log.Printf("%s on %s recovered %s", st.Module, st.ProviderNode, st.Consumer)
+	}
+	_ = m.Took()
+}

internal/link/standing_test.go

@@ -0,0 +1,203 @@
+package link
+
+import (
+	"context"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/novox/mesh-controller/internal/broker"
+)
+
+// A provider's standing (novox/hq ADR 0224): who said it is read from the subject the bus let it
+// publish on, a failing one is kept and a recovery cleared, and a recovery is never lost to a store
+// that is away — said once, it would leave status naming a consumer that is fine.
+
+type keptStandings struct {
+	kept []Standing
+	err  error
+}
+
+func (k *keptStandings) Stood(_ context.Context, s Standing) (bool, error) {
+	if k.err != nil {
+		return false, k.err
+	}
+	k.kept = append(k.kept, s)
+	return !s.Failing, nil
+}
+
+func standingSays(t *testing.T, in *fakeInbound, to *settled, subject string, body map[string]any) Control {
+	t.Helper()
+	m := in.sends(t, to, KindProvisioner, body).(*fakeControl)
+	m.subject = subject
+	return m
+}
+
+func TestTheControllerFollowsEveryProvidersStandingAndNothingElse(t *testing.T) {
+	for subject, want := range map[string]string{
+		"mesh.mod.keycloak.event.provisioner.failing":   "keycloak",
+		"mesh.mod.postgres.event.provisioner.recovered": "postgres",
+		"mesh.mod.*.event.provisioner.failing":          "*",
+	} {
+		got, ok := ProvisionerEmitter(subject)
+		if !ok || got != want {
+			t.Errorf("%s: %q %v", subject, got, ok)
+		}
+		if kind, _ := kindOfSubject(subject); kind != KindProvisioner {
+			t.Errorf("%s decodes to %q", subject, kind)
+		}
+	}
+	for _, subject := range []string{
+		"mesh.mod.keycloak.event.provisioner.other",
+		"mesh.mod.keycloak.event.client.created",
+		"mesh.mod.a.b.event.provisioner.failing",
+		"mesh.seat.keycloak.event.provisioner.failing",
+	} {
+		if _, ok := ProvisionerEmitter(subject); ok {
+			t.Errorf("%s read as a provider's standing", subject)
+		}
+	}
+	var follows int
+	for _, s := range broker.ControllerFollows {
+		if kind, _ := kindOfSubject(s); kind == KindProvisioner {
+			follows++
+		}
+	}
+	if follows != 2 {
+		t.Fatalf("the controller follows %d standing subjects, want failing and recovered", follows)
+	}
+}
+
+func TestAFailingStandingIsKeptNamingTheEmitterFromTheSubject(t *testing.T) {
+	s, in := serving()
+	kept := &keptStandings{}
+	if err := s.Watches(kept); err != nil {
+		t.Fatal(err)
+	}
+	to := &settled{}
+	since := time.Date(2026, 10, 5, 0, 49, 0, 0, time.UTC)
+	s.act(t.Context(), standingSays(t, in, to, "mesh.mod.keycloak.event.provisioner.failing", map[string]any{
+		"provider": "oidc-client", "provider-node": "anchor", "consumer": "mesh_home_grafana",
+		"node": "home-server", "class": "credentials-rejected", "error": "401 invalid_grant",
+		"since": since, "attempts": 31000,
+		// A body naming another module is not believed: the subject is the bus's word.
+		"module": "postgres",
+	}))
+	if !to.acked || len(kept.kept) != 1 {
+		t.Fatalf("settled %+v, kept %+v", to, kept.kept)
+	}
+	got := kept.kept[0]
+	if got.Module != "keycloak" || !got.Failing || got.Consumer != "mesh_home_grafana" || got.Node != "home-server" ||
+		got.ProviderNode != "anchor" || got.Class != "credentials-rejected" || got.Attempts != 31000 || !got.Since.Equal(since) {
+		t.Fatalf("%+v", got)
+	}
+
+	to = &settled{}
+	s.act(t.Context(), standingSays(t, in, to, "mesh.mod.keycloak.event.provisioner.recovered", map[string]any{
+		"provider": "oidc-client", "provider-node": "anchor", "consumer": "mesh_home_grafana",
+	}))
+	if !to.acked || len(kept.kept) != 2 || kept.kept[1].Failing {
+		t.Fatalf("settled %+v, kept %+v", to, kept.kept)
+	}
+}
+
+func TestARecoveryIsHeldWhileTheStoreIsAway(t *testing.T) {
+	s, in := serving()
+	kept := &keptStandings{err: restarting}
+	if err := s.Watches(kept); err != nil {
+		t.Fatal(err)
+	}
+	to := &settled{}
+	s.act(t.Context(), standingSays(t, in, to, "mesh.mod.keycloak.event.provisioner.recovered",
+		map[string]any{"consumer": "mesh_home_grafana"}))
+	if !to.unsettled() || len(in.held) != 1 {
+		t.Fatalf("a recovery was settled while the store was away: %+v", to)
+	}
+	kept.err = nil
+	in.retries(t.Context(), s)
+	if !to.acked || len(kept.kept) != 1 {
+		t.Fatalf("the held recovery was not kept when the store came back: %+v %+v", to, kept.kept)
+	}
+}
+
+func TestAStandingThatNamesNoConsumerIsTakenAndForgotten(t *testing.T) {
+	s, in := serving()
+	kept := &keptStandings{}
+	if err := s.Watches(kept); err != nil {
+		t.Fatal(err)
+	}
+	to := &settled{}
+	s.act(t.Context(), standingSays(t, in, to, "mesh.mod.keycloak.event.provisioner.failing", map[string]any{}))
+	if !to.acked || len(kept.kept) != 0 {
+		t.Fatalf("%+v %+v", to, kept.kept)
+	}
+}
+
+func TestAStandingWithNothingKeepingItIsTaken(t *testing.T) {
+	s, in := serving()
+	to := &settled{}
+	s.act(t.Context(), standingSays(t, in, to, "mesh.mod.keycloak.event.provisioner.failing",
+		map[string]any{"consumer": "x"}))
+	if !to.acked {
+		t.Fatal("a standing nothing keeps was left for the bus to hand over again")
+	}
+}
+
+// Over a real bus: a provider's standing published under its own module's namespace reaches the
+// controller through the events consumer's filter — the one wildcard filter on it — names the emitter
+// from the subject, and is acknowledged.
+func TestNatsAProvidersStandingReachesTheController(t *testing.T) {
+	js := aBus(t)
+	kept := &lockedStandings{}
+	s := &Server{inbound: Nats(js), bus: OverNATS{Conn: js.Conn(), JS: js.Context()}, log: quiet()}
+	if err := s.Follows(&toldAbout{}); err != nil {
+		t.Fatal(err)
+	}
+	if err := s.Watches(kept); err != nil {
+		t.Fatal(err)
+	}
+	ctx, stop := context.WithCancel(context.Background())
+	defer stop()
+	go func() { _ = s.Serve(ctx) }()
+	eventually(t, "the controller's event consumer being made", func() bool {
+		_, err := js.Context().ConsumerInfo("EVENTS", broker.ControllerName)
+		return err == nil
+	})
+
+	for _, event := range []string{broker.ProvisionerFailing, broker.ProvisionerRecovered} {
+		if _, err := js.Context().Publish("mesh.mod.keycloak.event."+event,
+			[]byte(`{"consumer":"mesh_home_grafana","provider-node":"anchor","class":"credentials-rejected"}`)); err != nil {
+			t.Fatal(err)
+		}
+	}
+	// Somebody else's event under the same prefix is not the controller's to hear.
+	if _, err := js.Context().Publish("mesh.mod.keycloak.event.client.created", []byte(`{}`)); err != nil {
+		t.Fatal(err)
+	}
+	eventually(t, "both standings being kept, in order, naming the emitter", func() bool {
+		got := kept.all()
+		return len(got) == 2 && got[0].Module == "keycloak" && got[0].Failing && !got[1].Failing
+	})
+	eventually(t, "both being acknowledged and nothing else delivered", func() bool {
+		info, err := js.Context().ConsumerInfo("EVENTS", broker.ControllerName)
+		return err == nil && info.NumAckPending == 0 && info.Delivered.Consumer == 2
+	})
+}
+
+type lockedStandings struct {
+	mu   sync.Mutex
+	kept []Standing
+}
+
+func (l *lockedStandings) Stood(_ context.Context, s Standing) (bool, error) {
+	l.mu.Lock()
+	defer l.mu.Unlock()
+	l.kept = append(l.kept, s)
+	return !s.Failing, nil
+}
+
+func (l *lockedStandings) all() []Standing {
+	l.mu.Lock()
+	defer l.mu.Unlock()
+	return append([]Standing(nil), l.kept...)
+}