Report a provider that keeps failing a consumer in status (hq ADR 0224)

The identity provider failed every consumer for a day and status called the mesh well (hq issue 179). The controller now follows every provider's provisioner.failing/recovered, keeps the newest failing word per provider, machine and consumer (migration 0065), and status, its JSON and node show name it until it recovers. Every module that receives contributions is granted the two events, so no manifest can forget them.
2026-10-06 00:13:23 +02:00
parent cc25baa563
commit 8d9d33ae85
23 changed files with 913 additions and 11 deletions
@@ -0,0 +1,120 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"strings"
+	"time"
+
+	"github.com/novox/mesh-controller/internal/inventory"
+	"github.com/novox/mesh-controller/internal/link"
+)
+
+// A provider that keeps failing a consumer is a problem the controller reports (novox/hq ADR 0224).
+//
+// On 2026-10-05 the identity provider's provisioner failed every consumer from shortly after midnight
+// until it was fixed by hand that night — 31,000 refused logins after its database was moved and its
+// admin kept an older password — and `status` called the mesh well all day (novox/hq issue 179). A
+// provider now announces a consumer it has failed for minutes; the controller keeps it until the
+// provider says it recovered; and `status`, its JSON and `node show` name it, breaking "all well".
+
+// standings keeps what providers say, in the inventory.
+type standings struct{ inv *inventory.Inventory }
+
+func (s standings) Stood(ctx context.Context, st link.Standing) (bool, error) {
+	return s.inv.KeepStanding(ctx, st.Failing, inventory.ProviderStanding{
+		Module: st.Module, ProviderNode: st.ProviderNode, Provision: st.Provider,
+		Consumer: st.Consumer, ConsumerNode: st.Node,
+		Class: st.Class, Error: st.Error, Since: st.Since, Attempts: st.Attempts,
+	})
+}
+
+// failingProviders is every consumer a provider still assigned where it ran says it keeps failing.
+//
+// **A provider no longer assigned is not asked about.** Its last word stays in the store, and is
+// not a problem: nothing runs there to fail anybody. Assigned again, its first success for each
+// consumer clears it.
+func failingProviders(ctx context.Context, inv *inventory.Inventory) ([]inventory.ProviderStanding, error) {
+	all, err := inv.FailingProviders(ctx)
+	if err != nil {
+		return nil, fmt.Errorf("what providers say they keep failing cannot be read: %w", err)
+	}
+	assigned := map[string]map[string]bool{}
+	var out []inventory.ProviderStanding
+	for _, s := range all {
+		on, asked := assigned[s.ProviderNode]
+		if !asked {
+			modules, err := inv.Assigned(ctx, s.ProviderNode)
+			if err != nil {
+				// A provider on a machine the mesh no longer knows has nothing running to fail anybody.
+				modules = nil
+			}
+			on = map[string]bool{}
+			for _, m := range modules {
+				on[m] = true
+			}
+			assigned[s.ProviderNode] = on
+		}
+		if on[s.Module] {
+			out = append(out, s)
+		}
+	}
+	return out, nil
+}
+
+// failingLines is how status says them: one consumer per entry, the error under it, and a provider
+// that stopped repeating itself said so.
+func failingLines(list []inventory.ProviderStanding, now time.Time) []string {
+	var out []string
+	for _, s := range list {
+		where := s.Module
+		if s.ProviderNode != "" {
+			where += " on " + s.ProviderNode
+		}
+		whom := s.Consumer
+		if s.ConsumerNode != "" {
+			whom += " (" + s.ConsumerNode + ")"
+		}
+		out = append(out, fmt.Sprintf("  %-24s fails %s: %s, for %s (%d attempts since %s)",
+			where, whom, orUnclassed(s.Class), roughly(now.Sub(s.Since)), s.Attempts,
+			s.Since.Local().Format("2006-01-02 15:04")))
+		if e := strings.TrimSpace(s.Error); e != "" {
+			out = append(out, fmt.Sprintf("  %-24s %s", "", firstLine(e)))
+		}
+		if s.Quiet(now) {
+			out = append(out, fmt.Sprintf("  %-24s not said again for %s — the provider has stopped "+
+				"saying anything, so this is its last word", "", roughly(now.Sub(s.SaidAt))))
+		}
+	}
+	return out
+}
+
+func orUnclassed(class string) string {
+	if class == "" {
+		return "failing"
+	}
+	return class
+}
+
+// printFailing is the status section, said when there is anything to say.
+func printFailing(list []inventory.ProviderStanding, now time.Time) {
+	if len(list) == 0 {
+		return
+	}
+	fmt.Printf("%d consumer(s) a provider keeps failing (ADR 0224):\n\n", len(list))
+	for _, line := range failingLines(list, now) {
+		fmt.Println(line)
+	}
+	fmt.Printf("\n  the provider's journal has every attempt; it says recovered on its next success\n\n")
+}
+
+// failingOn is the standings that concern one machine: a provider running there, or a consumer.
+func failingOn(list []inventory.ProviderStanding, node string) []inventory.ProviderStanding {
+	var out []inventory.ProviderStanding
+	for _, s := range list {
+		if s.ProviderNode == node || s.ConsumerNode == node {
+			out = append(out, s)
+		}
+	}
+	return out
+}