Hold a consumer's findings under its unhealthy provider, and say them once there (hq ADR 0240, to-be 48 Phase C)

With twelve consumers of the database provision, one provider down would be
twelve conditions for one fault and twelve gates failed for something none of
them did. A consumer's check names the provision it exercises; while the
provider composed for it — its recorded binding, or the machine its credential
comes from — is unhealthy on the record, what that check finds raises nothing
of its own: the provider's condition lists it as waiting and is urgent, and
the consumer's gate waits, past its bound too, rather than putting a build
back. Liveness findings and checks naming no provision stay the consumer's own,
and once the provider is healthy a consumer still failing is raised at once.

cmd/mesh-controller/module_health.go

@@ -8,6 +8,7 @@ import (
 	"sync/atomic"
 	"time"
 
+	"github.com/novox/mesh-controller/internal/catalogue"
 	"github.com/novox/mesh-controller/internal/conditions"
 	"github.com/novox/mesh-controller/internal/inventory"
 	"github.com/novox/mesh-controller/internal/link"
@@ -93,13 +94,15 @@ func stateHealth(ctx context.Context, inv *inventory.Inventory, k *conditions.Ke
 	if k == nil {
 		return nil
 	}
-	return judgeModuleHealth(ctx, k, node, unhealthy, streaks, now)
+	return judgeModuleHealth(ctx, inv, k, node, unhealthy, streaks, now)
 }
 
 // judgeModuleHealth raises a module's condition on a machine on the second statement in a row that says a
 // resource of it is unhealthy — or on the first while it is already open — and clears every one this
-// statement no longer says.
-func judgeModuleHealth(ctx context.Context, k *conditions.Keeper, node string,
+// statement no longer says. **A consumer whose findings wait on an unhealthy provider is held** (to-be 48
+// §6): raised as nothing of its own, listed at the provider's condition, which is urgent while anyone
+// waits on it.
+func judgeModuleHealth(ctx context.Context, inv *inventory.Inventory, k *conditions.Keeper, node string,
 	unhealthy map[string][]inventory.ResourceHealth, streaks map[string]int, now time.Time) error {
 	open, err := k.Open(ctx)
 	if err != nil {
@@ -111,6 +114,12 @@ func judgeModuleHealth(ctx context.Context, k *conditions.Keeper, node string,
 			standing[c.Key] = c
 		}
 	}
+	var hold *holding
+	if inv != nil {
+		if hold, err = readHolding(ctx, inv, open); err != nil {
+			return err
+		}
+	}
 	var problems []string
 	modules := make([]string, 0, len(unhealthy))
 	for m := range unhealthy {
@@ -118,8 +127,23 @@ func judgeModuleHealth(ctx context.Context, k *conditions.Keeper, node string,
 	}
 	sort.Strings(modules)
 	seen := map[string]bool{}
+	heldOn := map[string]string{}
+	providers := map[catalogue.Chosen]bool{}
 	for _, m := range modules {
 		o := moduleUnhealthyObservation(m, node, unhealthy[m])
+		if hold != nil {
+			if p, held := hold.heldUnder(node, m, unhealthy[m]); held {
+				// Held under the provider's condition: nothing of its own, and the provider's says it waits.
+				heldOn[o.Key()] = p.Module + " on " + p.Node
+				providers[p] = true
+				continue
+			}
+			if waiters := hold.waitersOn(catalogue.Chosen{Node: node, Module: m}); len(waiters) > 0 {
+				o.Severity = conditions.Urgent
+				o.Said += "; " + waitingWords(waiters)
+				o.Summary += fmt.Sprintf("; %d consumer(s) wait on it", len(waiters))
+			}
+		}
 		seen[o.Key()] = true
 		c, isOpen := standing[o.Key()]
 		if streaks[m] < moduleUnhealthyAfter && !isOpen {
@@ -137,7 +161,18 @@ func judgeModuleHealth(ctx context.Context, k *conditions.Keeper, node string,
 			continue
 		}
 		module := strings.TrimSuffix(c.Subject.ID, "."+node)
-		if _, err := k.Clear(ctx, key, fmt.Sprintf("%s says no resource of %s is unhealthy", node, module)); err != nil {
+		why := fmt.Sprintf("%s says no resource of %s is unhealthy", node, module)
+		if on, held := heldOn[key]; held {
+			why = fmt.Sprintf("what %s finds on %s waits on %s, which is unhealthy: held under its condition", module, node, on)
+		}
+		if _, err := k.Clear(ctx, key, why); err != nil {
+			problems = append(problems, err.Error())
+		}
+	}
+	// And each provider a consumer here now waits on, when its own condition is open: said again with who
+	// waits on it, so the wait is listed at the provider whichever machine's statement arrived first.
+	for p := range providers {
+		if err := sayWaiters(ctx, k, hold, p, now); err != nil {
 			problems = append(problems, err.Error())
 		}
 	}
@@ -147,6 +182,37 @@ func judgeModuleHealth(ctx context.Context, k *conditions.Keeper, node string,
 	return nil
 }
 
+// sayWaiters observes a provider's open condition again, with who waits on it, from its machine's newest
+// statement. Nothing when its condition is not open: it is raised by its own statements, on its own looks.
+func sayWaiters(ctx context.Context, k *conditions.Keeper, hold *holding, p catalogue.Chosen, now time.Time) error {
+	var raisedAt *conditions.Condition
+	for i, c := range hold.open {
+		if c.Key == moduleUnhealthyKey(p.Module, p.Node) {
+			raisedAt = &hold.open[i]
+		}
+	}
+	if raisedAt == nil {
+		return nil
+	}
+	var rs []inventory.ResourceHealth
+	for _, r := range hold.healths[p.Node].Resources {
+		if r.Module == p.Module && r.State == link.StateUnhealthy {
+			rs = append(rs, r)
+		}
+	}
+	if len(rs) == 0 {
+		return nil
+	}
+	o := moduleUnhealthyObservation(p.Module, p.Node, rs)
+	if waiters := hold.waitersOn(p); len(waiters) > 0 {
+		o.Severity = conditions.Urgent
+		o.Said += "; " + waitingWords(waiters)
+		o.Summary += fmt.Sprintf("; %d consumer(s) wait on it", len(waiters))
+	}
+	_, err := k.Observe(ctx, o)
+	return err
+}
+
 // moduleUnhealthyObservation is a module unhealthy on a machine, in words: the summary names the module,
 // the machine and what is wrong with each resource; the detail — targets, streaks, since — is evidence.
 func moduleUnhealthyObservation(module, node string, rs []inventory.ResourceHealth) conditions.Observation {
@@ -211,6 +277,10 @@ func moduleHealthWord(module, machine string, since time.Time, f gateFacts) (hea
 		case link.StateStarting:
 			return healthNotYet, fmt.Sprintf("its %s %s on %s is still starting", r.Kind, r.Resource, machine)
 		case link.StateUnhealthy:
+			if on, held := f.heldOn[module+"@"+machine]; held {
+				return healthWaiting, fmt.Sprintf("its %s %s on %s waits on %s, which is unhealthy", r.Kind,
+					r.Resource, machine, on)
+			}
 			return healthNotYet, fmt.Sprintf("its %s %s on %s %s", r.Kind, r.Resource, machine, reasonWords(r))
 		default:
 			return healthNotYet, fmt.Sprintf("its %s %s on %s is %s%s", r.Kind, r.Resource, machine, r.State,