Merge pull request 'Hold a consumer's findings under its unhealthy provider, and say them once there (hq ADR 0240, to-be 48 Phase C)' (#111) from feat/health-the-provider-hold into main

cmd/mesh-controller/gate.go

@@ -77,6 +77,10 @@ type health int
 
 const (
 	healthGood health = iota
+	// healthWaiting is not a pass and not a fault: what the module's checks find waits on an unhealthy
+	// provider (ADR 0240 rule 5), so the judging waits — past the bound too — rather than putting back a
+	// build for something it did not do.
+	healthWaiting
 	healthNotYet
 	healthBroken
 )
@@ -109,6 +113,8 @@ type gateFacts struct {
 	// healthErr is why they could not be read.
 	health    map[string]inventory.NodeHealth
 	healthErr error
+	// heldOn is, per "<module>@<machine>", the provider its findings are held under (ADR 0240 rule 5).
+	heldOn map[string]string
 }
 
 // gatherGateFacts reads what a judging needs, from the store, the bus and this controller's memory. A
@@ -147,6 +153,17 @@ var gatherGateFacts = func(ctx context.Context, open *stores, component string)
 	} else {
 		f.servedErr = errors.New("this process does not serve the mesh, so it cannot ask the bus who serves what")
 	}
+	// Whose findings wait on an unhealthy provider (ADR 0240 rule 5): their gates wait, not fail.
+	if f.healthErr == nil && f.openErr == nil {
+		if hold, err := readHolding(ctx, inv, f.open); err == nil {
+			f.heldOn = map[string]string{}
+			for machine := range f.health {
+				for module, p := range hold.heldModules(machine) {
+					f.heldOn[module+"@"+machine] = p.Module + " on " + p.Node
+				}
+			}
+		}
+	}
 	if theLease != nil {
 		h, found, err := theLease.holder(ctx)
 		switch {
@@ -435,6 +452,10 @@ func judgeMoves(ctx context.Context, open *stores, g *inventory.PlanGate, pairs
 		// What broke is put back; what was only not yet healthy beside it is too — they moved together.
 		g.Failing = failing
 		decide(g, inventory.GateFailed, why, now)
+	case worst == healthWaiting:
+		// Waiting on a provider that is unhealthy: not a pass, and not a failure at the bound either —
+		// the provider's own condition says what is wrong (ADR 0240 rule 5).
+		g.Passes, g.LastPass, g.Last, g.Failing = 0, nil, why, failing
 	case worst == healthNotYet:
 		g.Passes, g.LastPass, g.Last, g.Failing = 0, nil, why, failing
 		if now.Sub(*g.Since) > gateBound {

cmd/mesh-controller/module_health.go

@@ -8,6 +8,7 @@ import (
 	"sync/atomic"
 	"time"
 
+	"github.com/novox/mesh-controller/internal/catalogue"
 	"github.com/novox/mesh-controller/internal/conditions"
 	"github.com/novox/mesh-controller/internal/inventory"
 	"github.com/novox/mesh-controller/internal/link"
@@ -93,13 +94,15 @@ func stateHealth(ctx context.Context, inv *inventory.Inventory, k *conditions.Ke
 	if k == nil {
 		return nil
 	}
-	return judgeModuleHealth(ctx, k, node, unhealthy, streaks, now)
+	return judgeModuleHealth(ctx, inv, k, node, unhealthy, streaks, now)
 }
 
 // judgeModuleHealth raises a module's condition on a machine on the second statement in a row that says a
 // resource of it is unhealthy — or on the first while it is already open — and clears every one this
-// statement no longer says.
-func judgeModuleHealth(ctx context.Context, k *conditions.Keeper, node string,
+// statement no longer says. **A consumer whose findings wait on an unhealthy provider is held** (to-be 48
+// §6): raised as nothing of its own, listed at the provider's condition, which is urgent while anyone
+// waits on it.
+func judgeModuleHealth(ctx context.Context, inv *inventory.Inventory, k *conditions.Keeper, node string,
 	unhealthy map[string][]inventory.ResourceHealth, streaks map[string]int, now time.Time) error {
 	open, err := k.Open(ctx)
 	if err != nil {
@@ -111,6 +114,12 @@ func judgeModuleHealth(ctx context.Context, k *conditions.Keeper, node string,
 			standing[c.Key] = c
 		}
 	}
+	var hold *holding
+	if inv != nil {
+		if hold, err = readHolding(ctx, inv, open); err != nil {
+			return err
+		}
+	}
 	var problems []string
 	modules := make([]string, 0, len(unhealthy))
 	for m := range unhealthy {
@@ -118,8 +127,23 @@ func judgeModuleHealth(ctx context.Context, k *conditions.Keeper, node string,
 	}
 	sort.Strings(modules)
 	seen := map[string]bool{}
+	heldOn := map[string]string{}
+	providers := map[catalogue.Chosen]bool{}
 	for _, m := range modules {
 		o := moduleUnhealthyObservation(m, node, unhealthy[m])
+		if hold != nil {
+			if p, held := hold.heldUnder(node, m, unhealthy[m]); held {
+				// Held under the provider's condition: nothing of its own, and the provider's says it waits.
+				heldOn[o.Key()] = p.Module + " on " + p.Node
+				providers[p] = true
+				continue
+			}
+			if waiters := hold.waitersOn(catalogue.Chosen{Node: node, Module: m}); len(waiters) > 0 {
+				o.Severity = conditions.Urgent
+				o.Said += "; " + waitingWords(waiters)
+				o.Summary += fmt.Sprintf("; %d consumer(s) wait on it", len(waiters))
+			}
+		}
 		seen[o.Key()] = true
 		c, isOpen := standing[o.Key()]
 		if streaks[m] < moduleUnhealthyAfter && !isOpen {
@@ -137,7 +161,18 @@ func judgeModuleHealth(ctx context.Context, k *conditions.Keeper, node string,
 			continue
 		}
 		module := strings.TrimSuffix(c.Subject.ID, "."+node)
-		if _, err := k.Clear(ctx, key, fmt.Sprintf("%s says no resource of %s is unhealthy", node, module)); err != nil {
+		why := fmt.Sprintf("%s says no resource of %s is unhealthy", node, module)
+		if on, held := heldOn[key]; held {
+			why = fmt.Sprintf("what %s finds on %s waits on %s, which is unhealthy: held under its condition", module, node, on)
+		}
+		if _, err := k.Clear(ctx, key, why); err != nil {
+			problems = append(problems, err.Error())
+		}
+	}
+	// And each provider a consumer here now waits on, when its own condition is open: said again with who
+	// waits on it, so the wait is listed at the provider whichever machine's statement arrived first.
+	for p := range providers {
+		if err := sayWaiters(ctx, k, hold, p, now); err != nil {
 			problems = append(problems, err.Error())
 		}
 	}
@@ -147,6 +182,37 @@ func judgeModuleHealth(ctx context.Context, k *conditions.Keeper, node string,
 	return nil
 }
 
+// sayWaiters observes a provider's open condition again, with who waits on it, from its machine's newest
+// statement. Nothing when its condition is not open: it is raised by its own statements, on its own looks.
+func sayWaiters(ctx context.Context, k *conditions.Keeper, hold *holding, p catalogue.Chosen, now time.Time) error {
+	var raisedAt *conditions.Condition
+	for i, c := range hold.open {
+		if c.Key == moduleUnhealthyKey(p.Module, p.Node) {
+			raisedAt = &hold.open[i]
+		}
+	}
+	if raisedAt == nil {
+		return nil
+	}
+	var rs []inventory.ResourceHealth
+	for _, r := range hold.healths[p.Node].Resources {
+		if r.Module == p.Module && r.State == link.StateUnhealthy {
+			rs = append(rs, r)
+		}
+	}
+	if len(rs) == 0 {
+		return nil
+	}
+	o := moduleUnhealthyObservation(p.Module, p.Node, rs)
+	if waiters := hold.waitersOn(p); len(waiters) > 0 {
+		o.Severity = conditions.Urgent
+		o.Said += "; " + waitingWords(waiters)
+		o.Summary += fmt.Sprintf("; %d consumer(s) wait on it", len(waiters))
+	}
+	_, err := k.Observe(ctx, o)
+	return err
+}
+
 // moduleUnhealthyObservation is a module unhealthy on a machine, in words: the summary names the module,
 // the machine and what is wrong with each resource; the detail — targets, streaks, since — is evidence.
 func moduleUnhealthyObservation(module, node string, rs []inventory.ResourceHealth) conditions.Observation {
@@ -211,6 +277,10 @@ func moduleHealthWord(module, machine string, since time.Time, f gateFacts) (hea
 		case link.StateStarting:
 			return healthNotYet, fmt.Sprintf("its %s %s on %s is still starting", r.Kind, r.Resource, machine)
 		case link.StateUnhealthy:
+			if on, held := f.heldOn[module+"@"+machine]; held {
+				return healthWaiting, fmt.Sprintf("its %s %s on %s waits on %s, which is unhealthy", r.Kind,
+					r.Resource, machine, on)
+			}
 			return healthNotYet, fmt.Sprintf("its %s %s on %s %s", r.Kind, r.Resource, machine, reasonWords(r))
 		default:
 			return healthNotYet, fmt.Sprintf("its %s %s on %s is %s%s", r.Kind, r.Resource, machine, r.State,

cmd/mesh-controller/provider_hold.go

@@ -0,0 +1,192 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"sort"
+	"strings"
+
+	"github.com/novox/mesh-controller/internal/catalogue"
+	"github.com/novox/mesh-controller/internal/conditions"
+	"github.com/novox/mesh-controller/internal/inventory"
+	"github.com/novox/mesh-controller/internal/link"
+)
+
+// A provider down is said once, at the provider (novox/hq ADR 0240 rule 5, to-be 48 §6, Phase C).
+//
+// With twelve consumers of the database provision and thirty-six of a route, one provider down would be
+// twelve conditions for one fault and twelve gates failed for something none of them did. So a consumer's
+// check names, in `needs`, the provision it exercises; while **that provision's provider for this
+// consumer** — the one the controller composed the consumer against: its recorded binding (ADR 0232), or
+// the provider its credential for the provision is from — is unhealthy on the record, what the check finds
+// is held under the provider's condition: listed there as waiting on it, raised as nothing of its own, and
+// the consumer's gate waits rather than fails. The provider's condition is urgent while consumers wait.
+//
+// **Only what the check finds is held.** A consumer that is down or restarting is its own, whatever its
+// provider does; so is anything a check that names no provision finds, and anything found while the
+// provider is healthy. A machine-level fault is never pinned on a module (issue 281), and this does not
+// change that.
+
+// holding is what one reading of the record needs to say who waits on whom: every machine's newest
+// statement, the open conditions, and the catalogue — read once, asked many times.
+type holding struct {
+	ctx     context.Context
+	inv     *inventory.Inventory
+	healths map[string]inventory.NodeHealth
+	open    []conditions.Condition
+	shelf   map[string]catalogue.Manifest
+	// providers memoises providerFor by machine, consumer and provision.
+	providers map[string]providerLookup
+}
+
+type providerLookup struct {
+	chosen catalogue.Chosen
+	ok     bool
+}
+
+// readHolding reads what the hold is judged from.
+func readHolding(ctx context.Context, inv *inventory.Inventory, open []conditions.Condition) (*holding, error) {
+	healths, err := inv.Healths(ctx)
+	if err != nil {
+		return nil, err
+	}
+	return &holding{ctx: ctx, inv: inv, healths: healths, open: open, providers: map[string]providerLookup{}}, nil
+}
+
+// heldFinding says a resource's state is a finding of its declared check that names a provision: what
+// may be held. Down and restarting are liveness, the resource's own.
+func heldFinding(r inventory.ResourceHealth) bool {
+	return r.State == link.StateUnhealthy && r.Check != "" && r.Needs != "" &&
+		r.Reason != "down" && r.Reason != "restarting"
+}
+
+// providerFor is the provider composed for a consumer's provision: its recorded binding, else the
+// machine its credential for the provision comes from and the module there that provides it.
+func (h *holding) providerFor(machine, consumer, provision string) (catalogue.Chosen, bool) {
+	key := machine + "\x00" + consumer + "\x00" + provision
+	if p, known := h.providers[key]; known {
+		return p.chosen, p.ok
+	}
+	chosen, ok := h.lookUpProvider(machine, consumer, provision)
+	h.providers[key] = providerLookup{chosen, ok}
+	return chosen, ok
+}
+
+func (h *holding) lookUpProvider(machine, consumer, provision string) (catalogue.Chosen, bool) {
+	if bound, err := h.inv.BindingsFor(h.ctx, machine); err == nil {
+		if c, ok := bound[consumer][provision]; ok && c.Node != "" && c.Module != "" {
+			return c, true
+		}
+	}
+	secrets, err := h.inv.SecretsOf(h.ctx, machine, consumer)
+	if err != nil {
+		return catalogue.Chosen{}, false
+	}
+	for _, s := range secrets {
+		if s.Name != provision || s.Provider == "" {
+			continue
+		}
+		if h.shelf == nil {
+			if h.shelf, err = h.inv.Catalogue(h.ctx); err != nil {
+				return catalogue.Chosen{}, false
+			}
+		}
+		assigned, err := h.inv.Assigned(h.ctx, s.Provider)
+		if err != nil {
+			return catalogue.Chosen{}, false
+		}
+		for _, module := range assigned {
+			for _, offer := range h.shelf[module].Offers() {
+				if offer == provision {
+					return catalogue.Chosen{Node: s.Provider, Module: module}, true
+				}
+			}
+		}
+	}
+	return catalogue.Chosen{}, false
+}
+
+// unhealthy says a provider is unhealthy on the record: its condition is open, or its machine's newest
+// statement says a resource of it is unhealthy.
+func (h *holding) unhealthy(p catalogue.Chosen) bool {
+	key := moduleUnhealthyKey(p.Module, p.Node)
+	for _, c := range h.open {
+		if c.Key == key {
+			return true
+		}
+	}
+	for _, r := range h.healths[p.Node].Resources {
+		if r.Module == p.Module && r.State == link.StateUnhealthy {
+			return true
+		}
+	}
+	return false
+}
+
+// heldUnder is the provider a consumer's unhealthy resources wait on: when every one of them is a finding
+// of a check naming a provision whose provider for this consumer is unhealthy on the record. False when any
+// is the consumer's own.
+func (h *holding) heldUnder(machine, module string, rs []inventory.ResourceHealth) (catalogue.Chosen, bool) {
+	var on catalogue.Chosen
+	for _, r := range rs {
+		if r.State != link.StateUnhealthy {
+			continue
+		}
+		if !heldFinding(r) {
+			return catalogue.Chosen{}, false
+		}
+		p, ok := h.providerFor(machine, module, r.Needs)
+		if !ok || (p.Node == machine && p.Module == module) || !h.unhealthy(p) {
+			return catalogue.Chosen{}, false
+		}
+		on = p
+	}
+	return on, on.Module != ""
+}
+
+// waitersOn is every consumer held under a provider, as "<module> on <machine>", sorted.
+func (h *holding) waitersOn(p catalogue.Chosen) []string {
+	var out []string
+	for machine, nh := range h.healths {
+		byModule := map[string][]inventory.ResourceHealth{}
+		for _, r := range nh.Resources {
+			if r.Module != "" && r.State == link.StateUnhealthy {
+				byModule[r.Module] = append(byModule[r.Module], r)
+			}
+		}
+		for module, rs := range byModule {
+			if on, held := h.heldUnder(machine, module, rs); held && on == p {
+				out = append(out, module+" on "+machine)
+			}
+		}
+	}
+	sort.Strings(out)
+	return out
+}
+
+// heldModules is, for one machine's statement, each module whose finding is held, with the provider.
+func (h *holding) heldModules(machine string) map[string]catalogue.Chosen {
+	out := map[string]catalogue.Chosen{}
+	byModule := map[string][]inventory.ResourceHealth{}
+	for _, r := range h.healths[machine].Resources {
+		if r.Module != "" && r.State == link.StateUnhealthy {
+			byModule[r.Module] = append(byModule[r.Module], r)
+		}
+	}
+	for module, rs := range byModule {
+		if on, held := h.heldUnder(machine, module, rs); held {
+			out[module] = on
+		}
+	}
+	return out
+}
+
+// moduleUnhealthyKey is a module's health condition's key on a machine.
+func moduleUnhealthyKey(module, machine string) string {
+	return conditions.ScopeModule + "." + module + "." + machine + ".unhealthy"
+}
+
+// waitingWords is the provider's evidence that consumers wait on it.
+func waitingWords(waiters []string) string {
+	return fmt.Sprintf("waiting on it — %s", strings.Join(waiters, ", "))
+}

cmd/mesh-controller/provider_hold_test.go

@@ -0,0 +1,172 @@
+package main
+
+import (
+	"context"
+	"slices"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/novox/mesh-controller/internal/catalogue"
+	"github.com/novox/mesh-controller/internal/conditions"
+	"github.com/novox/mesh-controller/internal/inventory"
+	"github.com/novox/mesh-controller/internal/link"
+)
+
+// A provider down is said once, at the provider (novox/hq ADR 0240 rule 5, "how it is checked"): one
+// unhealthy database provider and three consumers failing their checks that need it — one condition, at the
+// provider, urgent, the consumers listed as waiting; their gates wait, not fail; once the provider is
+// healthy, a consumer still failing is its own. A consumer failing while its provider is healthy is raised
+// on its own from the start.
+func TestOneProviderDownAndThreeConsumersFailingAreOneCondition(t *testing.T) {
+	open := aMesh(t)
+	ctx := t.Context()
+	inv := open.inventory
+	k := conditionsFrom
+	register(t, open, catalogue.Manifest{Module: "db", Version: "1",
+		Provides: []catalogue.Offer{{Name: "postgres-database", Scope: catalogue.ScopeMesh}}})
+	consumers := []string{"shop", "wiki", "crm"}
+	for _, c := range consumers {
+		register(t, open, catalogue.Manifest{Module: c, Version: "1", Requires: []string{"postgres-database"}})
+	}
+	if _, err := inv.Assign(ctx, "anchor", "db"); err != nil {
+		t.Fatal(err)
+	}
+	machineOf := map[string]string{"shop": "laptop", "wiki": "laptop", "crm": "anchor"}
+	for _, c := range consumers {
+		if _, err := inv.Assign(ctx, machineOf[c], c); err != nil {
+			t.Fatal(err)
+		}
+		if err := inv.RecordBindings(ctx, machineOf[c], []inventory.Binding{{Machine: machineOf[c], Consumer: c,
+			Provision: "postgres-database", Provider: catalogue.Chosen{Node: "anchor", Module: "db"}}}); err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	at := h0
+	say := func(machine string, rs ...link.ResourceHealth) {
+		t.Helper()
+		at = at.Add(time.Second)
+		for i := range rs {
+			rs[i].Since = at
+		}
+		if err := stateHealth(ctx, inv, k, machine, link.Health{Contract: link.ReadinessContract, At: at, Resources: rs}, at); err != nil {
+			t.Fatal(err)
+		}
+	}
+	dbDown := link.ResourceHealth{Module: "db", Resource: "db.server", Kind: "container", Target: "db", State: link.StateUnhealthy,
+		Reason: "its command: the database refuses connections", Check: "exec"}
+	dbUp := dbDown
+	dbUp.State, dbUp.Reason = link.StateHealthy, ""
+	failing := func(module string) link.ResourceHealth {
+		return link.ResourceHealth{Module: module, Resource: module + ".web", Kind: "container", Target: module,
+			State: link.StateUnhealthy, Reason: "http /health on web: answered 500", Check: "http", Needs: "postgres-database"}
+	}
+	fine := func(module string) link.ResourceHealth {
+		r := failing(module)
+		r.State, r.Reason = link.StateHealthy, ""
+		return r
+	}
+	openKeys := func() []conditions.Condition {
+		t.Helper()
+		list, err := k.Open(ctx)
+		if err != nil {
+			t.Fatal(err)
+		}
+		return list
+	}
+
+	// Two looks of the provider down and its consumers failing, in either order on each machine.
+	for look := 0; look < 2; look++ {
+		say("laptop", failing("shop"), failing("wiki"))
+		say("anchor", dbDown, failing("crm"))
+	}
+	raised := openKeys()
+	if len(raised) != 1 || raised[0].Key != "module.db.anchor.unhealthy" {
+		var keys []string
+		for _, c := range raised {
+			keys = append(keys, c.Key)
+		}
+		t.Fatalf("one provider down and three consumers failing raised %v; want the provider's alone", keys)
+	}
+	c := raised[0]
+	if c.Severity != conditions.Urgent {
+		t.Errorf("consumers wait on the provider and its condition is %s", c.Severity)
+	}
+	said := c.Evidence[0].Said
+	for _, w := range []string{"shop on laptop", "wiki on laptop", "crm on anchor"} {
+		if !strings.Contains(said, w) {
+			t.Errorf("the provider's condition does not list %s as waiting on it: %s", w, said)
+		}
+	}
+
+	// Their gates wait, not fail: the judging is neither a pass nor a fault, past the bound too.
+	f, err := gatherGateFacts(ctx, open, "")
+	if err != nil {
+		t.Fatal(err)
+	}
+	f.open, f.openErr, f.judged = raised, nil, false
+	for _, m := range consumers {
+		h, why := moduleHealthWord(m, machineOf[m], h0, f)
+		if h != healthWaiting || !strings.Contains(why, "waits on db on anchor") {
+			t.Errorf("%s's gate: %v %q; want it waiting on its provider", m, h, why)
+		}
+	}
+	// And a whole judging past the bound puts nothing back: it waits.
+	long := h0.Add(-time.Hour)
+	for _, m := range []string{"anchor", "laptop"} {
+		f.reports[m] = inventory.Reported{Node: m, Outcome: inventory.OutcomeApplied, At: &f.now, Current: true}
+	}
+	gatherWas := gatherGateFacts
+	defer func() { gatherGateFacts = gatherWas }()
+	gatherGateFacts = func(context.Context, *stores, string) (gateFacts, error) { return f, nil }
+	g := &inventory.PlanGate{Machines: []string{"laptop"}, Since: &long}
+	verdict, err := judgeMoves(ctx, open, g, []judged{{module: "shop", node: "laptop"}}, time.Now())
+	if err != nil || verdict != "" || !strings.Contains(g.Last, "waits on db on anchor") {
+		t.Fatalf("a held consumer's gate past its bound: verdict %q (%v), last %q; want it waiting", verdict, err, g.Last)
+	}
+
+	// The provider healthy again: a consumer still failing is now its own, at once (its streak stood).
+	say("anchor", dbUp, fine("crm"))
+	say("laptop", failing("shop"), fine("wiki"))
+	var keys []string
+	for _, c := range openKeys() {
+		keys = append(keys, c.Key)
+	}
+	if !slices.Equal(keys, []string{"module.shop.laptop.unhealthy"}) {
+		t.Fatalf("after the provider recovered: %v; want shop's own and nothing else", keys)
+	}
+}
+
+// A consumer failing while its provider is healthy is raised on its own.
+func TestAConsumerFailingBesideAHealthyProviderIsItsOwn(t *testing.T) {
+	open := aMesh(t)
+	ctx := t.Context()
+	inv := open.inventory
+	register(t, open, catalogue.Manifest{Module: "db", Version: "1",
+		Provides: []catalogue.Offer{{Name: "postgres-database", Scope: catalogue.ScopeMesh}}})
+	register(t, open, catalogue.Manifest{Module: "shop", Version: "1", Requires: []string{"postgres-database"}})
+	for _, a := range [][2]string{{"anchor", "db"}, {"laptop", "shop"}} {
+		if _, err := inv.Assign(ctx, a[0], a[1]); err != nil {
+			t.Fatal(err)
+		}
+	}
+	if err := inv.RecordBindings(ctx, "laptop", []inventory.Binding{{Machine: "laptop", Consumer: "shop",
+		Provision: "postgres-database", Provider: catalogue.Chosen{Node: "anchor", Module: "db"}}}); err != nil {
+		t.Fatal(err)
+	}
+	healthy := link.ResourceHealth{Module: "db", Resource: "db.server", Kind: "container", Target: "db", State: link.StateHealthy, Check: "exec"}
+	shop := link.ResourceHealth{Module: "shop", Resource: "shop.web", Kind: "container", Target: "shop", State: link.StateUnhealthy,
+		Reason: "http /health on web: answered 500", Check: "http", Needs: "postgres-database"}
+	for i := 1; i <= 2; i++ {
+		at := h0.Add(time.Duration(i) * time.Minute)
+		_ = stateHealth(ctx, inv, conditionsFrom, "anchor", link.Health{Contract: 2, At: at, Resources: []link.ResourceHealth{healthy}}, at)
+		if err := stateHealth(ctx, inv, conditionsFrom, "laptop", link.Health{Contract: 2, At: at, Resources: []link.ResourceHealth{shop}}, at); err != nil {
+			t.Fatal(err)
+		}
+	}
+	list, _ := conditionsFrom.Open(ctx)
+	if len(list) != 1 || list[0].Key != "module.shop.laptop.unhealthy" {
+		t.Fatalf("a consumer failing beside a healthy provider raised %v", list)
+	}
+}