Hold consumers under a provider waiting for the operator, and say a wait beside another condition (issue 405)

A provider whose only part not healthy waits for the operator's secret or
setting let its consumers raise their own unhealthy conditions, and a
wait beside a relogin, a directory used as found or a fault was not said
until the other cleared.

cmd/mesh-controller/module_health.go

@@ -153,11 +153,9 @@ func judgeModuleHealth(ctx context.Context, inv *inventory.Inventory, k *conditi
 			standing[c.Key] = c
 		}
 	}
-	var hold *holding
-	if inv != nil {
-		if hold, err = readHolding(ctx, inv, open); err != nil {
-			return err
-		}
+	hold, err := readHoldingFor(ctx, inv, open)
+	if err != nil {
+		return err
 	}
 	var problems []string
 	modules := make([]string, 0, len(unhealthy))
@@ -176,6 +174,11 @@ func judgeModuleHealth(ctx context.Context, inv *inventory.Inventory, k *conditi
 		// the operator's, never urgent, its words naming the act.
 		if waits, waiting := operatorWait(m, unhealthy[m]); waiting {
 			o := needsOperatorObservation(m, node, waits, unhealthy[m])
+			// **A provider waiting for the operator says who waits on it** (novox/hq issue 405), as one waiting for a
+			// login does: its consumers are held under it.
+			if hold != nil {
+				sayWaitingOn(&o, hold.waitersOn(catalogue.Chosen{Node: node, Module: m}))
+			}
 			seen[o.Key()] = true
 			became[m] = kindNeedsOperator
 			if _, isOpen := standing[o.Key()]; streaks[m] < moduleUnhealthyAfter && !isOpen {
@@ -186,6 +189,20 @@ func judgeModuleHealth(ctx context.Context, inv *inventory.Inventory, k *conditi
 			}
 			continue
 		}
+		// **A wait for the operator beside anything else is said too** (novox/hq issue 405): a module with a checked
+		// wait beside a new login owed, a directory used as found or a fault of its own is said as that, and the
+		// operator's secret or setting it waits for was not mentioned until the other cleared. Its needs-operator
+		// condition stands beside the other, on the same looks; what follows judges the rest without the wait.
+		if waits, rest := besideAWait(m, unhealthy[m]); len(waits) > 0 {
+			o := needsOperatorObservation(m, node, waits, waitingOf(m, unhealthy[m]))
+			seen[o.Key()] = true
+			if _, isOpen := standing[o.Key()]; streaks[m] >= moduleUnhealthyAfter || isOpen {
+				if _, err := k.Observe(ctx, o); err != nil {
+					problems = append(problems, err.Error())
+				}
+			}
+			unhealthy[m] = rest
+		}
 		// **A directory used as found is said as that** (novox/hq issue 339): the operator's to hand over at the
 		// machine, never urgent — nothing is broken by the wait that a person was not told of — and its own kind,
 		// so the gate never reads it as a fault of the build that happened to be sent beside it.
@@ -259,7 +276,7 @@ func judgeModuleHealth(ctx context.Context, inv *inventory.Inventory, k *conditi
 			why = fmt.Sprintf("%s says %s no longer waits for the operator", node, module)
 		}
 		if on, held := heldOn[key]; held {
-			why = fmt.Sprintf("what %s finds on %s waits on %s, which is unhealthy: held under its condition", module, node, on)
+			why = fmt.Sprintf("what %s finds on %s waits on %s, which is not healthy: held under its condition", module, node, on)
 		}
 		// **A condition that became the other kind** is not "working again" (issue 318 review): its clearing
 		// line says what it became.
@@ -298,36 +315,72 @@ func judgeModuleHealth(ctx context.Context, inv *inventory.Inventory, k *conditi
 // sayWaiters observes a provider's open condition again, with who waits on it, from its machine's newest
 // statement. Nothing when its condition is not open: it is raised by its own statements, on its own looks.
 func sayWaiters(ctx context.Context, k *conditions.Keeper, hold *holding, p catalogue.Chosen, now time.Time) error {
-	var raisedAt *conditions.Condition
-	for i, c := range hold.open {
-		if c.Key == moduleUnhealthyKey(p.Module, p.Node) || c.Key == reloginKey(p.Module, p.Node) {
-			raisedAt = &hold.open[i]
-		}
-	}
-	if raisedAt == nil {
-		return nil
-	}
 	var rs []inventory.ResourceHealth
 	for _, r := range hold.healths[p.Node].Resources {
-		if r.Module == p.Module && r.State == link.StateUnhealthy {
+		if r.Module == p.Module && (r.State == link.StateUnhealthy || r.State == link.StateWaiting) {
 			rs = append(rs, r)
 		}
 	}
 	if len(rs) == 0 {
 		return nil
 	}
-	o := moduleUnhealthyObservation(p.Module, p.Node, rs)
-	if said, waits := personWait(p.Module, p.Node, rs); waits {
-		o = reloginObservation(p.Module, p.Node, said, operatorOn(ctx, hold.inv, p.Node), rs)
+	// The condition its own statement says it under: needs-operator while it only waits for the operator (novox/hq
+	// issue 405), else that for the rest of it, a wait beside it said on its own.
+	var o conditions.Observation
+	if waits, waiting := operatorWait(p.Module, rs); waiting {
+		o = needsOperatorObservation(p.Module, p.Node, waits, rs)
+	} else {
+		_, rest := besideAWait(p.Module, rs)
+		o = moduleUnhealthyObservation(p.Module, p.Node, rest)
+		if said, waits := personWait(p.Module, p.Node, rest); waits {
+			o = reloginObservation(p.Module, p.Node, said, operatorOn(ctx, hold.inv, p.Node), rest)
+		}
 	}
-	if o.Key() != raisedAt.Key {
-		return nil // its own statement says it next
+	raised := false
+	for _, c := range hold.open {
+		raised = raised || c.Key == o.Key()
+	}
+	if !raised {
+		return nil // not open yet, or open as another kind: its own statement says it next
 	}
 	sayWaitingOn(&o, hold.waitersOn(p))
 	_, err := k.Observe(ctx, o)
 	return err
 }
 
+// besideAWait is, for a module with something not healthy beside a part waiting for the operator, the waits (checked
+// already) and the rest without the waiting parts (novox/hq issue 405); no waits when nothing waits, or when the
+// module only waits (operatorWait says that whole). Pure.
+func besideAWait(module string, rs []inventory.ResourceHealth) ([]inventory.Wait, []inventory.ResourceHealth) {
+	if _, only := operatorWait(module, rs); only {
+		return nil, rs
+	}
+	var waits []inventory.Wait
+	var rest []inventory.ResourceHealth
+	for _, r := range rs {
+		if r.Module == module && r.State == link.StateWaiting {
+			waits = append(waits, r.Waits...)
+			continue
+		}
+		rest = append(rest, r)
+	}
+	if len(waits) == 0 {
+		return nil, rs
+	}
+	return waits, rest
+}
+
+// waitingOf is a module's waiting resources: the evidence of its wait.
+func waitingOf(module string, rs []inventory.ResourceHealth) []inventory.ResourceHealth {
+	var out []inventory.ResourceHealth
+	for _, r := range rs {
+		if r.Module == module && r.State == link.StateWaiting {
+			out = append(out, r)
+		}
+	}
+	return out
+}
+
 // reloginKey is a module's relogin-needed condition on a machine.
 func reloginKey(module, node string) string {
 	return conditions.Key(conditions.ScopeModule, module+"."+node, kindReloginNeeded)
@@ -591,7 +644,7 @@ func moduleHealthWord(module, machine string, since time.Time, f gateFacts) (hea
 			return healthNotYet, fmt.Sprintf("its %s %s on %s is still starting", r.Kind, r.Resource, machine)
 		case link.StateUnhealthy:
 			if on, held := f.heldOn[module+"@"+machine]; held {
-				return healthWaiting, fmt.Sprintf("its %s %s on %s waits on %s, which is unhealthy", r.Kind,
+				return healthWaiting, fmt.Sprintf("its %s %s on %s waits on %s, which is not healthy", r.Kind,
 					r.Resource, machine, on)
 			}
 			return healthNotYet, fmt.Sprintf("its %s %s on %s %s", r.Kind, r.Resource, machine, reasonWords(r))

cmd/mesh-controller/provider_hold.go

@@ -3,6 +3,7 @@ package main
 import (
 	"context"
 	"fmt"
+	"slices"
 	"sort"
 	"strings"
 
@@ -53,6 +54,15 @@ func readHolding(ctx context.Context, inv *inventory.Inventory, open []condition
 	return &holding{ctx: ctx, inv: inv, healths: healths, open: open, providers: map[string]providerLookup{}}, nil
 }
 
+// readHoldingFor is how a judging reads its holding: nothing without a store. A variable so a test can hand a
+// judging the record it holds under (novox/hq issue 405).
+var readHoldingFor = func(ctx context.Context, inv *inventory.Inventory, open []conditions.Condition) (*holding, error) {
+	if inv == nil {
+		return nil, nil
+	}
+	return readHolding(ctx, inv, open)
+}
+
 // heldFinding says a resource's state is a finding of its declared check that names a provision: what
 // may be held. Down and restarting are liveness, the resource's own.
 func heldFinding(r inventory.ResourceHealth) bool {
@@ -106,17 +116,20 @@ func (h *holding) lookUpProvider(machine, consumer, provision string) (catalogue
 	return catalogue.Chosen{}, false
 }
 
-// unhealthy says a provider is unhealthy on the record: its condition is open, or its machine's newest
-// statement says a resource of it is unhealthy.
+// unhealthy says a provider is not healthy on the record: its unhealthy or needs-operator condition is open, or its
+// machine's newest statement says a resource of it is unhealthy or waiting. **A provider waiting for the operator
+// holds its consumers too** (novox/hq issue 405): its secret or setting not given, what its consumers find is no more
+// theirs than when it is broken, and ADR 0240 rule 5 says it once, at the provider — here under its needs-operator
+// condition, which names the act.
 func (h *holding) unhealthy(p catalogue.Chosen) bool {
-	key := moduleUnhealthyKey(p.Module, p.Node)
+	keys := []string{moduleUnhealthyKey(p.Module, p.Node), needsOperatorKey(p.Module, p.Node)}
 	for _, c := range h.open {
-		if c.Key == key {
+		if slices.Contains(keys, c.Key) {
 			return true
 		}
 	}
 	for _, r := range h.healths[p.Node].Resources {
-		if r.Module == p.Module && r.State == link.StateUnhealthy {
+		if r.Module == p.Module && (r.State == link.StateUnhealthy || r.State == link.StateWaiting) {
 			return true
 		}
 	}

cmd/mesh-controller/waiting_provider_test.go

@@ -0,0 +1,234 @@
+package main
+
+import (
+	"context"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/novox/mesh-controller/internal/catalogue"
+	"github.com/novox/mesh-controller/internal/conditions"
+	"github.com/novox/mesh-controller/internal/inventory"
+	"github.com/novox/mesh-controller/internal/link"
+)
+
+// A provider waiting for the operator holds its consumers, and a wait beside another wait is said (novox/hq
+// issue 405, found in the review of ADR 0283's controller change).
+//
+// The shapes are those of issue 386 (mounts waiting for smb-password-games: waitingResource, passwordWait) and of
+// the openrazer wait on the workstation of 2026-10-11 (relogin, userUnit): an account in its group whose session
+// began before it was, and its unit failed in that account's own service manager.
+
+// waitingDatabase is a provider whose only part not healthy waits for the operator's secret: a database's
+// process stated waiting, as the node-engine states a tool check answering waits (ADR 0283 decision 2).
+func waitingDatabase() inventory.ResourceHealth {
+	return inventory.ResourceHealth{Module: "db", Resource: "db.server", Kind: "process", Target: "db.service",
+		State: link.StateWaiting, Check: "tool", Reason: "the database waits for its licence",
+		Waits: []inventory.Wait{{Part: "the server", Secret: "licence", What: "the licence key of the database"}}}
+}
+
+// failingConsumer is a consumer whose check that needs the database fails.
+func failingConsumer(module string) inventory.ResourceHealth {
+	return inventory.ResourceHealth{Module: module, Resource: module + ".web", Kind: "container", Target: module,
+		State: link.StateUnhealthy, Reason: "http /health on web: answered 500", Check: "http", Needs: "postgres-database"}
+}
+
+// holdingOf is one reading of the record without a store: the newest statements, the open conditions, and each
+// consumer's provider already looked up, as providerFor memoises it.
+func holdingOf(open []conditions.Condition, healths map[string]inventory.NodeHealth, bound map[[3]string]catalogue.Chosen) *holding {
+	h := &holding{ctx: context.Background(), healths: healths, open: open, providers: map[string]providerLookup{}}
+	for k, p := range bound {
+		h.providers[k[0]+"\x00"+k[1]+"\x00"+k[2]] = providerLookup{p, true}
+	}
+	return h
+}
+
+var theDatabase = catalogue.Chosen{Node: "anchor", Module: "db"}
+
+// (1) A provider whose only part not healthy waits for the operator holds the findings of its consumers under its
+// needs-operator condition, as ADR 0240 rule 5 holds them under an unhealthy one.
+func TestAProviderWaitingForTheOperatorHoldsItsConsumers(t *testing.T) {
+	healths := map[string]inventory.NodeHealth{
+		"anchor": {Node: "anchor", Resources: []inventory.ResourceHealth{waitingDatabase()}},
+		"laptop": {Node: "laptop", Resources: []inventory.ResourceHealth{failingConsumer("shop")}},
+	}
+	bound := map[[3]string]catalogue.Chosen{{"laptop", "shop", "postgres-database"}: theDatabase}
+	hold := holdingOf(nil, healths, bound)
+	p, held := hold.heldUnder("laptop", "shop", []inventory.ResourceHealth{failingConsumer("shop")})
+	if !held || p != theDatabase {
+		t.Fatalf("a consumer of a provider waiting for the operator is not held under it: %v %v", p, held)
+	}
+	if got := hold.waitersOn(theDatabase); len(got) != 1 || got[0] != "shop on laptop" {
+		t.Fatalf("the waiting provider lists %v as waiting on it; want shop on laptop", got)
+	}
+	// Its needs-operator condition open is enough, as its unhealthy one is.
+	open := []conditions.Condition{{Key: needsOperatorKey("db", "anchor"), Kind: kindNeedsOperator}}
+	healths["anchor"] = inventory.NodeHealth{Node: "anchor"}
+	if _, held := holdingOf(open, healths, bound).heldUnder("laptop", "shop",
+		[]inventory.ResourceHealth{failingConsumer("shop")}); !held {
+		t.Fatal("a consumer is not held under its provider's open needs-operator condition")
+	}
+	// A provider healthy holds nothing: what the consumer finds is its own.
+	healthy := waitingDatabase()
+	healthy.State, healthy.Waits = link.StateHealthy, nil
+	healths["anchor"] = inventory.NodeHealth{Node: "anchor", Resources: []inventory.ResourceHealth{healthy}}
+	if _, held := holdingOf(nil, healths, bound).heldUnder("laptop", "shop",
+		[]inventory.ResourceHealth{failingConsumer("shop")}); held {
+		t.Fatal("a consumer of a healthy provider is held")
+	}
+}
+
+// (1) The consumer raises nothing of its own, and the provider's needs-operator condition says who waits on it.
+func TestAWaitingProvidersConditionListsWhoWaitsOnIt(t *testing.T) {
+	k, _ := withConditionsInMemory(t)
+	ctx := t.Context()
+	healths := map[string]inventory.NodeHealth{
+		"anchor": {Node: "anchor", Resources: []inventory.ResourceHealth{waitingDatabase()}},
+		"laptop": {Node: "laptop", Resources: []inventory.ResourceHealth{failingConsumer("shop")}},
+	}
+	bound := map[[3]string]catalogue.Chosen{{"laptop", "shop", "postgres-database"}: theDatabase}
+	was := readHoldingFor
+	t.Cleanup(func() { readHoldingFor = was })
+	readHoldingFor = func(_ context.Context, _ *inventory.Inventory, open []conditions.Condition) (*holding, error) {
+		return holdingOf(open, healths, bound), nil
+	}
+	for look := 1; look <= 2; look++ {
+		if err := judgeModuleHealth(ctx, nil, k, "anchor", map[string][]inventory.ResourceHealth{"db": {waitingDatabase()}},
+			map[string]int{"db": look}, time.Now()); err != nil {
+			t.Fatal(err)
+		}
+		if err := judgeModuleHealth(ctx, nil, k, "laptop", map[string][]inventory.ResourceHealth{"shop": {failingConsumer("shop")}},
+			map[string]int{"shop": look}, time.Now()); err != nil {
+			t.Fatal(err)
+		}
+	}
+	open, err := k.Open(ctx)
+	if err != nil {
+		t.Fatal(err)
+	}
+	var keys []string
+	var provider *conditions.Condition
+	for i, c := range open {
+		keys = append(keys, c.Key)
+		if c.Key == needsOperatorKey("db", "anchor") {
+			provider = &open[i]
+		}
+	}
+	if len(open) != 1 || provider == nil {
+		t.Fatalf("a provider waiting for the operator and a consumer failing on it raised %v; want the provider's "+
+			"needs-operator alone", keys)
+	}
+	if said := provider.Evidence[0].Said; !strings.Contains(said, "shop on laptop") {
+		t.Fatalf("the provider's needs-operator does not list shop on laptop as waiting on it: %s", said)
+	}
+	// Its words still name the act: it is the operator's, whoever waits on it.
+	if provider.Resolver != conditions.ResolverOperator || !strings.Contains(provider.Needs, "desk prompt") {
+		t.Fatalf("the provider's condition: %+v", provider)
+	}
+}
+
+// relogin and userUnit are person_wait_test.go's; mounts is said here with the same account and unit beside its
+// wait for the password.
+func reloginBesideAWait() []inventory.ResourceHealth {
+	return []inventory.ResourceHealth{relogin("mounts", "operator"), userUnit("mounts", "operator"),
+		waitingResource(passwordWait)}
+}
+
+// (2) A module with a checked wait beside a relogin-needed account says both: the new login, and the act the
+// operator owes it.
+func TestAWaitBesideAReloginIsSaid(t *testing.T) {
+	k, _ := withConditionsInMemory(t)
+	ctx := t.Context()
+	for look := 1; look <= 2; look++ {
+		if err := judgeModuleHealth(ctx, nil, k, "workstation", map[string][]inventory.ResourceHealth{"mounts": reloginBesideAWait()},
+			map[string]int{"mounts": look}, time.Now()); err != nil {
+			t.Fatal(err)
+		}
+	}
+	got, open := needsOperatorOpen(t, k)
+	if got == nil {
+		t.Fatalf("a wait for the operator beside a relogin is not said: %v", openKeysOf(open))
+	}
+	if !strings.Contains(got.Summary, "smb-password-games") || got.Severity != conditions.Warning {
+		t.Fatalf("the needs-operator beside the relogin: %+v", got)
+	}
+	relogged := false
+	for _, c := range open {
+		relogged = relogged || c.Key == reloginKey("mounts", "workstation")
+	}
+	if !relogged {
+		t.Fatalf("the relogin is no longer said beside the wait: %v", openKeysOf(open))
+	}
+	// The login done, the wait stays said and the relogin clears.
+	if err := judgeModuleHealth(ctx, nil, k, "workstation", map[string][]inventory.ResourceHealth{"mounts": {waitingResource(passwordWait)}},
+		map[string]int{"mounts": 3}, time.Now()); err != nil {
+		t.Fatal(err)
+	}
+	got, open = needsOperatorOpen(t, k)
+	if got == nil || len(open) != 1 {
+		t.Fatalf("after the new login: %v", openKeysOf(open))
+	}
+}
+
+// (2) The same beside a directory used as found.
+func TestAWaitBesideADirectoryUsedAsFoundIsSaid(t *testing.T) {
+	k, _ := withConditionsInMemory(t)
+	ctx := t.Context()
+	found := foundDirectory("mounts", time.Now().Add(-time.Hour))
+	for look := 1; look <= 2; look++ {
+		if err := judgeModuleHealth(ctx, nil, k, "workstation", map[string][]inventory.ResourceHealth{
+			"mounts": {found, waitingResource(passwordWait)}}, map[string]int{"mounts": look}, time.Now()); err != nil {
+			t.Fatal(err)
+		}
+	}
+	got, open := needsOperatorOpen(t, k)
+	if got == nil {
+		t.Fatalf("a wait for the operator beside a directory used as found is not said: %v", openKeysOf(open))
+	}
+	asFound := false
+	for _, c := range open {
+		asFound = asFound || c.Key == usedAsFoundKey("mounts", "workstation")
+	}
+	if !asFound {
+		t.Fatalf("the directory used as found is no longer said beside the wait: %v", openKeysOf(open))
+	}
+}
+
+// (2) Beside a fault of its own, the wait is said too, and the fault's words do not count the waiting part among
+// what fails.
+func TestAWaitBesideAFaultIsSaidAndTheFaultIsTheFaultAlone(t *testing.T) {
+	k, _ := withConditionsInMemory(t)
+	ctx := t.Context()
+	down := inventory.ResourceHealth{Module: "mounts", Resource: "mounts.apply", Kind: "process",
+		Target: "mesh-mounts-apply.service", State: link.StateUnhealthy, Reason: "down"}
+	for look := 1; look <= 2; look++ {
+		if err := judgeModuleHealth(ctx, nil, k, "workstation", map[string][]inventory.ResourceHealth{
+			"mounts": {down, waitingResource(passwordWait)}}, map[string]int{"mounts": look}, time.Now()); err != nil {
+			t.Fatal(err)
+		}
+	}
+	got, open := needsOperatorOpen(t, k)
+	if got == nil {
+		t.Fatalf("a wait for the operator beside a fault is not said: %v", openKeysOf(open))
+	}
+	var fault *conditions.Condition
+	for i, c := range open {
+		if c.Key == moduleUnhealthyKey("mounts", "workstation") {
+			fault = &open[i]
+		}
+	}
+	if fault == nil {
+		t.Fatalf("the fault is not said: %v", openKeysOf(open))
+	}
+	if strings.Contains(fault.Summary, "mounts.watch") {
+		t.Fatalf("the fault's summary counts the waiting part as failing: %q", fault.Summary)
+	}
+}
+
+func openKeysOf(cs []conditions.Condition) []string {
+	var out []string
+	for _, c := range cs {
+		out = append(out, c.Key)
+	}
+	return out
+}