Say a part waiting for the operator as needs-operator and pass its gate; add secret families (issue 386, hq ADR 0283)

A module waiting for the operator's secret failed its first-node gate, held
every later walk and was said as 'nothing for you to do'. The node-engine's
new waiting state is checked against the manifest and the secrets given,
read by the gate as a wait for a person, and raised as needs-operator naming
the act. A secret family gives each part its own one-line secret, which the
mesh never makes, so the desk prompt can take each password.

cmd/mesh-controller/module_health.go

@@ -74,9 +74,21 @@ func stateHealth(ctx context.Context, inv *inventory.Inventory, k *conditions.Ke
 		kept := inventory.ResourceHealth{Module: r.Module, Resource: r.Resource, Kind: r.Kind, Target: r.Target,
 			State: r.State, Reason: r.Reason, Since: r.Since, Streak: r.Streak, Restarts: r.Restarts,
 			Check: r.Check, Needs: r.Needs, Account: r.Account, Root: r.Root}
+		for _, w := range r.Waits {
+			kept.Waits = append(kept.Waits, inventory.Wait{Part: w.Part, Secret: w.Secret, Setting: w.Setting, What: w.What})
+		}
 		resources = append(resources, kept)
-		if r.State == link.StateUnhealthy && r.Module != "" {
-			unhealthy[r.Module] = append(unhealthy[r.Module], kept)
+	}
+	// **A wait for the operator is checked before it is excused** (novox/hq ADR 0283): a waiting resource whose
+	// wait does not check out is judged unhealthy, saying why; one that does is kept beside the unhealthy ones, so
+	// judgeModuleHealth can say it as needs-operator. What is stored is what the machine said.
+	var wf operatorWaitFacts
+	if mods := waitingModules(resources); len(mods) > 0 {
+		readWaitFacts(ctx, inv, node, mods, nil, &wf)
+	}
+	for _, r := range checkWaiting(node, resources, wf) {
+		if (r.State == link.StateUnhealthy || r.State == link.StateWaiting) && r.Module != "" {
+			unhealthy[r.Module] = append(unhealthy[r.Module], r)
 		}
 	}
 	streaks := map[string]int{}
@@ -135,7 +147,8 @@ func judgeModuleHealth(ctx context.Context, inv *inventory.Inventory, k *conditi
 	}
 	standing := map[string]conditions.Condition{}
 	for _, c := range open {
-		if (c.Kind == kindModuleUnhealthy || c.Kind == kindReloginNeeded || c.Kind == kindUsedAsFound) &&
+		if (c.Kind == kindModuleUnhealthy || c.Kind == kindReloginNeeded || c.Kind == kindUsedAsFound ||
+			c.Kind == kindNeedsOperator) &&
 			c.Subject.Machine == node {
 			standing[c.Key] = c
 		}
@@ -159,6 +172,20 @@ func judgeModuleHealth(ctx context.Context, inv *inventory.Inventory, k *conditi
 	heldOn := map[string]string{}
 	providers := map[catalogue.Chosen]bool{}
 	for _, m := range modules {
+		// **A part that waits for the operator is said as that** (novox/hq ADR 0283): its waits already checked,
+		// the operator's, never urgent, its words naming the act.
+		if waits, waiting := operatorWait(m, unhealthy[m]); waiting {
+			o := needsOperatorObservation(m, node, waits, unhealthy[m])
+			seen[o.Key()] = true
+			became[m] = kindNeedsOperator
+			if _, isOpen := standing[o.Key()]; streaks[m] < moduleUnhealthyAfter && !isOpen {
+				continue
+			}
+			if _, err := k.Observe(ctx, o); err != nil {
+				problems = append(problems, err.Error())
+			}
+			continue
+		}
 		// **A directory used as found is said as that** (novox/hq issue 339): the operator's to hand over at the
 		// machine, never urgent — nothing is broken by the wait that a person was not told of — and its own kind,
 		// so the gate never reads it as a fault of the build that happened to be sent beside it.
@@ -228,6 +255,9 @@ func judgeModuleHealth(ctx context.Context, inv *inventory.Inventory, k *conditi
 		if c.Kind == kindUsedAsFound {
 			why = fmt.Sprintf("%s says no directory of %s is used as found any more", node, module)
 		}
+		if c.Kind == kindNeedsOperator {
+			why = fmt.Sprintf("%s says %s no longer waits for the operator", node, module)
+		}
 		if on, held := heldOn[key]; held {
 			why = fmt.Sprintf("what %s finds on %s waits on %s, which is unhealthy: held under its condition", module, node, on)
 		}
@@ -238,6 +268,12 @@ func judgeModuleHealth(ctx context.Context, inv *inventory.Inventory, k *conditi
 		case c.Kind == kindModuleUnhealthy && became[module] == kindReloginNeeded:
 			why = fmt.Sprintf("%s on %s now waits only for a new login", module, node)
 			resolved = fmt.Sprintf("%s on %s now waits only for a new login", module, node)
+		case c.Kind == kindModuleUnhealthy && became[module] == kindNeedsOperator:
+			why = fmt.Sprintf("%s on %s now only waits for the operator", module, node)
+			resolved = fmt.Sprintf("%s on %s now only waits for you", module, node)
+		case c.Kind == kindNeedsOperator && became[module] == kindModuleUnhealthy:
+			why = fmt.Sprintf("%s on %s no longer only waits for the operator, and is not healthy", module, node)
+			resolved = fmt.Sprintf("What %s on %s waited for is given, and it still does not work", module, node)
 		case c.Kind == kindReloginNeeded && became[module] == kindModuleUnhealthy:
 			why = fmt.Sprintf("%s on %s no longer waits for a new login, and is not healthy", module, node)
 			resolved = fmt.Sprintf("The new login on %s is done, and %s still does not work", node, module)
@@ -420,6 +456,11 @@ func reasonWords(r inventory.ResourceHealth) string {
 	case "":
 		return "is unhealthy"
 	}
+	// A wait for the operator that did not check out is said as the controller found it (ADR 0283): names of
+	// secrets and settings only, never what the check itself said.
+	if strings.HasPrefix(r.Reason, waitRefusedPrefix) {
+		return r.Reason
+	}
 	// What a declared check found says an endpoint, a path or an address: evidence, never the summary the
 	// operator's channel carries (ADR 0234 §6). The summary names the check.
 	if r.Check != "" {
@@ -511,7 +552,9 @@ func moduleHealthWord(module, machine string, since time.Time, f gateFacts) (hea
 	if h.HeardAt.Before(since) {
 		return healthNotYet, fmt.Sprintf("%s has not said how what %s runs is since it was sent", machine, module)
 	}
-	wait, waits := personWait(module, machine, h.Resources)
+	// **A wait for the operator is checked first** (novox/hq ADR 0283): one that does not check out is unhealthy.
+	resources := checkWaiting(machine, h.Resources, f.waits)
+	wait, waits := personWait(module, machine, resources)
 	// **Only a build whose own send put the account in a new group is excused** (issue 318 review): read from
 	// what the controller sent, never from when the machine says the wait began — that time is the engine's
 	// memory, reset by its restart and moved by a change of words. A build that adds no account group cannot
@@ -520,10 +563,17 @@ func moduleHealthWord(module, machine string, since time.Time, f gateFacts) (hea
 		waits = false
 	}
 	var found []string
-	for _, r := range h.Resources {
+	var forOperator []inventory.Wait
+	for _, r := range resources {
 		if r.Module != module {
 			continue
 		}
+		// **Any build is excused while its wait for the operator checks out** (ADR 0283 decision 4): a secret not
+		// given is owed by every build alike, so it is no fault of this one, and the verdict carries it.
+		if r.State == link.StateWaiting {
+			forOperator = append(forOperator, r.Waits...)
+			continue
+		}
 		if waits && r.State == link.StateUnhealthy {
 			continue
 		}
@@ -550,11 +600,14 @@ func moduleHealthWord(module, machine string, since time.Time, f gateFacts) (hea
 				reasonAfter(r.Reason))
 		}
 	}
-	if waits || len(found) > 0 {
+	if waits || len(found) > 0 || len(forOperator) > 0 {
 		var said []string
 		if waits {
 			said = append(said, wait)
 		}
+		if len(forOperator) > 0 {
+			said = append(said, operatorWaitSaid(module, machine, forOperator))
+		}
 		if len(found) > 0 {
 			said = append(said, fmt.Sprintf("on %s, %s uses %s as found and waits for the operator to hand it over "+
 				"(`nox node hand-over %s <directory>` on the control-node)", machine, module, strings.Join(found, ", "), machine))