Pass a wait for a person's new login with the wait carried, and keep the pass of a module healthy beside a failure (hq ADR 0254, issue 318)

cmd/mesh-controller/gate.go

@@ -77,6 +77,10 @@ type health int
 
 const (
 	healthGood health = iota
+	// healthPerson is a wait for a person (novox/hq ADR 0254): everything unhealthy of the module waits for
+	// one person's new login (personWait). The build did what it should; no bound a machine can meet ends
+	// the wait. It counts as a pass, and the gate carries the reading along in its verdict.
+	healthPerson
 	// healthWaiting is not a pass and not a fault: what the module's checks find waits on an unhealthy
 	// provider (ADR 0240 rule 5), so the judging waits — past the bound too — rather than putting back a
 	// build for something it did not do.
@@ -209,7 +213,9 @@ func judgeHealth(module, component string, m catalogue.Manifest, machine string,
 				firstLine(f.openErr.Error())
 		}
 		for _, c := range f.open {
-			if c.Source == gateProbe || c.Raised.Before(since) {
+			// A wait for a person's new login is the module's reading, not a fault raised since the send: the
+			// gate reads it from the statement below (ADR 0254).
+			if c.Source == gateProbe || c.Raised.Before(since) || c.Kind == kindReloginNeeded {
 				continue
 			}
 			onIt := c.Subject.Machine == machine || slices.Contains(c.Subject.Also, machine) ||
@@ -435,13 +441,19 @@ func judgeMoves(ctx context.Context, open *stores, g *inventory.PlanGate, pairs
 	for node, moved := range byMachine {
 		words[node] = aboutTheMachine(node, moved, *g.Since, facts)
 	}
+	// **Each module is judged on its own** (novox/hq ADR 0254, issue 318): its reading is the worst of its
+	// machines', its passes are counted apart, and the send's verdict is still one — but a module that was
+	// healthy on its own for the passes the gate asks keeps a pass when another beside it fails.
 	worst, why := healthGood, ""
 	var failing []string
 	broken := map[string]bool{}
+	reading := map[string]health{}
+	waits := map[string]string{}
+	var modules []string
 	for _, j := range pairs {
 		w := words[j.node]
 		h, said := judgeHealth(j.module, coreComponent(j.module), shelf[j.module], j.node, *g.Since, w.facts)
-		if h == healthGood {
+		if h == healthGood || h == healthPerson {
 			if on, named := w.on[j.module]; named {
 				h, said = healthNotYet, on
 			} else if w.whole != "" {
@@ -450,7 +462,16 @@ func judgeMoves(ctx context.Context, open *stores, g *inventory.PlanGate, pairs
 				h, said = healthWaiting, w.waiting
 			}
 		}
-		if h != healthGood && !slices.Contains(failing, j.module) {
+		if _, seen := reading[j.module]; !seen {
+			modules = append(modules, j.module)
+		}
+		if h > reading[j.module] {
+			reading[j.module] = h
+		}
+		if h == healthPerson {
+			waits[j.module] = joinSaid(waits[j.module], said)
+		}
+		if h > healthPerson && !slices.Contains(failing, j.module) {
 			failing = append(failing, j.module)
 		}
 		if h == healthBroken {
@@ -458,14 +479,45 @@ func judgeMoves(ctx context.Context, open *stores, g *inventory.PlanGate, pairs
 		}
 		if h > worst {
 			worst, why = h, said
-		} else if h == worst && h != healthGood && why == "" {
+		} else if h == worst && h > healthPerson && why == "" {
 			why = said
 		}
 	}
+	if g.Healthy == nil {
+		g.Healthy = map[string]int{}
+	}
+	g.Waits = nil
+	for _, m := range modules {
+		switch {
+		case reading[m] > healthPerson:
+			g.Healthy[m] = 0
+		default:
+			g.Healthy[m]++
+			if w := waits[m]; w != "" {
+				if g.Waits == nil {
+					g.Waits = map[string]string{}
+				}
+				g.Waits[m] = w
+			}
+		}
+	}
+	// passedAlone is every module that passed on its own while the send as a whole did not.
+	passedAlone := func() []string {
+		var out []string
+		if now.Sub(*g.Since) < gateSettle {
+			return nil
+		}
+		for _, m := range modules {
+			if reading[m] <= healthPerson && g.Healthy[m] >= gatePasses {
+				out = append(out, m)
+			}
+		}
+		return out
+	}
 	switch {
 	case worst == healthBroken:
 		// What broke is put back; what was only not yet healthy beside it is too — they moved together.
-		g.Failing = failing
+		g.Failing, g.Passing = failing, passedAlone()
 		decide(g, inventory.GateFailed, why, now)
 	case worst == healthWaiting:
 		// Waiting on a provider that is unhealthy: not a pass, and not a failure at the bound either —
@@ -474,19 +526,48 @@ func judgeMoves(ctx context.Context, open *stores, g *inventory.PlanGate, pairs
 	case worst == healthNotYet:
 		g.Passes, g.LastPass, g.Last, g.Failing = 0, nil, why, failing
 		if now.Sub(*g.Since) > gateBound {
+			g.Passing = passedAlone()
 			decide(g, inventory.GateFailed, fmt.Sprintf("not healthy within %s of its apply: %s", gateBound, why), now)
 		}
 	default:
+		// Healthy, or waiting for a person (ADR 0254): a pass, the wait carried along in the verdict.
 		g.Passes++
 		g.LastPass, g.Last, g.Failing = &now, "", nil
 		if g.Passes >= gatePasses && now.Sub(*g.Since) >= gateSettle {
 			decide(g, inventory.GatePassed, fmt.Sprintf("healthy %d times over %s", g.Passes,
-				now.Sub(*g.Since).Round(time.Second)), now)
+				now.Sub(*g.Since).Round(time.Second))+waitsSaid(g.Waits), now)
 		}
 	}
 	return g.Verdict, nil
 }
 
+// waitsSaid is the waits for a person a passing gate carries, as its verdict says them.
+func waitsSaid(waits map[string]string) string {
+	if len(waits) == 0 {
+		return ""
+	}
+	modules := make([]string, 0, len(waits))
+	for m := range waits {
+		modules = append(modules, m)
+	}
+	sort.Strings(modules)
+	var said []string
+	for _, m := range modules {
+		said = append(said, waits[m])
+	}
+	return "; and it waits for a person: " + strings.Join(said, "; ")
+}
+
+func joinSaid(a, b string) string {
+	switch {
+	case a == "":
+		return b
+	case b == "" || strings.Contains(a, b):
+		return a
+	}
+	return a + "; " + b
+}
+
 // decide sets a gate's verdict.
 func decide(g *inventory.PlanGate, verdict, why string, now time.Time) {
 	g.Verdict, g.Why, g.JudgedAt = verdict, why, &now
@@ -959,6 +1040,12 @@ func gateLine(g *inventory.PlanGate) string {
 	} else if g.Verdict == "" {
 		line += fmt.Sprintf(" (%d of %d passes)", g.Passes, gatePasses)
 	}
+	if g.Verdict == "" && len(g.Waits) > 0 {
+		line += waitsSaid(g.Waits)
+	}
+	if len(g.Passing) > 0 {
+		line += "; passed on their own and kept: " + strings.Join(g.Passing, ", ")
+	}
 	if g.Rollback != "" {
 		line += "; " + g.Rollback
 	}