Give every module of a failed send a verdict, and excuse only the wait a build's own send brought (hq issue 318 review)

cmd/mesh-controller/gate.go

@@ -442,11 +442,12 @@ func judgeMoves(ctx context.Context, open *stores, g *inventory.PlanGate, pairs
 		words[node] = aboutTheMachine(node, moved, *g.Since, facts)
 	}
 	// **Each module is judged on its own** (novox/hq ADR 0254, issue 318): its reading is the worst of its
-	// machines', its passes are counted apart, and the send's verdict is still one — but a module that was
+	// machines', its passes are counted apart, and the send's verdict is still one. **Every module of a send
-	// healthy on its own for the passes the gate asks keeps a pass when another beside it fails.
+	// leaves it with a verdict** (issue 318 review): one healthy for the passes the gate asks, after the
+	// settle time, keeps a pass when another beside it fails; one found broken holds the send's verdict until
+	// the modules beside it have theirs, or the bound; and at the verdict, every module that did not pass is
+	// put back with what failed, so none is left on the machine unjudged for other walks to wait on.
 	worst, why := healthGood, ""
-	var failing []string
-	broken := map[string]bool{}
 	reading := map[string]health{}
 	waits := map[string]string{}
 	var modules []string
@@ -465,18 +466,21 @@ func judgeMoves(ctx context.Context, open *stores, g *inventory.PlanGate, pairs
 		if _, seen := reading[j.module]; !seen {
 			modules = append(modules, j.module)
 		}
+		if h == healthBroken && !slices.Contains(g.Broken, j.module) {
+			g.Broken = append(g.Broken, j.module)
+			if g.BrokenWhy == "" {
+				g.BrokenWhy = said
+			}
+		}
+		if slices.Contains(g.Broken, j.module) {
+			h = healthBroken // found broken once, broken for the rest of the judging
+		}
 		if h > reading[j.module] {
 			reading[j.module] = h
 		}
 		if h == healthPerson {
 			waits[j.module] = joinSaid(waits[j.module], said)
 		}
-		if h > healthPerson && !slices.Contains(failing, j.module) {
-			failing = append(failing, j.module)
-		}
-		if h == healthBroken {
-			broken[j.module] = true
-		}
 		if h > worst {
 			worst, why = h, said
 		} else if h == worst && h > healthPerson && why == "" {
@@ -486,54 +490,77 @@ func judgeMoves(ctx context.Context, open *stores, g *inventory.PlanGate, pairs
 	if g.Healthy == nil {
 		g.Healthy = map[string]int{}
 	}
+	if g.HealthyAt == nil {
+		g.HealthyAt = map[string]time.Time{}
+	}
 	g.Waits = nil
+	var failing []string
 	for _, m := range modules {
-		switch {
+		if reading[m] > healthPerson {
-		case reading[m] > healthPerson:
+			failing = append(failing, m)
 			g.Healthy[m] = 0
-		default:
+			delete(g.HealthyAt, m)
+			continue
+		}
+		// **A pass is counted only gateEvery after the one before** (issue 318 review): a send judged more
+		// often while another module beside it is not yet healthy counts no faster.
+		if at, counted := g.HealthyAt[m]; !counted || now.Sub(at) >= gateEvery {
 			g.Healthy[m]++
-			if w := waits[m]; w != "" {
+			g.HealthyAt[m] = now
-				if g.Waits == nil {
+		}
-					g.Waits = map[string]string{}
+		if w := waits[m]; w != "" {
-				}
+			if g.Waits == nil {
-				g.Waits[m] = w
+				g.Waits = map[string]string{}
 			}
+			g.Waits[m] = w
 		}
 	}
-	// passedAlone is every module that passed on its own while the send as a whole did not.
+	settled := now.Sub(*g.Since) >= gateSettle
-	passedAlone := func() []string {
+	passedAlone := func(m string) bool {
-		var out []string
+		return settled && reading[m] <= healthPerson && g.Healthy[m] >= gatePasses
-		if now.Sub(*g.Since) < gateSettle {
+	}
-			return nil
+	// fail decides the send failed: what passed on its own keeps its pass, everything else is put back.
-		}
+	fail := func(why string) {
+		g.Passing, g.Failing = nil, nil
 		for _, m := range modules {
-			if reading[m] <= healthPerson && g.Healthy[m] >= gatePasses {
+			if passedAlone(m) {
-				out = append(out, m)
+				g.Passing = append(g.Passing, m)
+			} else {
+				g.Failing = append(g.Failing, m)
 			}
 		}
-		return out
+		decide(g, inventory.GateFailed, why, now)
 	}
+	pastBound := now.Sub(*g.Since) > gateBound
 	switch {
 	case worst == healthBroken:
-		// What broke is put back; what was only not yet healthy beside it is too — they moved together.
+		var judging []string
-		g.Failing, g.Passing = failing, passedAlone()
+		for _, m := range modules {
-		decide(g, inventory.GateFailed, why, now)
+			if reading[m] != healthBroken && !passedAlone(m) {
+				judging = append(judging, m)
+			}
+		}
+		if len(judging) == 0 || pastBound {
+			fail(g.BrokenWhy)
+			break
+		}
+		// What broke fails the send; what is beside it is judged to its own verdict first, within the bound.
+		g.Passes, g.LastPass, g.Failing = 0, nil, failing
+		g.Last = fmt.Sprintf("%s; %s judged to its own verdict before the send's", g.BrokenWhy, strings.Join(judging, ", "))
 	case worst == healthWaiting:
 		// Waiting on a provider that is unhealthy: not a pass, and not a failure at the bound either —
 		// the provider's own condition says what is wrong (ADR 0240 rule 5).
 		g.Passes, g.LastPass, g.Last, g.Failing = 0, nil, why, failing
 	case worst == healthNotYet:
 		g.Passes, g.LastPass, g.Last, g.Failing = 0, nil, why, failing
-		if now.Sub(*g.Since) > gateBound {
+		if pastBound {
-			g.Passing = passedAlone()
+			fail(fmt.Sprintf("not healthy within %s of its apply: %s", gateBound, why))
-			decide(g, inventory.GateFailed, fmt.Sprintf("not healthy within %s of its apply: %s", gateBound, why), now)
 		}
 	default:
 		// Healthy, or waiting for a person (ADR 0254): a pass, the wait carried along in the verdict.
 		g.Passes++
 		g.LastPass, g.Last, g.Failing = &now, "", nil
-		if g.Passes >= gatePasses && now.Sub(*g.Since) >= gateSettle {
+		if g.Passes >= gatePasses && settled {
 			decide(g, inventory.GatePassed, fmt.Sprintf("healthy %d times over %s", g.Passes,
 				now.Sub(*g.Since).Round(time.Second))+waitsSaid(g.Waits), now)
 		}
@@ -541,6 +568,18 @@ func judgeMoves(ctx context.Context, open *stores, g *inventory.PlanGate, pairs
 	return g.Verdict, nil
 }
 
+// whyFor is a passing gate's why as one module's verdict says it: the send's, and that module's own wait
+// for a person, never another's (issue 318 review).
+func whyFor(g *inventory.PlanGate, module string) string {
+	why, _, _ := strings.Cut(g.Why, waitsPrefix)
+	if w := g.Waits[module]; w != "" {
+		why += waitsPrefix + w
+	}
+	return why
+}
+
+const waitsPrefix = "; and it waits for a person: "
+
 // waitsSaid is the waits for a person a passing gate carries, as its verdict says them.
 func waitsSaid(waits map[string]string) string {
 	if len(waits) == 0 {
@@ -555,7 +594,7 @@ func waitsSaid(waits map[string]string) string {
 	for _, m := range modules {
 		said = append(said, waits[m])
 	}
-	return "; and it waits for a person: " + strings.Join(said, "; ")
+	return waitsPrefix + strings.Join(said, "; ")
 }
 
 func joinSaid(a, b string) string {
@@ -579,7 +618,7 @@ func gatePassed(ctx context.Context, open *stores, p *inventory.Plan, module str
 	g := state.Gate
 	err := open.inventory.RecordGate(ctx, inventory.GateVerdict{Build: state.Build, Module: module,
 		Commit: state.Commit, Previous: state.Previous, Plan: p.ID, Machines: g.Machines,
-		Verdict: inventory.GatePassed, Why: g.Why, Component: g.Component, JudgingFrom: g.Since})
+		Verdict: inventory.GatePassed, Why: whyFor(g, module), Component: g.Component, JudgingFrom: g.Since})
 	if err != nil && state.Build != "" {
 		fmt.Printf("%s: %s passed its gate, and the verdict could not be kept: %v\n", p.ID, module, err)
 	}

cmd/mesh-controller/gate_followup_test.go

@@ -0,0 +1,279 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"reflect"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/novox/mesh-controller/internal/catalogue"
+	"github.com/novox/mesh-controller/internal/conditions"
+	"github.com/novox/mesh-controller/internal/inventory"
+	"github.com/novox/mesh-controller/internal/lease"
+	"github.com/novox/mesh-controller/internal/link"
+)
+
+// What the review of issue 318's fix found (novox/hq ADR 0254): every module of a send leaves it with a
+// verdict, a pass is counted no faster than the gate's spacing, a carried build's verdict carries only its own
+// wait, and a provider waiting for a login says who waits on it.
+
+// withGateBounds sets the gate's bounds for one test.
+func withGateBounds(t *testing.T, settle, every, bound time.Duration) {
+	t.Helper()
+	wasSettle, wasEvery, wasBound := gateSettle, gateEvery, gateBound
+	gateSettle, gateEvery, gateBound = settle, every, bound
+	t.Cleanup(func() { gateSettle, gateEvery, gateBound = wasSettle, wasEvery, wasBound })
+}
+
+// factsOn is a judging's facts for one machine that applied its send: the node tools answer, and what it says
+// of its modules' resources.
+func factsOn(t *testing.T, machine string, since time.Time, rs ...inventory.ResourceHealth) func() gateFacts {
+	return func() gateFacts {
+		now := time.Now()
+		at := now
+		return gateFacts{now: now,
+			reports:    map[string]inventory.Reported{machine: {Node: machine, Outcome: inventory.OutcomeApplied, At: &at, Current: true}},
+			engines:    map[string]string{},
+			rolledBack: map[string][]lease.Rollback{},
+			served:     map[string]served{machine: {runtime: true, tools: map[string]bool{}}},
+			health:     map[string]inventory.NodeHealth{machine: {Node: machine, HeardAt: now, Resources: rs}},
+		}
+	}
+}
+
+func healthyContainer(module string) inventory.ResourceHealth {
+	return inventory.ResourceHealth{Module: module, Resource: module + ".web", Kind: "container", Target: module,
+		State: link.StateHealthy}
+}
+
+// **A fault decided before the settle time does not strand the module beside it** (review (a)): the node
+// tools' witness put its build back at once, and the send waits for the healthy module's own verdict — a pass,
+// counted over the gate's spacing — before it says its own. The broken one alone is put back.
+func TestAFaultBeforeTheSettleTimeWaitsForTheModuleBesideIt(t *testing.T) {
+	open := aMesh(t)
+	ctx := t.Context()
+	withGateBounds(t, 150*time.Millisecond, 20*time.Millisecond, 10*time.Second)
+	since := time.Now().Add(-time.Second)
+	base := factsOn(t, "laptop", since, healthyContainer("app"))
+	wasGather := gatherGateFacts
+	t.Cleanup(func() { gatherGateFacts = wasGather })
+	gatherGateFacts = func(context.Context, *stores, string) (gateFacts, error) {
+		f := base()
+		f.rolledBack["laptop"] = []lease.Rollback{{Component: lease.ComponentNodeTools, Outcome: lease.OutcomeRolledBack,
+			At: since.Add(100 * time.Millisecond), Why: "the node tools did not answer"}}
+		return f, nil
+	}
+	g := &inventory.PlanGate{Machines: []string{"laptop"}, Since: &since}
+	pairs := []judged{{module: broker_runtime, node: "laptop"}, {module: "app", node: "laptop"}}
+	judgings := 0
+	for deadline := time.Now().Add(5 * time.Second); g.Verdict == "" && time.Now().Before(deadline); {
+		if _, err := judgeMoves(ctx, open, g, pairs, time.Now()); err != nil {
+			t.Fatal(err)
+		}
+		judgings++
+		time.Sleep(30 * time.Millisecond)
+	}
+	if g.Verdict != inventory.GateFailed || judgings < gatePasses {
+		t.Fatalf("verdict %q after %d judging(s): %+v; want failed, after the module beside it was judged", g.Verdict,
+			judgings, g)
+	}
+	if !reflect.DeepEqual(g.Passing, []string{"app"}) || !reflect.DeepEqual(g.Failing, []string{broker_runtime}) {
+		t.Fatalf("passing %v, failing %v; want app kept and the node tools put back", g.Passing, g.Failing)
+	}
+	if !strings.Contains(g.Why, "witness") {
+		t.Errorf("the verdict does not say what broke: %q", g.Why)
+	}
+}
+
+// broker_runtime is the node tools' module name, a core component a witness judges.
+const broker_runtime = "node-tools"
+
+// **A pass is counted only the gate's spacing after the one before** (review (c)): a send judged every tick
+// while a module beside it is not yet healthy counts the healthy one once.
+func TestAPassIsCountedNoFasterThanTheGatesSpacing(t *testing.T) {
+	open := aMesh(t)
+	ctx := t.Context()
+	withGateBounds(t, 0, time.Hour, time.Hour)
+	since := time.Now().Add(-time.Minute)
+	base := factsOn(t, "laptop", since, healthyContainer("app"), inventory.ResourceHealth{Module: "late",
+		Resource: "late.web", Kind: "container", Target: "late", State: link.StateUnhealthy, Reason: "down"})
+	wasGather := gatherGateFacts
+	t.Cleanup(func() { gatherGateFacts = wasGather })
+	gatherGateFacts = func(context.Context, *stores, string) (gateFacts, error) { return base(), nil }
+	g := &inventory.PlanGate{Machines: []string{"laptop"}, Since: &since}
+	pairs := []judged{{module: "app", node: "laptop"}, {module: "late", node: "laptop"}}
+	now := time.Now()
+	for i := 0; i < 3; i++ {
+		if _, err := judgeMoves(ctx, open, g, pairs, now.Add(time.Duration(i)*time.Second)); err != nil {
+			t.Fatal(err)
+		}
+	}
+	if g.Healthy["app"] != 1 || g.Healthy["late"] != 0 {
+		t.Fatalf("counted %v in three judgings within a second; want app once and late never", g.Healthy)
+	}
+}
+
+// **A carried build's pass carries its own wait, never another's** (review (e)).
+func TestACarriedPassCarriesOnlyItsOwnWait(t *testing.T) {
+	b := aBacklog(t)
+	ctx := t.Context()
+	inv := b.open.inventory
+	since := time.Now().Add(-time.Minute)
+	g := &inventory.PlanGate{Machines: []string{"laptop"}, Since: &since, Verdict: inventory.GatePassed,
+		Why:   "healthy 3 times over 2m0s" + waitsSaid(map[string]string{"late": "relogin needed on laptop: late waits"}),
+		Waits: map[string]string{"late": "relogin needed on laptop: late waits"},
+		Carried: []inventory.CarriedMove{{Module: "app", Node: "laptop", From: "c1", To: "c2", Build: "build-app-c2"},
+			{Module: "late", Node: "laptop", From: "c1", To: "c2", Build: "build-late-c2"}}}
+	passCarried(ctx, b.open, &inventory.Plan{ID: "release-test"}, g, "")
+	app, _, _ := inv.GateOf(ctx, "build-app-c2")
+	late, _, _ := inv.GateOf(ctx, "build-late-c2")
+	if strings.Contains(app.Why, "waits for a person") || app.Verdict != inventory.GatePassed {
+		t.Errorf("app's pass: %+v; want it without late's wait", app)
+	}
+	if !strings.Contains(late.Why, "relogin needed on laptop") {
+		t.Errorf("late's pass: %+v; want its own wait", late)
+	}
+}
+
+// **The tier path keeps the pass of the module the gate is kept on** (review (b)): a plan's first send
+// carried its own module, healthy, and a build waiting on that machine that never became healthy. At the
+// bound the waiting one is put back and marked; the plan's own module keeps its pass, so no walk waits on it.
+func TestThePlansOwnModuleKeepsItsPassWhenItsSendFails(t *testing.T) {
+	g := aGateMesh(t)
+	ctx := t.Context()
+	inv := g.open.inventory
+	// `late` waits on anchor for a gate: c1 sent, c2 registered and built.
+	for _, b := range []inventory.Build{
+		{ID: "build-late-1", Module: "late", Commit: "l1", Repository: "novox/mesh-catalog", Path: "modules/late",
+			Asked: time.Now().Add(-2 * time.Hour), At: time.Now().Add(-2 * time.Hour)},
+		{ID: "build-late-2", Module: "late", Commit: "l2", Repository: "novox/mesh-catalog", Path: "modules/late",
+			Asked: time.Now().Add(-time.Minute), At: time.Now().Add(-time.Minute)},
+	} {
+		manifest, _ := json.Marshal(catalogue.Manifest{Module: "late", Version: b.Commit})
+		b.Manifest = manifest
+		b.Made = []inventory.Artifact{{Name: "x", Kind: "bundle", Reference: "sha256:" + b.Commit}}
+		if err := inv.RecordBuild(ctx, b); err != nil {
+			t.Fatal(err)
+		}
+		if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "late", Version: b.Commit}, inventory.Source{
+			Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/late", BuiltFrom: b.Commit, Head: b.Commit,
+			Asked: b.Asked}); err != nil {
+			t.Fatal(err)
+		}
+		if b.Commit == "l1" {
+			if _, err := inv.Assign(ctx, "anchor", "late"); err != nil {
+				t.Fatal(err)
+			}
+			if err := inv.RecordSent(ctx, nodeID(t, g.open, "anchor"), "d-anchor-late", map[string]string{"app": "c1",
+				"late": "l1"}); err != nil {
+				t.Fatal(err)
+			}
+		}
+	}
+	wasMoves := machineMoves
+	t.Cleanup(func() { machineMoves = wasMoves })
+	machineMoves = func(ctx context.Context, open *stores, f moveFacts, node string, all bool) ([]inventory.CarriedMove, error) {
+		modules, err := open.inventory.Assigned(ctx, node)
+		if err != nil {
+			return nil, err
+		}
+		sent, known, err := open.inventory.SentBuilds(ctx, node)
+		if err != nil {
+			return nil, err
+		}
+		return f.moves(node, modules, sent, known, all), nil
+	}
+	gather := gatherGateFacts
+	gatherGateFacts = func(ctx context.Context, open *stores, component string) (gateFacts, error) {
+		f, err := gather(ctx, open, component)
+		f.health = map[string]inventory.NodeHealth{"anchor": {Node: "anchor", HeardAt: time.Now(), Resources: []inventory.ResourceHealth{
+			healthyContainer("app"), {Module: "late", Resource: "late.web", Kind: "container", Target: "late",
+				State: link.StateUnhealthy, Reason: "down"}}}}
+		return f, err
+	}
+	gateEvery, gateBound = 0, 400*time.Millisecond
+	for deadline := time.Now().Add(5 * time.Second); time.Now().Before(deadline); {
+		advancePlans(ctx, g.open)
+		if p := g.plan(t); p.State == inventory.PlanFailed || p.State == inventory.PlanDone {
+			break
+		}
+		time.Sleep(30 * time.Millisecond)
+	}
+	p := g.plan(t)
+	if p.State != inventory.PlanFailed {
+		t.Fatalf("the plan is %s: %s; want it failed on late", p.State, p.Note)
+	}
+	if v, found, err := inv.GateOf(ctx, "build-2"); err != nil || !found || v.Verdict != inventory.GatePassed ||
+		!strings.Contains(v.Why, "on its own") {
+		t.Fatalf("app's verdict: %+v (found %v, %v); want its own pass kept", v, found, err)
+	}
+	if failed, _ := inv.GateFailed(ctx, "build-late-2"); !failed {
+		t.Fatal("late's build is not marked failed at its gate")
+	}
+	if current, _ := inv.CurrentBuilds(ctx); current["app"].Commit != "c2" || current["late"].Commit != "l1" {
+		t.Fatalf("registered app %s, late %s; want app kept at c2 and late put back to l1", current["app"].Commit,
+			current["late"].Commit)
+	}
+}
+
+// **A provider waiting for a login says who waits on it** (review (g)): its consumer, failing a check that
+// needs it, is held under it, and the provider's relogin-needed condition lists it and is urgent.
+func TestAProviderWaitingForALoginSaysWhoWaitsOnIt(t *testing.T) {
+	open := aMesh(t)
+	ctx := t.Context()
+	inv := open.inventory
+	k := conditionsFrom
+	register(t, open, catalogue.Manifest{Module: "db", Version: "1",
+		Provides: []catalogue.Offer{{Name: "postgres-database", Scope: catalogue.ScopeMesh}}})
+	register(t, open, catalogue.Manifest{Module: "shop", Version: "1", Requires: []string{"postgres-database"}})
+	for _, a := range [][2]string{{"anchor", "db"}, {"laptop", "shop"}} {
+		if _, err := inv.Assign(ctx, a[0], a[1]); err != nil {
+			t.Fatal(err)
+		}
+	}
+	if err := inv.RecordBindings(ctx, "laptop", []inventory.Binding{{Machine: "laptop", Consumer: "shop",
+		Provision: "postgres-database", Provider: catalogue.Chosen{Node: "anchor", Module: "db"}}}); err != nil {
+		t.Fatal(err)
+	}
+	at := h0
+	say := func(machine string, rs ...link.ResourceHealth) {
+		t.Helper()
+		at = at.Add(time.Second)
+		for i := range rs {
+			rs[i].Since = at
+		}
+		if err := stateHealth(ctx, inv, k, machine, link.Health{Contract: link.ReadinessContract, At: at, Resources: rs}, at); err != nil {
+			t.Fatal(err)
+		}
+	}
+	account := link.ResourceHealth{Module: "db", Resource: "db.operator", Kind: link.KindAccount, Target: "operator",
+		Account: "operator", State: link.StateUnhealthy, Reason: link.ReasonRelogin + ": operator is in the group db"}
+	unit := link.ResourceHealth{Module: "db", Resource: "db.server", Kind: link.KindUnit, Target: "db.service",
+		Account: "operator", State: link.StateUnhealthy, Reason: "failed in the account's own service manager (exit-code)"}
+	shop := link.ResourceHealth{Module: "shop", Resource: "shop.web", Kind: "container", Target: "shop",
+		State: link.StateUnhealthy, Reason: "http /health on web: answered 500", Check: "http", Needs: "postgres-database"}
+	for look := 0; look < 2; look++ {
+		say("anchor", account, unit)
+		say("laptop", shop)
+	}
+	list, err := k.Open(ctx)
+	if err != nil {
+		t.Fatal(err)
+	}
+	var keys []string
+	var c conditions.Condition
+	for _, x := range list {
+		keys = append(keys, x.Key)
+		if x.Key == "module.db.anchor.relogin-needed" {
+			c = x
+		}
+	}
+	if !reflect.DeepEqual(keys, []string{"module.db.anchor.relogin-needed"}) {
+		t.Fatalf("open %v; want the provider's wait alone, its consumer held under it", keys)
+	}
+	if c.Severity != conditions.Urgent || !strings.Contains(c.Evidence[0].Said, "shop on laptop") {
+		t.Fatalf("the provider's wait: %s, %q; want it urgent and naming its consumer", c.Severity, c.Evidence[0].Said)
+	}
+}

cmd/mesh-controller/module_health.go

@@ -152,14 +152,23 @@ func judgeModuleHealth(ctx context.Context, inv *inventory.Inventory, k *conditi
 	}
 	sort.Strings(modules)
 	seen := map[string]bool{}
+	// became is, per module, the kind of condition this statement says of it: what a standing one of the
+	// other kind turned into.
+	became := map[string]string{}
 	heldOn := map[string]string{}
 	providers := map[catalogue.Chosen]bool{}
 	for _, m := range modules {
 		// **A wait for a person's new login is said as that** (novox/hq ADR 0254): one plain sentence to the
 		// operator, never urgent, cleared on the first statement that no longer says it.
 		if said, waits := personWait(m, node, unhealthy[m]); waits {
-			o := reloginObservation(m, node, said, unhealthy[m])
+			o := reloginObservation(m, node, said, operatorOn(ctx, inv, node), unhealthy[m])
+			// **A provider waiting for a login says who waits on it** (novox/hq issue 318 review): its
+			// consumers are held under it, and its own condition is where they are said.
+			if hold != nil {
+				sayWaitingOn(&o, hold.waitersOn(catalogue.Chosen{Node: node, Module: m}))
+			}
 			seen[o.Key()] = true
+			became[m] = kindReloginNeeded
 			if _, isOpen := standing[o.Key()]; streaks[m] < moduleUnhealthyAfter && !isOpen {
 				continue
 			}
@@ -176,14 +185,10 @@ func judgeModuleHealth(ctx context.Context, inv *inventory.Inventory, k *conditi
 				providers[p] = true
 				continue
 			}
-			if waiters := hold.waitersOn(catalogue.Chosen{Node: node, Module: m}); len(waiters) > 0 {
+			sayWaitingOn(&o, hold.waitersOn(catalogue.Chosen{Node: node, Module: m}))
-				o.Severity = conditions.Urgent
-				o.Said += "; " + waitingWords(waiters)
-				o.Summary += fmt.Sprintf("; %d consumer(s) wait on it", len(waiters))
-				o.Explanation += fmt.Sprintf(" %d module(s) that depend on it wait for it.", len(waiters))
-			}
 		}
 		seen[o.Key()] = true
+		became[m] = kindModuleUnhealthy
 		c, isOpen := standing[o.Key()]
 		if streaks[m] < moduleUnhealthyAfter && !isOpen {
 			continue // unconfirmed: one statement can be wrong; `node show` lists it
@@ -207,7 +212,18 @@ func judgeModuleHealth(ctx context.Context, inv *inventory.Inventory, k *conditi
 		if on, held := heldOn[key]; held {
 			why = fmt.Sprintf("what %s finds on %s waits on %s, which is unhealthy: held under its condition", module, node, on)
 		}
-		if _, err := k.Clear(ctx, key, why); err != nil {
+		// **A condition that became the other kind** is not "working again" (issue 318 review): its clearing
+		// line says what it became.
+		resolved := ""
+		switch {
+		case c.Kind == kindModuleUnhealthy && became[module] == kindReloginNeeded:
+			why = fmt.Sprintf("%s on %s now waits only for a new login", module, node)
+			resolved = fmt.Sprintf("%s on %s now waits only for a new login", module, node)
+		case c.Kind == kindReloginNeeded && became[module] == kindModuleUnhealthy:
+			why = fmt.Sprintf("%s on %s no longer waits for a new login, and is not healthy", module, node)
+			resolved = fmt.Sprintf("The new login on %s is done, and %s still does not work", node, module)
+		}
+		if _, err := k.ClearSaying(ctx, key, why, resolved); err != nil {
 			problems = append(problems, err.Error())
 		}
 	}
@@ -229,7 +245,7 @@ func judgeModuleHealth(ctx context.Context, inv *inventory.Inventory, k *conditi
 func sayWaiters(ctx context.Context, k *conditions.Keeper, hold *holding, p catalogue.Chosen, now time.Time) error {
 	var raisedAt *conditions.Condition
 	for i, c := range hold.open {
-		if c.Key == moduleUnhealthyKey(p.Module, p.Node) {
+		if c.Key == moduleUnhealthyKey(p.Module, p.Node) || c.Key == reloginKey(p.Module, p.Node) {
 			raisedAt = &hold.open[i]
 		}
 	}
@@ -246,35 +262,97 @@ func sayWaiters(ctx context.Context, k *conditions.Keeper, hold *holding, p cata
 		return nil
 	}
 	o := moduleUnhealthyObservation(p.Module, p.Node, rs)
-	if waiters := hold.waitersOn(p); len(waiters) > 0 {
+	if said, waits := personWait(p.Module, p.Node, rs); waits {
-		o.Severity = conditions.Urgent
+		o = reloginObservation(p.Module, p.Node, said, operatorOn(ctx, hold.inv, p.Node), rs)
-		o.Said += "; " + waitingWords(waiters)
-		o.Summary += fmt.Sprintf("; %d consumer(s) wait on it", len(waiters))
 	}
+	if o.Key() != raisedAt.Key {
+		return nil // its own statement says it next
+	}
+	sayWaitingOn(&o, hold.waitersOn(p))
 	_, err := k.Observe(ctx, o)
 	return err
 }
 
+// reloginKey is a module's relogin-needed condition on a machine.
+func reloginKey(module, node string) string {
+	return conditions.Key(conditions.ScopeModule, module+"."+node, kindReloginNeeded)
+}
+
+// operatorOn is the operator's account on a machine, or "" when it is not known (to-be 29).
+func operatorOn(ctx context.Context, inv *inventory.Inventory, node string) string {
+	if inv == nil {
+		return ""
+	}
+	n, err := inv.NodeByName(ctx, node)
+	if err != nil {
+		return ""
+	}
+	return n.Account
+}
+
 // reloginObservation is a module waiting for a person's new login on a machine (novox/hq ADR 0254): the
 // operator's, a warning, its summary the one sentence that says what to do; the resources are evidence. Its
 // plain words (ADR 0253) are the kind's: it needs the operator, and offers no answer — no verb can log a
-// person in again, and a restart of the module's service would start it in the same session.
+// person in again, and a restart of the module's service would start it in the same session. operator is
-func reloginObservation(module, node, said string, rs []inventory.ResourceHealth) conditions.Observation {
+// the machine's operator account, when known: the words say "your account" only for it.
+func reloginObservation(module, node, said, operator string, rs []inventory.ResourceHealth) conditions.Observation {
 	o := moduleUnhealthyObservation(module, node, rs)
 	o.Token, o.Kind, o.Resolver, o.Summary = kindReloginNeeded, kindReloginNeeded, conditions.ResolverOperator, said
-	w := reloginWords(module, node)
+	accounts := waitingAccounts(module, rs)
+	yours := operator != "" && len(accounts) > 0
+	for _, a := range accounts {
+		yours = yours && a == operator
+	}
+	w := reloginWords(module, node, yours)
 	o.Headline, o.Explanation, o.Resolved, o.Needs, o.Actions = w.Headline, w.Explanation, w.Resolved, w.Needs, nil
 	return o
 }
 
-// reloginWords is what the operator reads of a module waiting for their new login on a machine (ADR 0253,
+// reloginWords is what the operator reads of a module waiting for a new login on a machine (ADR 0253,
-// ADR 0254): never quiet, since only the operator can do it, and no button, since nothing else can.
+// ADR 0254): never quiet, since only a person can do it, and no button, since nothing else can. yours says
-func reloginWords(module, node string) words {
+// the account waiting is the operator's own on that machine; otherwise the words do not claim it is.
+func reloginWords(module, node string, yours bool) words {
+	if yours {
+		return words{Headline: fmt.Sprintf("%s waits for a new login on %s", module, node),
+			Needs: fmt.Sprintf("log out of %s completely and log in again, or restart it.", node),
+			Explanation: fmt.Sprintf("%s put your account in a group it needs. You logged in before that, so %s "+
+				"cannot run until you log in again. Its update is in place and nothing was undone.",
+				conditions.Capital(module), module),
+			Resolved: fmt.Sprintf("%s runs on %s after your new login", module, node)}
+	}
 	return words{Headline: fmt.Sprintf("%s waits for a new login on %s", module, node),
-		Needs: fmt.Sprintf("log out of %s completely and log in again, or restart it.", node),
+		Needs: fmt.Sprintf("have the account it names log out of %s completely and log in again, or restart %s.", node, node),
-		Explanation: fmt.Sprintf("%s put your account in a group it needs. You logged in before that, so %s "+
+		Explanation: fmt.Sprintf("%s put an account on %s in a group it needs. That account logged in before that, "+
-			"cannot run until you log in again. Its update is in place and nothing was undone.", conditions.Capital(module), module),
+			"so %s cannot run until it logs in again. Its update is in place and nothing was undone.",
-		Resolved: fmt.Sprintf("%s runs on %s after your new login", module, node)}
+			conditions.Capital(module), node, module),
+		Resolved: fmt.Sprintf("%s runs on %s after the new login", module, node)}
+}
+
+// waitingAccounts is every account of a module whose resource says it waits for a new login, sorted.
+func waitingAccounts(module string, rs []inventory.ResourceHealth) []string {
+	seen := map[string]bool{}
+	var out []string
+	for _, r := range rs {
+		if r.Module == module && r.Kind == link.KindAccount && r.State == link.StateUnhealthy &&
+			strings.HasPrefix(r.Reason, link.ReasonRelogin) && accountOf(r) != "" && !seen[accountOf(r)] {
+			seen[accountOf(r)] = true
+			out = append(out, accountOf(r))
+		}
+	}
+	sort.Strings(out)
+	return out
+}
+
+// sayWaitingOn adds to a module's condition the consumers held under it (to-be 48 §6): urgent while anyone
+// waits on it, whether it is not working or waits for a new login.
+func sayWaitingOn(o *conditions.Observation, waiters []string) {
+	if len(waiters) == 0 {
+		return
+	}
+	o.Severity = conditions.Urgent
+	o.Said += "; " + waitingWords(waiters)
+	o.Summary += fmt.Sprintf("; %d consumer(s) wait on it", len(waiters))
+	o.Explanation += fmt.Sprintf(" %d module(s) that depend on it wait for it.", len(waiters))
 }
 
 // moduleUnhealthyObservation is a module unhealthy on a machine, in words: the summary names the module,
@@ -353,18 +431,14 @@ const kindReloginNeeded = "relogin-needed"
 // not in its group or that could not be read — is not a wait, and the module is judged as before. A
 // resource still starting is not unhealthy and does not make a wait either. Pure.
 func personWait(module, machine string, rs []inventory.ResourceHealth) (string, bool) {
-	waiting := map[string]bool{}
+	accounts := waitingAccounts(module, rs)
-	var accounts []string
-	for _, r := range rs {
-		if r.Module == module && r.Kind == link.KindAccount && r.State == link.StateUnhealthy &&
-			strings.HasPrefix(r.Reason, link.ReasonRelogin) && accountOf(r) != "" && !waiting[accountOf(r)] {
-			waiting[accountOf(r)] = true
-			accounts = append(accounts, accountOf(r))
-		}
-	}
 	if len(accounts) == 0 {
 		return "", false
 	}
+	waiting := map[string]bool{}
+	for _, a := range accounts {
+		waiting[a] = true
+	}
 	var units []string
 	for _, r := range rs {
 		if r.Module != module || r.State != link.StateUnhealthy {
@@ -378,7 +452,6 @@ func personWait(module, machine string, rs []inventory.ResourceHealth) (string,
 			return "", false
 		}
 	}
-	sort.Strings(accounts)
 	said := fmt.Sprintf("relogin needed on %s: %s waits for a new login of %s, which is in its group and whose "+
 		"running session began before it was; log out of every session and in again, or reboot", machine, module,
 		strings.Join(accounts, ", "))
@@ -416,6 +489,18 @@ func moduleHealthWord(module, machine string, since time.Time, f gateFacts) (hea
 		return healthNotYet, fmt.Sprintf("%s has not said how what %s runs is since it was sent", machine, module)
 	}
 	wait, waits := personWait(module, machine, h.Resources)
+	// **Only the build whose own send put the account into its wait is excused** (issue 318 review): an
+	// account already waiting before this send was not brought to it by this build, and a later build of
+	// the module sent while the login is still owed is judged as before. Two seconds of skew between the
+	// machine's clock and the controller's, as the witness allows.
+	if waits {
+		for _, r := range h.Resources {
+			if r.Module == module && r.Kind == link.KindAccount && r.State == link.StateUnhealthy &&
+				strings.HasPrefix(r.Reason, link.ReasonRelogin) && r.Since.Before(since.Add(-2*time.Second)) {
+				waits = false
+			}
+		}
+	}
 	for _, r := range h.Resources {
 		if r.Module != module {
 			continue

cmd/mesh-controller/person_wait_test.go

@@ -75,11 +75,24 @@ func TestOnlyWhatDependsOnTheNewLoginIsAWait(t *testing.T) {
 func TestAWaitForAPersonIsAPassCarriedAlong(t *testing.T) {
 	now := time.Now()
 	since := now.Add(-time.Minute)
+	account := relogin("lights", "operator")
+	account.Since = since.Add(10 * time.Second) // the send put it in its wait
 	f := gateFacts{now: now, health: map[string]inventory.NodeHealth{"laptop": {Node: "laptop", HeardAt: now,
-		Resources: []inventory.ResourceHealth{relogin("lights", "operator"), userUnit("lights", "operator")}}}}
+		Resources: []inventory.ResourceHealth{account, userUnit("lights", "operator")}}}}
 	if h, why := moduleHealthWord("lights", "laptop", since, f); h != healthPerson || !strings.Contains(why, "relogin needed on laptop") {
 		t.Fatalf("a wait for a new login reads %v %q; want a wait for a person", h, why)
 	}
+	// **Only the send that put the account in its wait is excused** (issue 318 review): an account waiting
+	// since before the send is judged as before, so a later build of the module is not passed on an old wait.
+	older := f.health["laptop"]
+	older.Resources = []inventory.ResourceHealth{relogin("lights", "operator"), userUnit("lights", "operator")}
+	older.Resources[0].Since = since.Add(-time.Hour)
+	f.health["laptop"] = older
+	if h, why := moduleHealthWord("lights", "laptop", since, f); h != healthNotYet {
+		t.Fatalf("a wait from before the send reads %v %q; want not yet", h, why)
+	}
+	f.health["laptop"] = inventory.NodeHealth{Node: "laptop", HeardAt: now,
+		Resources: []inventory.ResourceHealth{account, userUnit("lights", "operator")}}
 	h := f.health["laptop"]
 	h.Resources = append(h.Resources, inventory.ResourceHealth{Module: "lights", Resource: "lights.web", Kind: "container",
 		Target: "lights", State: link.StateUnhealthy, Reason: "down"})
@@ -197,7 +210,7 @@ func TestAModuleHealthyOnItsOwnKeepsItsPassWhenItsSendFails(t *testing.T) {
 // What the operator reads of the wait (ADR 0253, ADR 0254): it needs them, so it is never quiet, and it offers
 // no button, since nothing but their own new login can do it.
 func TestTheReloginConditionNeedsTheOperatorAndOffersNoButton(t *testing.T) {
-	o := reloginObservation("openrazer", "g14", "relogin needed on g14: …", []inventory.ResourceHealth{
+	o := reloginObservation("openrazer", "g14", "relogin needed on g14: …", "operator", []inventory.ResourceHealth{
 		relogin("openrazer", "operator"), userUnit("openrazer", "operator")})
 	plainExample(t, o, "openrazer waits for a new login on g14",
 		"Needs you: log out of g14 completely and log in again, or restart it. Openrazer put your account in a "+
@@ -206,9 +219,64 @@ func TestTheReloginConditionNeedsTheOperatorAndOffersNoButton(t *testing.T) {
 	if len(o.Actions) != 0 || o.Needs == "" {
 		t.Errorf("relogin: needs %q, actions %+v; want needs and no button", o.Needs, o.Actions)
 	}
-	// The kind's own wording, for a condition raised without words, says the same.
+	// **Not "your account" when it is not the operator's** (issue 318 review).
-	if w := plainWordings[kindReloginNeeded](conditions.Observation{Scope: conditions.ScopeModule, ID: "openrazer.g14", Machine: "g14"}); w.Needs != o.Needs ||
+	other := reloginObservation("openrazer", "g14", "relogin needed on g14: …", "operator", []inventory.ResourceHealth{
-		w.Headline != o.Headline || len(w.Actions) != 0 {
+		relogin("openrazer", "guest"), userUnit("openrazer", "guest")})
+	plainExample(t, other, "openrazer waits for a new login on g14",
+		"Needs you: have the account it names log out of g14 completely and log in again, or restart g14. Openrazer "+
+			"put an account on g14 in a group it needs. That account logged in before that, so openrazer cannot run "+
+			"until it logs in again. Its update is in place and nothing was undone.")
+	if unknown := reloginObservation("openrazer", "g14", "…", "", []inventory.ResourceHealth{relogin("openrazer", "operator")}); strings.Contains(unknown.Explanation, "your account") {
+		t.Errorf("an operator not known is still told it is their account: %q", unknown.Explanation)
+	}
+	// The kind's own wording, for a condition raised without words, names the module whole, dots and all,
+	// and never claims the account is the operator's.
+	w := plainWordings[kindReloginNeeded](conditions.Observation{Scope: conditions.ScopeModule, ID: "razer.lights.g14", Machine: "g14"})
+	if w.Headline != "razer.lights waits for a new login on g14" || w.Needs == "" || len(w.Actions) != 0 ||
+		strings.Contains(w.Explanation, "your account") {
 		t.Errorf("the kind's wording: %+v", w)
 	}
 }
+
+// When a module not working turns into one waiting for a new login, and back, the clearing line says what it
+// became, never that it works again (issue 318 review).
+func TestAConditionThatBecameTheOtherKindSaysSoWhenItClears(t *testing.T) {
+	open := aMesh(t)
+	ctx := t.Context()
+	store := conditions.NewInMemory()
+	told := &conditions.Told{}
+	k := conditions.NewKeeper(ctx, conditions.Options{Store: store, History: store, Teller: told})
+	t.Cleanup(func() { k.Close(context.Background()) })
+	at := h0
+	say := func(rs ...link.ResourceHealth) {
+		t.Helper()
+		at = at.Add(time.Second)
+		for i := range rs {
+			rs[i].Since = at
+		}
+		if err := stateHealth(ctx, open.inventory, k, "laptop", link.Health{Contract: link.ReadinessContract, At: at,
+			Resources: rs}, at); err != nil {
+			t.Fatal(err)
+		}
+	}
+	unit := link.ResourceHealth{Module: "lights", Resource: "lights.daemon", Kind: link.KindUnit, Target: "lights.service",
+		Account: "operator", State: link.StateUnhealthy, Reason: "failed in the account's own service manager (exit-code)"}
+	account := link.ResourceHealth{Module: "lights", Resource: "lights.operator", Kind: link.KindAccount, Target: "operator",
+		Account: "operator", State: link.StateUnhealthy, Reason: link.ReasonRelogin + ": operator is in the group lights"}
+	say(unit)
+	say(unit) // lights not working
+	say(account, unit)
+	say(account, unit) // now it waits for a login
+	var resolved []string
+	for deadline := time.Now().Add(5 * time.Second); time.Now().Before(deadline) && len(resolved) == 0; {
+		time.Sleep(20 * time.Millisecond)
+		for _, e := range told.Said() {
+			if e.Change == conditions.ChangeCleared {
+				resolved = append(resolved, e.Condition.Key+": "+e.Condition.Resolved)
+			}
+		}
+	}
+	if len(resolved) != 1 || !strings.Contains(resolved[0], "module.lights.laptop.unhealthy: lights on laptop now waits only for a new login") {
+		t.Fatalf("cleared %v; want the not-working condition cleared as now waiting for a login", resolved)
+	}
+}

cmd/mesh-controller/plain_words.go

@@ -184,11 +184,12 @@ var plainWordings = map[string]func(conditions.Observation) words{
 			Resolved:    conditions.Capital(thing) + " works again"}
 	}),
 	kindReloginNeeded: worded(func(o conditions.Observation) words {
+		// A module's name may hold dots: it is the id without its machine.
 		module := ""
-		if o.Scope == conditions.ScopeModule {
+		if o.Scope == conditions.ScopeModule && o.Machine != "" {
-			module = idPart(o, 0)
+			module = strings.TrimSuffix(o.ID, "."+o.Machine)
 		}
-		return reloginWords(orModule(module), machineOr(o, "a machine"))
+		return reloginWords(orModule(module), machineOr(o, "a machine"), false)
 	}),
 	kindProviderFailing: worded(func(o conditions.Observation) words {
 		thing, consumer := conditions.ThingWords(o), idPart(o, 2)

cmd/mesh-controller/release.go

@@ -319,7 +319,7 @@ func passCarried(ctx context.Context, open *stores, p *inventory.Plan, g *invent
 		}
 		seen[c.Build] = true
 		if err := open.inventory.RecordGate(ctx, inventory.GateVerdict{Build: c.Build, Module: c.Module, Commit: c.To,
-			Previous: c.From, Plan: p.ID, Machines: []string{c.Node}, Verdict: inventory.GatePassed, Why: g.Why,
+			Previous: c.From, Plan: p.ID, Machines: []string{c.Node}, Verdict: inventory.GatePassed, Why: whyFor(g, c.Module),
 			Component: coreComponent(c.Module), JudgingFrom: g.Since}); err != nil {
 			fmt.Printf("%s: %s passed its gate on %s, and the verdict could not be kept: %v\n", p.ID, c.Module, c.Node, err)
 		}
@@ -375,10 +375,7 @@ func keepPassing(ctx context.Context, open *stores, p *inventory.Plan, g *invent
 			continue
 		}
 		seen[c.Build] = true
-		why := fmt.Sprintf("healthy %d times on its own while the send failed: %s", g.Healthy[c.Module], g.Why)
+		why := passedAloneWhy(g, c.Module)
-		if w := g.Waits[c.Module]; w != "" {
-			why += "; and it waits for a person: " + w
-		}
 		if err := open.inventory.RecordGate(ctx, inventory.GateVerdict{Build: c.Build, Module: c.Module, Commit: c.To,
 			Previous: c.From, Plan: p.ID, Machines: []string{c.Node}, Verdict: inventory.GatePassed, Why: why,
 			Component: coreComponent(c.Module), JudgingFrom: g.Since}); err != nil {
@@ -393,6 +390,15 @@ func keepPassing(ctx context.Context, open *stores, p *inventory.Plan, g *invent
 	return kept
 }
 
+// passedAloneWhy is the verdict of a module that passed on its own while its send failed, with its own wait.
+func passedAloneWhy(g *inventory.PlanGate, module string) string {
+	why := fmt.Sprintf("healthy %d times on its own while the send failed: %s", g.Healthy[module], g.Why)
+	if w := g.Waits[module]; w != "" {
+		why += waitsPrefix + w
+	}
+	return why
+}
+
 // sentTheBuild is every machine running a module that was last sent this build of it.
 func sentTheBuild(ctx context.Context, open *stores, module, commit string) ([]string, error) {
 	running, err := open.inventory.Running(ctx, module)

cmd/mesh-controller/release_plan.go

@@ -703,7 +703,7 @@ func advanceOnce(ctx context.Context, open *stores, p *inventory.Plan,
 			if lead.Gate.Verdict != inventory.GatePassed {
 				continue
 			}
-			passedWith(state, state.GatedBy, lead.Gate)
+			passedWith(m, state, state.GatedBy, lead.Gate)
 		}
 		switch {
 		case step.failed != "":
@@ -937,14 +937,14 @@ func firstSend(ctx context.Context, open *stores, p *inventory.Plan, node string
 
 // passedWith keeps, on a module sent with others, the verdict of the gate that judged the send: the gate
 // on the first of them passed, and its pass was kept for every build it carried (passCarried).
-func passedWith(s *inventory.PlanModule, lead string, g *inventory.PlanGate) {
+func passedWith(module string, s *inventory.PlanModule, lead string, g *inventory.PlanGate) {
 	if s.Gate == nil {
 		s.Gate = &inventory.PlanGate{Machines: g.Machines, From: s.Previous, To: s.Commit, Since: g.Since}
 	}
 	if s.Gate.Verdict != "" {
 		return
 	}
-	s.Gate.Verdict, s.Gate.Why, s.Gate.JudgedAt, s.Gate.Took = g.Verdict, "judged with "+lead+": "+g.Why, g.JudgedAt, g.Took
+	s.Gate.Verdict, s.Gate.Why, s.Gate.JudgedAt, s.Gate.Took = g.Verdict, "judged with "+lead+": "+whyFor(g, module), g.JudgedAt, g.Took
 	s.Gate.Passes, s.Gate.Kept = g.Passes, true
 }
 
@@ -963,6 +963,18 @@ func failFirstSend(ctx context.Context, open *stores, p *inventory.Plan, module
 	g := state.Gate
 	if g == nil || g.Verdict == "" || len(g.Failing) == 0 || slices.Contains(g.Failing, module) {
 		gateFailed(batched, open, p, module, state, machines, why)
+	} else if slices.Contains(g.Passing, module) && state.Build != "" {
+		// **The module the gate is kept on keeps its own pass too** (issue 318 review): healthy for the passes
+		// asked while another of its send failed, its build has a verdict, and other walks do not wait on it.
+		if err := open.inventory.RecordGate(ctx, inventory.GateVerdict{Build: state.Build, Module: module,
+			Commit: state.Commit, Previous: state.Previous, Plan: p.ID, Machines: g.Machines, Verdict: inventory.GatePassed,
+			Why: passedAloneWhy(g, module), Component: g.Component, JudgingFrom: g.Since}); err != nil {
+			fmt.Printf("%s: %s passed its gate on its own, and the verdict could not be kept: %v\n", p.ID, module, err)
+		}
+		state.Why = "passed on its own; stopped with the send that carried it: " + g.Why
+		p.State = inventory.PlanFailed
+		p.Note = fmt.Sprintf("the send to %s in tier %d failed its gate: %s; %s passed on its own and is kept",
+			strings.Join(g.Machines, ", "), p.Tier, g.Why, module)
 	} else {
 		state.Why = "not found wanting; stopped with the send that carried it: " + g.Why
 		p.State = inventory.PlanFailed

cmd/mesh-controller/tier_send_test.go

@@ -255,8 +255,13 @@ func TestACoreBehindWhileTheNodeToolsSettleFailsNoOtherModule(t *testing.T) {
 			t.Fatalf("%s's build was marked failed for the node tools' condition", m)
 		}
 	}
-	if !strings.Contains(p.Note, "app1 was not found wanting") {
+	// Healthy for the passes asked, the module the gate is kept on keeps its own pass (novox/hq issue 318
-		t.Fatalf("the plan blames the module its gate was kept on: %s", p.Note)
+	// review): it is not blamed, and not left without a verdict.
+	if !strings.Contains(p.Note, "app1 passed on its own and is kept") {
+		t.Fatalf("the plan blames the module its gate was kept on, or leaves it unjudged: %s", p.Note)
+	}
+	if v, found, _ := inv.GateOf(ctx, "build-app1-2"); !found || v.Verdict != inventory.GatePassed {
+		t.Fatalf("app1's own pass was not kept: %+v", v)
 	}
 }
 

internal/conditions/store.go

@@ -276,6 +276,13 @@ func (k *Keeper) Observe(ctx context.Context, o Observation) (Condition, error)
 
 // Clear removes a condition an observation says is resolved, and says so. False when none was open.
 func (k *Keeper) Clear(ctx context.Context, key, why string) (bool, error) {
+	return k.ClearSaying(ctx, key, why, "")
+}
+
+// ClearSaying is Clear with the line the operator reads when it clears, in place of the condition's own
+// resolved line: for a condition that ends because it became another (novox/hq ADR 0254: a module not
+// working that now waits for a new login is not "working again"). Empty keeps the condition's own.
+func (k *Keeper) ClearSaying(ctx context.Context, key, why, resolved string) (bool, error) {
 	for i := 0; i < tries; i++ {
 		entry, found, err := k.store.Get(ctx, key)
 		if err != nil {
@@ -292,6 +299,9 @@ func (k *Keeper) Clear(ctx context.Context, key, why string) (bool, error) {
 		if err := k.stamp(&c); err != nil {
 			return false, err
 		}
+		if resolved != "" {
+			c.Resolved = resolved
+		}
 		if err := k.store.Delete(ctx, key, entry.Revision); errors.Is(err, ErrMoved) {
 			continue
 		} else if err != nil {

internal/inventory/plans.go

@@ -152,6 +152,13 @@ type PlanGate struct {
 	// Passing names the modules that passed on their own when the send failed (ADR 0254): each keeps its
 	// pass, and is not put back.
 	Passing []string `json:"passing,omitempty"`
+	// HealthyAt is, per module, when its newest healthy judging was counted: a pass is counted only gateEvery
+	// after the one before (issue 318 review).
+	HealthyAt map[string]time.Time `json:"healthy_at,omitempty"`
+	// Broken names the modules a judging found broken, kept for the rest of the judging, and BrokenWhy the
+	// first reason: the send fails, and the modules beside them are judged to their own verdict first.
+	Broken    []string `json:"broken,omitempty"`
+	BrokenWhy string   `json:"broken_why,omitempty"`
 }
 
 // CarriedMove is one module's build moving on a machine with a gated send.