Pass a wait for a person's new login with the wait carried, and keep the pass of a module healthy beside a failure (hq ADR 0254, issue 318)

cmd/mesh-controller/gate.go

@@ -77,6 +77,10 @@ type health int
 
 const (
 	healthGood health = iota
+	// healthPerson is a wait for a person (novox/hq ADR 0254): everything unhealthy of the module waits for
+	// one person's new login (personWait). The build did what it should; no bound a machine can meet ends
+	// the wait. It counts as a pass, and the gate carries the reading along in its verdict.
+	healthPerson
 	// healthWaiting is not a pass and not a fault: what the module's checks find waits on an unhealthy
 	// provider (ADR 0240 rule 5), so the judging waits — past the bound too — rather than putting back a
 	// build for something it did not do.
@@ -209,7 +213,9 @@ func judgeHealth(module, component string, m catalogue.Manifest, machine string,
 				firstLine(f.openErr.Error())
 		}
 		for _, c := range f.open {
-			if c.Source == gateProbe || c.Raised.Before(since) {
+			// A wait for a person's new login is the module's reading, not a fault raised since the send: the
+			// gate reads it from the statement below (ADR 0254).
+			if c.Source == gateProbe || c.Raised.Before(since) || c.Kind == kindReloginNeeded {
 				continue
 			}
 			onIt := c.Subject.Machine == machine || slices.Contains(c.Subject.Also, machine) ||
@@ -435,13 +441,19 @@ func judgeMoves(ctx context.Context, open *stores, g *inventory.PlanGate, pairs
 	for node, moved := range byMachine {
 		words[node] = aboutTheMachine(node, moved, *g.Since, facts)
 	}
+	// **Each module is judged on its own** (novox/hq ADR 0254, issue 318): its reading is the worst of its
+	// machines', its passes are counted apart, and the send's verdict is still one — but a module that was
+	// healthy on its own for the passes the gate asks keeps a pass when another beside it fails.
 	worst, why := healthGood, ""
 	var failing []string
 	broken := map[string]bool{}
+	reading := map[string]health{}
+	waits := map[string]string{}
+	var modules []string
 	for _, j := range pairs {
 		w := words[j.node]
 		h, said := judgeHealth(j.module, coreComponent(j.module), shelf[j.module], j.node, *g.Since, w.facts)
-		if h == healthGood {
+		if h == healthGood || h == healthPerson {
 			if on, named := w.on[j.module]; named {
 				h, said = healthNotYet, on
 			} else if w.whole != "" {
@@ -450,7 +462,16 @@ func judgeMoves(ctx context.Context, open *stores, g *inventory.PlanGate, pairs
 				h, said = healthWaiting, w.waiting
 			}
 		}
-		if h != healthGood && !slices.Contains(failing, j.module) {
+		if _, seen := reading[j.module]; !seen {
+			modules = append(modules, j.module)
+		}
+		if h > reading[j.module] {
+			reading[j.module] = h
+		}
+		if h == healthPerson {
+			waits[j.module] = joinSaid(waits[j.module], said)
+		}
+		if h > healthPerson && !slices.Contains(failing, j.module) {
 			failing = append(failing, j.module)
 		}
 		if h == healthBroken {
@@ -458,14 +479,45 @@ func judgeMoves(ctx context.Context, open *stores, g *inventory.PlanGate, pairs
 		}
 		if h > worst {
 			worst, why = h, said
-		} else if h == worst && h != healthGood && why == "" {
+		} else if h == worst && h > healthPerson && why == "" {
 			why = said
 		}
 	}
+	if g.Healthy == nil {
+		g.Healthy = map[string]int{}
+	}
+	g.Waits = nil
+	for _, m := range modules {
+		switch {
+		case reading[m] > healthPerson:
+			g.Healthy[m] = 0
+		default:
+			g.Healthy[m]++
+			if w := waits[m]; w != "" {
+				if g.Waits == nil {
+					g.Waits = map[string]string{}
+				}
+				g.Waits[m] = w
+			}
+		}
+	}
+	// passedAlone is every module that passed on its own while the send as a whole did not.
+	passedAlone := func() []string {
+		var out []string
+		if now.Sub(*g.Since) < gateSettle {
+			return nil
+		}
+		for _, m := range modules {
+			if reading[m] <= healthPerson && g.Healthy[m] >= gatePasses {
+				out = append(out, m)
+			}
+		}
+		return out
+	}
 	switch {
 	case worst == healthBroken:
 		// What broke is put back; what was only not yet healthy beside it is too — they moved together.
-		g.Failing = failing
+		g.Failing, g.Passing = failing, passedAlone()
 		decide(g, inventory.GateFailed, why, now)
 	case worst == healthWaiting:
 		// Waiting on a provider that is unhealthy: not a pass, and not a failure at the bound either —
@@ -474,19 +526,48 @@ func judgeMoves(ctx context.Context, open *stores, g *inventory.PlanGate, pairs
 	case worst == healthNotYet:
 		g.Passes, g.LastPass, g.Last, g.Failing = 0, nil, why, failing
 		if now.Sub(*g.Since) > gateBound {
+			g.Passing = passedAlone()
 			decide(g, inventory.GateFailed, fmt.Sprintf("not healthy within %s of its apply: %s", gateBound, why), now)
 		}
 	default:
+		// Healthy, or waiting for a person (ADR 0254): a pass, the wait carried along in the verdict.
 		g.Passes++
 		g.LastPass, g.Last, g.Failing = &now, "", nil
 		if g.Passes >= gatePasses && now.Sub(*g.Since) >= gateSettle {
 			decide(g, inventory.GatePassed, fmt.Sprintf("healthy %d times over %s", g.Passes,
-				now.Sub(*g.Since).Round(time.Second)), now)
+				now.Sub(*g.Since).Round(time.Second))+waitsSaid(g.Waits), now)
 		}
 	}
 	return g.Verdict, nil
 }
 
+// waitsSaid is the waits for a person a passing gate carries, as its verdict says them.
+func waitsSaid(waits map[string]string) string {
+	if len(waits) == 0 {
+		return ""
+	}
+	modules := make([]string, 0, len(waits))
+	for m := range waits {
+		modules = append(modules, m)
+	}
+	sort.Strings(modules)
+	var said []string
+	for _, m := range modules {
+		said = append(said, waits[m])
+	}
+	return "; and it waits for a person: " + strings.Join(said, "; ")
+}
+
+func joinSaid(a, b string) string {
+	switch {
+	case a == "":
+		return b
+	case b == "" || strings.Contains(a, b):
+		return a
+	}
+	return a + "; " + b
+}
+
 // decide sets a gate's verdict.
 func decide(g *inventory.PlanGate, verdict, why string, now time.Time) {
 	g.Verdict, g.Why, g.JudgedAt = verdict, why, &now
@@ -959,6 +1040,12 @@ func gateLine(g *inventory.PlanGate) string {
 	} else if g.Verdict == "" {
 		line += fmt.Sprintf(" (%d of %d passes)", g.Passes, gatePasses)
 	}
+	if g.Verdict == "" && len(g.Waits) > 0 {
+		line += waitsSaid(g.Waits)
+	}
+	if len(g.Passing) > 0 {
+		line += "; passed on their own and kept: " + strings.Join(g.Passing, ", ")
+	}
 	if g.Rollback != "" {
 		line += "; " + g.Rollback
 	}

cmd/mesh-controller/module_health.go

@@ -73,7 +73,7 @@ func stateHealth(ctx context.Context, inv *inventory.Inventory, k *conditions.Ke
 	for _, r := range h.Resources {
 		kept := inventory.ResourceHealth{Module: r.Module, Resource: r.Resource, Kind: r.Kind, Target: r.Target,
 			State: r.State, Reason: r.Reason, Since: r.Since, Streak: r.Streak, Restarts: r.Restarts,
-			Check: r.Check, Needs: r.Needs}
+			Check: r.Check, Needs: r.Needs, Account: r.Account}
 		resources = append(resources, kept)
 		if r.State == link.StateUnhealthy && r.Module != "" {
 			unhealthy[r.Module] = append(unhealthy[r.Module], kept)
@@ -135,7 +135,7 @@ func judgeModuleHealth(ctx context.Context, inv *inventory.Inventory, k *conditi
 	}
 	standing := map[string]conditions.Condition{}
 	for _, c := range open {
-		if c.Kind == kindModuleUnhealthy && c.Subject.Machine == node {
+		if (c.Kind == kindModuleUnhealthy || c.Kind == kindReloginNeeded) && c.Subject.Machine == node {
 			standing[c.Key] = c
 		}
 	}
@@ -155,6 +155,19 @@ func judgeModuleHealth(ctx context.Context, inv *inventory.Inventory, k *conditi
 	heldOn := map[string]string{}
 	providers := map[catalogue.Chosen]bool{}
 	for _, m := range modules {
+		// **A wait for a person's new login is said as that** (novox/hq ADR 0254): one plain sentence to the
+		// operator, never urgent, cleared on the first statement that no longer says it.
+		if said, waits := personWait(m, node, unhealthy[m]); waits {
+			o := reloginObservation(m, node, said, unhealthy[m])
+			seen[o.Key()] = true
+			if _, isOpen := standing[o.Key()]; streaks[m] < moduleUnhealthyAfter && !isOpen {
+				continue
+			}
+			if _, err := k.Observe(ctx, o); err != nil {
+				problems = append(problems, err.Error())
+			}
+			continue
+		}
 		o := moduleUnhealthyObservation(m, node, unhealthy[m])
 		if hold != nil {
 			if p, held := hold.heldUnder(node, m, unhealthy[m]); held {
@@ -188,6 +201,9 @@ func judgeModuleHealth(ctx context.Context, inv *inventory.Inventory, k *conditi
 		}
 		module := strings.TrimSuffix(c.Subject.ID, "."+node)
 		why := fmt.Sprintf("%s says no resource of %s is unhealthy", node, module)
+		if c.Kind == kindReloginNeeded {
+			why = fmt.Sprintf("%s says %s no longer waits for a new login", node, module)
+		}
 		if on, held := heldOn[key]; held {
 			why = fmt.Sprintf("what %s finds on %s waits on %s, which is unhealthy: held under its condition", module, node, on)
 		}
@@ -239,6 +255,14 @@ func sayWaiters(ctx context.Context, k *conditions.Keeper, hold *holding, p cata
 	return err
 }
 
+// reloginObservation is a module waiting for a person's new login on a machine (novox/hq ADR 0254): the
+// operator's, a warning, its summary the one sentence that says what to do; the resources are evidence.
+func reloginObservation(module, node, said string, rs []inventory.ResourceHealth) conditions.Observation {
+	o := moduleUnhealthyObservation(module, node, rs)
+	o.Token, o.Kind, o.Resolver, o.Summary = kindReloginNeeded, kindReloginNeeded, conditions.ResolverOperator, said
+	return o
+}
+
 // moduleUnhealthyObservation is a module unhealthy on a machine, in words: the summary names the module,
 // the machine and what is wrong with each resource; the detail — targets, streaks, since — is evidence.
 func moduleUnhealthyObservation(module, node string, rs []inventory.ResourceHealth) conditions.Observation {
@@ -296,9 +320,76 @@ func orNotSaid(s string) string {
 	return s
 }
 
+// kindReloginNeeded is a module's condition while it waits for a person's new login on a machine (novox/hq
+// ADR 0254): its own kind, so that neither the gate nor anyone reading the conditions takes it for a fault.
+const kindReloginNeeded = "relogin-needed"
+
+// personWait is whether everything unhealthy of a module on a machine waits for one person's new login, and
+// that wait in one plain sentence (novox/hq ADR 0254, issue 318). Narrow on purpose, so that a fault is never
+// excused:
+//
+//   - at least one of the module's resources is its account (kind account), unhealthy with a reason that
+//     starts "relogin needed" (ADR 0252): the database lists the account in the group, and the session
+//     running began before;
+//   - every other unhealthy resource of the module runs in the own service manager of one of those very
+//     accounts (Account names it): the manager that began before the group and does not hold it.
+//
+// Anything else unhealthy of the module — a container, a unit of the machine's own manager, a unit in
+// another account's manager, a resource whose engine does not say whose manager it is in, an account
+// not in its group or that could not be read — is not a wait, and the module is judged as before. A
+// resource still starting is not unhealthy and does not make a wait either. Pure.
+func personWait(module, machine string, rs []inventory.ResourceHealth) (string, bool) {
+	waiting := map[string]bool{}
+	var accounts []string
+	for _, r := range rs {
+		if r.Module == module && r.Kind == link.KindAccount && r.State == link.StateUnhealthy &&
+			strings.HasPrefix(r.Reason, link.ReasonRelogin) && accountOf(r) != "" && !waiting[accountOf(r)] {
+			waiting[accountOf(r)] = true
+			accounts = append(accounts, accountOf(r))
+		}
+	}
+	if len(accounts) == 0 {
+		return "", false
+	}
+	var units []string
+	for _, r := range rs {
+		if r.Module != module || r.State != link.StateUnhealthy {
+			continue
+		}
+		switch {
+		case r.Kind == link.KindAccount && strings.HasPrefix(r.Reason, link.ReasonRelogin) && waiting[accountOf(r)]:
+		case r.Kind != link.KindAccount && r.Account != "" && waiting[r.Account]:
+			units = append(units, r.Target)
+		default:
+			return "", false
+		}
+	}
+	sort.Strings(accounts)
+	said := fmt.Sprintf("relogin needed on %s: %s waits for a new login of %s, which is in its group and whose "+
+		"running session began before it was; log out of every session and in again, or reboot", machine, module,
+		strings.Join(accounts, ", "))
+	if len(units) > 0 {
+		sort.Strings(units)
+		said += fmt.Sprintf(" (until then %s cannot run in that session)", strings.Join(units, ", "))
+	}
+	return said, true
+}
+
+// accountOf is the account a resource of kind account is: named by the engine, or its target.
+func accountOf(r inventory.ResourceHealth) string {
+	if r.Account != "" {
+		return r.Account
+	}
+	return r.Target
+}
+
 // moduleHealthWord is the gate's reading of a module's stated health on a machine (ADR 0240 §4, ADR 0236
 // §2 as amended): good when every long-running resource of it is stated healthy in a statement heard since
 // the send; not yet otherwise, saying which. A machine that never stated health is judged as before.
+//
+// **A wait for a person is its own reading** (novox/hq ADR 0254): when everything unhealthy of the module
+// waits for one person's new login (personWait), the reading is healthPerson — not a fault of the build,
+// and not something a machine can meet within a bound.
 func moduleHealthWord(module, machine string, since time.Time, f gateFacts) (health, string) {
 	if f.healthErr != nil {
 		return healthNotYet, "what " + machine + " says of its resources' health cannot be read: " + firstLine(f.healthErr.Error())
@@ -310,10 +401,14 @@ func moduleHealthWord(module, machine string, since time.Time, f gateFacts) (hea
 	if h.HeardAt.Before(since) {
 		return healthNotYet, fmt.Sprintf("%s has not said how what %s runs is since it was sent", machine, module)
 	}
+	wait, waits := personWait(module, machine, h.Resources)
 	for _, r := range h.Resources {
 		if r.Module != module {
 			continue
 		}
+		if waits && r.State == link.StateUnhealthy {
+			continue
+		}
 		switch r.State {
 		case link.StateHealthy:
 		case link.StateStarting:
@@ -329,6 +424,9 @@ func moduleHealthWord(module, machine string, since time.Time, f gateFacts) (hea
 				reasonAfter(r.Reason))
 		}
 	}
+	if waits {
+		return healthPerson, wait
+	}
 	return healthGood, ""
 }
 

cmd/mesh-controller/person_wait_test.go

@@ -0,0 +1,195 @@
+package main
+
+import (
+	"context"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/novox/mesh-controller/internal/conditions"
+	"github.com/novox/mesh-controller/internal/inventory"
+	"github.com/novox/mesh-controller/internal/link"
+)
+
+// A wait for a person is not a failure, and one module's verdict is not every module's (novox/hq ADR 0254,
+// issue 318).
+
+func relogin(module, account string) inventory.ResourceHealth {
+	return inventory.ResourceHealth{Module: module, Resource: module + "." + account, Kind: link.KindAccount, Target: account,
+		Account: account, State: link.StateUnhealthy, Reason: link.ReasonRelogin + ": " + account + " is in the group " + module +
+			", and its running session began before it was; log out of every session and in again, or reboot"}
+}
+
+func userUnit(module, account string) inventory.ResourceHealth {
+	return inventory.ResourceHealth{Module: module, Resource: module + ".daemon", Kind: link.KindUnit,
+		Target: module + "-daemon.service", Account: account, State: link.StateUnhealthy,
+		Reason: "failed in the account's own service manager (exit-code)"}
+}
+
+// Only what depends on the new login alone is a wait; anything else unhealthy of the module is judged as
+// before, so that a fault is never excused.
+func TestOnlyWhatDependsOnTheNewLoginIsAWait(t *testing.T) {
+	notInGroup := relogin("lights", "operator")
+	notInGroup.Reason = "not in the group lights: the apply has not put operator there; its outcome says why"
+	systemUnit := userUnit("lights", "operator")
+	systemUnit.Account = ""
+	otherAccount := userUnit("lights", "guest")
+	container := inventory.ResourceHealth{Module: "lights", Resource: "lights.web", Kind: "container", Target: "lights",
+		State: link.StateUnhealthy, Reason: "down"}
+	otherModule := userUnit("sound", "operator")
+	starting := userUnit("lights", "operator")
+	starting.State = link.StateStarting
+	byTarget := relogin("lights", "operator")
+	byTarget.Account = "" // an account said by its target alone is still that account
+
+	for _, c := range []struct {
+		name  string
+		rs    []inventory.ResourceHealth
+		waits bool
+	}{
+		{"the account alone", []inventory.ResourceHealth{relogin("lights", "operator")}, true},
+		{"the account and its unit in that account's manager", []inventory.ResourceHealth{relogin("lights", "operator"),
+			userUnit("lights", "operator")}, true},
+		{"an account named by its target", []inventory.ResourceHealth{byTarget, userUnit("lights", "operator")}, true},
+		{"another module's unit is not this module's", []inventory.ResourceHealth{relogin("lights", "operator"), otherModule}, true},
+		{"a unit still starting is no fault", []inventory.ResourceHealth{relogin("lights", "operator"), starting}, true},
+		{"a unit and no account", []inventory.ResourceHealth{userUnit("lights", "operator")}, false},
+		{"an account not in its group", []inventory.ResourceHealth{notInGroup, userUnit("lights", "operator")}, false},
+		{"a unit whose manager is not said", []inventory.ResourceHealth{relogin("lights", "operator"), systemUnit}, false},
+		{"a unit in another account's manager", []inventory.ResourceHealth{relogin("lights", "operator"), otherAccount}, false},
+		{"a container beside the wait", []inventory.ResourceHealth{relogin("lights", "operator"), container}, false},
+	} {
+		said, waits := personWait("lights", "laptop", c.rs)
+		if waits != c.waits {
+			t.Errorf("%s: waits %v; want %v", c.name, waits, c.waits)
+			continue
+		}
+		if waits && !strings.HasPrefix(said, "relogin needed on laptop: lights waits for a new login of operator") {
+			t.Errorf("%s: said %q", c.name, said)
+		}
+	}
+}
+
+// The gate reads a wait for a person as its own reading, never a fault, and passes with it; the same module
+// with a container down beside it is not yet healthy, as before.
+func TestAWaitForAPersonIsAPassCarriedAlong(t *testing.T) {
+	now := time.Now()
+	since := now.Add(-time.Minute)
+	f := gateFacts{now: now, health: map[string]inventory.NodeHealth{"laptop": {Node: "laptop", HeardAt: now,
+		Resources: []inventory.ResourceHealth{relogin("lights", "operator"), userUnit("lights", "operator")}}}}
+	if h, why := moduleHealthWord("lights", "laptop", since, f); h != healthPerson || !strings.Contains(why, "relogin needed on laptop") {
+		t.Fatalf("a wait for a new login reads %v %q; want a wait for a person", h, why)
+	}
+	h := f.health["laptop"]
+	h.Resources = append(h.Resources, inventory.ResourceHealth{Module: "lights", Resource: "lights.web", Kind: "container",
+		Target: "lights", State: link.StateUnhealthy, Reason: "down"})
+	f.health["laptop"] = h
+	if got, why := moduleHealthWord("lights", "laptop", since, f); got != healthNotYet {
+		t.Fatalf("a container down beside the wait reads %v %q; want not yet", got, why)
+	}
+}
+
+// The module's condition says the wait in one sentence for a person, and clears on the first statement that
+// no longer says it — said as the module's own fault when its unit still fails after the new login.
+func TestTheReloginConditionSaysTheWaitAndClearsAfterTheLogin(t *testing.T) {
+	open := aMesh(t)
+	ctx := t.Context()
+	k, _ := withConditionsInMemory(t)
+	at := h0
+	say := func(rs ...link.ResourceHealth) {
+		t.Helper()
+		at = at.Add(time.Second)
+		for i := range rs {
+			rs[i].Since = at
+		}
+		if err := stateHealth(ctx, open.inventory, k, "laptop", link.Health{Contract: link.ReadinessContract, At: at,
+			Resources: rs}, at); err != nil {
+			t.Fatal(err)
+		}
+	}
+	asLink := func(r inventory.ResourceHealth) link.ResourceHealth {
+		return link.ResourceHealth{Module: r.Module, Resource: r.Resource, Kind: r.Kind, Target: r.Target, State: r.State,
+			Reason: r.Reason, Account: r.Account}
+	}
+	account, unit := asLink(relogin("lights", "operator")), asLink(userUnit("lights", "operator"))
+	openNow := func() []conditions.Condition {
+		t.Helper()
+		list, err := k.Open(ctx)
+		if err != nil {
+			t.Fatal(err)
+		}
+		return list
+	}
+	say(account, unit)
+	say(account, unit)
+	list := openNow()
+	if len(list) != 1 || list[0].Key != "module.lights.laptop.relogin-needed" {
+		t.Fatalf("raised %+v; want lights' relogin-needed alone", list)
+	}
+	c := list[0]
+	if c.Severity != conditions.Warning || c.Resolver != conditions.ResolverOperator ||
+		!strings.HasPrefix(c.Summary, "relogin needed on laptop:") {
+		t.Fatalf("the condition is %s %s %q", c.Severity, c.Resolver, c.Summary)
+	}
+	// After the new login the account is healthy, and the unit, still failing, is the module's own fault.
+	account.State, account.Reason = link.StateHealthy, ""
+	say(account, unit)
+	list = openNow()
+	if len(list) != 1 || list[0].Key != "module.lights.laptop.unhealthy" {
+		t.Fatalf("after the login: %+v; want lights' own unhealthy condition alone", list)
+	}
+	unit.State, unit.Reason = link.StateHealthy, ""
+	say(account, unit)
+	if list = openNow(); len(list) != 0 {
+		t.Fatalf("after the unit runs: %+v; want nothing open", list)
+	}
+}
+
+// A module that is healthy on its own keeps its pass when another module of the same send fails at the
+// bound: it is neither put back nor left without a verdict for every other walk to wait on.
+func TestAModuleHealthyOnItsOwnKeepsItsPassWhenItsSendFails(t *testing.T) {
+	b := aBacklog(t)
+	ctx := t.Context()
+	inv := b.open.inventory
+	releaseHeard = func(context.Context, *stores) (map[string]bool, error) {
+		return map[string]bool{"laptop": true}, nil
+	}
+	backlogFacts := gatherGateFacts
+	gatherGateFacts = func(ctx context.Context, open *stores, component string) (gateFacts, error) {
+		f, err := backlogFacts(ctx, open, component)
+		f.health = map[string]inventory.NodeHealth{"laptop": {Node: "laptop", HeardAt: time.Now(),
+			Resources: []inventory.ResourceHealth{
+				{Module: "app", Resource: "app.web", Kind: "container", Target: "app", State: link.StateHealthy},
+				{Module: "late", Resource: "late.web", Kind: "container", Target: "late", State: link.StateUnhealthy,
+					Reason: "down"}}}}
+		return f, err
+	}
+	gateEvery, gateBound = 0, 300*time.Millisecond
+	deadline := time.Now().Add(5 * time.Second)
+	for time.Now().Before(deadline) {
+		advancePlans(ctx, b.open)
+		if p := b.release(t); p.State != inventory.PlanRolling {
+			break
+		}
+		time.Sleep(20 * time.Millisecond)
+	}
+	p := b.release(t)
+	if p.State != inventory.PlanFailed {
+		t.Fatalf("the walk is %s: %s; want it failed on late", p.State, p.Note)
+	}
+	if v, found, err := inv.GateOf(ctx, "build-app-c2"); err != nil || !found || v.Verdict != inventory.GatePassed ||
+		!strings.Contains(v.Why, "on its own") {
+		t.Fatalf("app's verdict: %+v (found %v, %v); want its own pass kept", v, found, err)
+	}
+	if failed, _ := inv.GateFailed(ctx, "build-late-c2"); !failed {
+		t.Fatal("late's build is not marked failed at its gate")
+	}
+	current, _ := inv.CurrentBuilds(ctx)
+	if current["app"].Commit != "c2" || current["late"].Commit != "c1" {
+		t.Fatalf("registered: app %s, late %s; want app kept at c2 and late put back to c1", current["app"].Commit,
+			current["late"].Commit)
+	}
+	if !strings.Contains(p.Note, "passed on their own and kept: app") {
+		t.Errorf("the walk does not say what kept its pass: %s", p.Note)
+	}
+}

cmd/mesh-controller/release.go

@@ -334,6 +334,15 @@ func failCarried(ctx context.Context, open *stores, p *inventory.Plan, g *invent
 		notes = append(notes, p.Note)
 	}
 	done := map[string]bool{except: true}
+	// **What passed on its own keeps its pass** (novox/hq ADR 0254, issue 318): healthy for the passes the
+	// gate asks while another module of the send failed, it is neither put back nor left unjudged — a build
+	// left without a verdict is one more move every other walk would wait for.
+	if kept := keepPassing(ctx, open, p, g, except); len(kept) > 0 {
+		notes = append(notes, "passed on their own and kept: "+strings.Join(kept, ", "))
+		for _, m := range kept {
+			done[m] = true
+		}
+	}
 	for _, c := range g.Carried {
 		if done[c.Module] || (len(g.Failing) > 0 && !slices.Contains(g.Failing, c.Module)) {
 			continue
@@ -356,6 +365,34 @@ func failCarried(ctx context.Context, open *stores, p *inventory.Plan, g *invent
 	p.Note = strings.Join(notes, "; ")
 }
 
+// keepPassing keeps a pass as the verdict of every build the gate carried that passed on its own when the
+// send failed (ADR 0254), except the plan's own module; answers the modules kept.
+func keepPassing(ctx context.Context, open *stores, p *inventory.Plan, g *inventory.PlanGate, except string) []string {
+	var kept []string
+	seen := map[string]bool{}
+	for _, c := range g.Carried {
+		if c.Module == except || c.Build == "" || seen[c.Build] || !slices.Contains(g.Passing, c.Module) {
+			continue
+		}
+		seen[c.Build] = true
+		why := fmt.Sprintf("healthy %d times on its own while the send failed: %s", g.Healthy[c.Module], g.Why)
+		if w := g.Waits[c.Module]; w != "" {
+			why += "; and it waits for a person: " + w
+		}
+		if err := open.inventory.RecordGate(ctx, inventory.GateVerdict{Build: c.Build, Module: c.Module, Commit: c.To,
+			Previous: c.From, Plan: p.ID, Machines: []string{c.Node}, Verdict: inventory.GatePassed, Why: why,
+			Component: coreComponent(c.Module), JudgingFrom: g.Since}); err != nil {
+			fmt.Printf("%s: %s passed its gate on %s on its own, and the verdict could not be kept: %v\n", p.ID, c.Module,
+				c.Node, err)
+			continue
+		}
+		if !slices.Contains(kept, c.Module) {
+			kept = append(kept, c.Module)
+		}
+	}
+	return kept
+}
+
 // sentTheBuild is every machine running a module that was last sent this build of it.
 func sentTheBuild(ctx context.Context, open *stores, module, commit string) ([]string, error) {
 	running, err := open.inventory.Running(ctx, module)

cmd/mesh-controller/replay318_test.go

@@ -0,0 +1,244 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"reflect"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/novox/mesh-controller/internal/catalogue"
+	"github.com/novox/mesh-controller/internal/conditions"
+	"github.com/novox/mesh-controller/internal/inventory"
+	"github.com/novox/mesh-controller/internal/lease"
+	"github.com/novox/mesh-controller/internal/link"
+)
+
+// novox/hq issue 318, replayed with only what the controller had before its fix, so it can be laid over the
+// older commit: the statements are read from the node-engine's JSON, as the controller hears them.
+//
+// 2026-10-08, 11:08 UTC: the walk of the builds waiting for a gate sent the laptop three moves in one send —
+// docker ba6058ab → c78b5fc9, node-tools 9a4140b4 → d4845f7c, openrazer 88135ad0 → c78b5fc9 — and judged
+// them together. openrazer's build put the operator's account in the group `openrazer` (ADR 0252); the
+// laptop said the account "relogin needed", and the daemon's unit failed in the account's own service
+// manager, which began before the group. The gate read "not yet healthy" for ten minutes, failed at its
+// bound, put openrazer back (which by ADR 0252 also took the group back), and the walk failed: docker and
+// node-tools, healthy at every judging, never reached the workstation, and every other walk was refused there.
+//
+// The rule's outcome: a wait for a person is not a failure. The send passes with the wait carried along,
+// nothing is put back, the walk goes on to the workstation, and openrazer's condition says, in one sentence
+// for the operator, that a new login is needed on the laptop.
+func TestReplay318(t *testing.T) {
+	open := aMesh(t)
+	ctx := t.Context()
+	inv := open.inventory
+	keeper, _ := withConditionsInMemory(t)
+	was := doctorFrom
+	doctorFrom = nil
+	t.Cleanup(func() { doctorFrom = was })
+
+	build := func(module, repository, commit string, asked time.Time) {
+		manifest, _ := json.Marshal(catalogue.Manifest{Module: module, Version: "1"})
+		if err := inv.RecordBuild(ctx, inventory.Build{ID: "build-" + module + "-" + commit, Module: module, Commit: commit,
+			Repository: repository, Path: "modules/" + module, Manifest: manifest, Asked: asked, At: asked,
+			Made: []inventory.Artifact{{Name: "x", Kind: "bundle", Reference: "sha256:" + module + "-" + commit}}}); err != nil {
+			t.Fatal(err)
+		}
+		if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: module, Version: "1"}, inventory.Source{
+			Repository: repository, Seat: "git", Path: "modules/" + module, BuiltFrom: commit, Head: commit,
+			Asked: asked}); err != nil {
+			t.Fatal(err)
+		}
+	}
+	type move struct{ module, repository, from, to string }
+	moves := []move{
+		{"docker", "novox/mesh-catalog", "ba6058ab", "c78b5fc9"},
+		{"node-tools", "novox/mesh-tools", "9a4140b4", "d4845f7c"},
+		{"openrazer", "novox/mesh-catalog", "88135ad0", "c78b5fc9"},
+	}
+	machines := []string{"laptop", "workstation"}
+	if _, err := inv.AddNode(ctx, "workstation"); err != nil {
+		t.Fatal(err)
+	}
+	old, recent := time.Now().Add(-3*time.Hour), time.Now().Add(-time.Minute)
+	for _, mv := range moves {
+		build(mv.module, mv.repository, mv.from, old)
+		for _, n := range machines {
+			if _, err := inv.Assign(ctx, n, mv.module); err != nil {
+				t.Fatal(err)
+			}
+		}
+	}
+	for _, n := range machines {
+		sent := map[string]string{}
+		for _, mv := range moves {
+			sent[mv.module] = mv.from
+		}
+		if err := inv.RecordSent(ctx, nodeID(t, open, n), "d-"+n, sent); err != nil {
+			t.Fatal(err)
+		}
+	}
+	for _, mv := range moves {
+		build(mv.module, mv.repository, mv.to, recent)
+	}
+
+	// What each machine says, as its node-engine words it: openrazer's account waits for a new login, and the
+	// daemon's unit failed in that account's own manager; docker's container is healthy.
+	statement := func(at time.Time) link.Health {
+		raw := fmt.Sprintf(`{"contract": %d, "at": %q, "resources": [
+			{"module": "docker", "resource": "docker.engine", "kind": "service", "target": "docker.service",
+			 "state": "healthy", "since": %q},
+			{"module": "openrazer", "resource": "openrazer.operator", "kind": "account", "target": "operator",
+			 "account": "operator", "state": "unhealthy", "since": %q, "streak": 3,
+			 "reason": "relogin needed: operator is in the group openrazer, and its running session began before it was; log out of every session and in again, or reboot"},
+			{"module": "openrazer", "resource": "openrazer.daemon", "kind": "unit", "target": "openrazer-daemon.service",
+			 "account": "operator", "state": "unhealthy", "since": %q, "streak": 3,
+			 "reason": "failed in the account's own service manager (exit-code)"}]}`,
+			link.ReadinessContract, at.Format(time.RFC3339Nano), at.Format(time.RFC3339Nano), at.Format(time.RFC3339Nano),
+			at.Format(time.RFC3339Nano))
+		var h link.Health
+		if err := json.Unmarshal([]byte(raw), &h); err != nil {
+			t.Fatal(err)
+		}
+		return h
+	}
+
+	wasMoves, wasHeard, wasSend, wasGather := machineMoves, releaseHeard, sendRollout, gatherGateFacts
+	wasSettle, wasEvery, wasBound := gateSettle, gateEvery, gateBound
+	t.Cleanup(func() {
+		machineMoves, releaseHeard, sendRollout, gatherGateFacts = wasMoves, wasHeard, wasSend, wasGather
+		gateSettle, gateEvery, gateBound = wasSettle, wasEvery, wasBound
+	})
+	machineMoves = func(ctx context.Context, open *stores, f moveFacts, node string, all bool) ([]inventory.CarriedMove, error) {
+		modules, err := open.inventory.Assigned(ctx, node)
+		if err != nil {
+			return nil, err
+		}
+		sent, known, err := open.inventory.SentBuilds(ctx, node)
+		if err != nil {
+			return nil, err
+		}
+		return f.moves(node, modules, sent, known, all), nil
+	}
+	releaseHeard = func(context.Context, *stores) (map[string]bool, error) {
+		return map[string]bool{"laptop": true, "workstation": true}, nil
+	}
+	var sends [][]string
+	n := 0
+	sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) {
+		sends = append(sends, append([]string(nil), names...))
+		current, err := open.inventory.CurrentBuilds(ctx)
+		if err != nil {
+			return nil, err
+		}
+		for _, node := range names {
+			modules, _ := open.inventory.Assigned(ctx, node)
+			carried := map[string]string{}
+			for _, m := range modules {
+				carried[m] = current[m].Commit
+			}
+			n++
+			digest := fmt.Sprintf("d-%s-%d", node, n)
+			if err := open.inventory.RecordSent(ctx, nodeID(t, open, node), digest, carried); err != nil {
+				return nil, err
+			}
+			if _, err := open.inventory.RecordDoing(ctx, nodeID(t, open, node), inventory.Doing{Node: node,
+				Outcome: inventory.OutcomeApplied, Declared: digest, Applied: 1, At: time.Now()}); err != nil {
+				return nil, err
+			}
+		}
+		return names, nil
+	}
+	// Every judging hears each machine's newest statement first, as the controller does between judgings:
+	// the account's wait and the failed unit are raised as openrazer's condition, after the send.
+	gatherGateFacts = func(ctx context.Context, open *stores, component string) (gateFacts, error) {
+		now := time.Now()
+		f := gateFacts{now: now, reports: map[string]inventory.Reported{}, engines: map[string]string{},
+			rolledBack: map[string][]lease.Rollback{}, served: map[string]served{}, health: map[string]inventory.NodeHealth{},
+			judged: true}
+		for _, m := range machines {
+			if err := stateHealth(ctx, open.inventory, keeper, m, statement(now), now); err != nil {
+				return f, err
+			}
+			f.served[m] = served{runtime: true, tools: map[string]bool{}}
+		}
+		reports, err := open.inventory.LastReports(ctx)
+		if err != nil {
+			return f, err
+		}
+		for _, r := range reports {
+			f.reports[r.Node] = r
+		}
+		if f.health, err = open.inventory.Healths(ctx); err != nil {
+			return f, err
+		}
+		f.open, f.openErr = keeper.Open(ctx)
+		return f, nil
+	}
+	// The bound is reached at the first judging that is not a pass: what happened at 11:21 happens at once.
+	gateSettle, gateEvery, gateBound = 0, 0, 0
+
+	for i := 0; i < 12; i++ {
+		advancePlans(ctx, open)
+		time.Sleep(5 * time.Millisecond)
+	}
+
+	var p inventory.Plan
+	plans, err := inv.RecentPlans(ctx, 10)
+	if err != nil {
+		t.Fatal(err)
+	}
+	for _, q := range plans {
+		if q.Release != nil {
+			p = q
+			break
+		}
+	}
+	if p.Release == nil {
+		t.Fatal("no walk of the builds waiting for a gate was opened")
+	}
+	if p.State != inventory.PlanDone || !reflect.DeepEqual(p.Release.Done, machines) {
+		t.Fatalf("the walk is %s on %v: %s; want it done on the laptop and then the workstation", p.State,
+			p.Release.Done, p.Note)
+	}
+	if !reflect.DeepEqual(sends, [][]string{{"laptop"}, {"workstation"}}) {
+		t.Fatalf("sent %v; want the laptop, then the workstation, and nothing put back", sends)
+	}
+	current, err := inv.CurrentBuilds(ctx)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if current["openrazer"].Commit != "c78b5fc9" {
+		t.Fatalf("openrazer is registered at %s: its build was put back for a wait for a person", current["openrazer"].Commit)
+	}
+	for _, mv := range moves {
+		v, found, err := inv.GateOf(ctx, "build-"+mv.module+"-"+mv.to)
+		if err != nil || !found || v.Verdict != inventory.GatePassed {
+			t.Fatalf("%s's build %s: verdict %+v (found %v, %v); want a pass", mv.module, mv.to, v, found, err)
+		}
+		if mv.module == "openrazer" && !strings.Contains(v.Why, "relogin needed on ") {
+			t.Errorf("openrazer's pass does not carry its wait for a person: %q", v.Why)
+		}
+	}
+	// What the operator reads: one condition per machine for openrazer, saying a new login is needed there,
+	// a warning for a person, and no fault of the build.
+	list, err := keeper.Open(ctx)
+	if err != nil {
+		t.Fatal(err)
+	}
+	var said []string
+	for _, c := range list {
+		said = append(said, c.Key+": "+c.Summary)
+		if c.Subject.ID == "openrazer.laptop" {
+			if !strings.HasPrefix(c.Summary, "relogin needed on laptop") || c.Severity == conditions.Urgent ||
+				c.Resolver != conditions.ResolverOperator {
+				t.Errorf("openrazer's condition on the laptop: %s %s %s %q", c.Key, c.Severity, c.Resolver, c.Summary)
+			}
+		}
+	}
+	if !strings.Contains(strings.Join(said, "\n"), "relogin needed on laptop") {
+		t.Errorf("nothing says a new login is needed on the laptop:\n%s", strings.Join(said, "\n"))
+	}
+}

internal/inventory/health.go

@@ -28,6 +28,9 @@ type ResourceHealth struct {
 	// (ADR 0240 Phase B, to-be 48 §6).
 	Check string `json:"check,omitempty"`
 	Needs string `json:"needs,omitempty"`
+	// Account is the account whose own service manager runs it, or the account a resource of kind account
+	// is (novox/hq ADR 0254).
+	Account string `json:"account,omitempty"`
 }
 
 // NodeHealth is a machine's newest statement, as kept.

internal/inventory/plans.go

@@ -143,6 +143,15 @@ type PlanGate struct {
 	Carried []CarriedMove `json:"carried,omitempty"`
 	// Failing names the modules the last judging found wanting.
 	Failing []string `json:"failing,omitempty"`
+	// Healthy counts, per module, the consecutive judgings that found it healthy on every machine judged,
+	// or waiting for a person (novox/hq ADR 0254): each module's passes, apart from the send's.
+	Healthy map[string]int `json:"healthy,omitempty"`
+	// Waits is, per module, the wait for a person the newest judging read (ADR 0254): carried along, never
+	// a reason to fail.
+	Waits map[string]string `json:"waits,omitempty"`
+	// Passing names the modules that passed on their own when the send failed (ADR 0254): each keeps its
+	// pass, and is not put back.
+	Passing []string `json:"passing,omitempty"`
 }
 
 // CarriedMove is one module's build moving on a machine with a gated send.

internal/link/protocol.go

@@ -441,6 +441,15 @@ type Health struct {
 // file it writes, a service whose lifecycle is the machine's — said unhealthy while it stays failed.
 const KindUnit = "unit"
 
+// KindAccount is an account a module puts in a group (novox/hq ADR 0252): healthy when the account's running
+// session has every group the module declares, or nobody is logged in; unhealthy otherwise, its reason
+// starting ReasonRelogin when only a new login is missing.
+const KindAccount = "account"
+
+// ReasonRelogin starts the reason of an account whose running session began before it was put in a group
+// (ADR 0252): the build did what it should, and a person has one step left (novox/hq ADR 0254).
+const ReasonRelogin = "relogin needed"
+
 // The states of a machine's service managers (issue 315).
 const (
 	UnitsRunning  = "running"
@@ -529,6 +538,10 @@ type ResourceHealth struct {
 	// alone; Needs is the provision the check exercises (to-be 48 §6).
 	Check string `json:"check,omitempty"`
 	Needs string `json:"needs,omitempty"`
+	// Account is the account whose own service manager runs it, for a unit or service in an account's
+	// manager, and the account itself for a resource of kind KindAccount (novox/hq ADR 0254). Empty from an
+	// engine older than that, and for anything the machine's own manager or runtime runs.
+	Account string `json:"account,omitempty"`
 }
 
 // HealthSaid is the health event's body: the machine and its statement. The machine is read from the