Reconcile a module from the machine's facts at engine start and at each edge of a declared-running resource going down (issue 1957, ADR 0568)

A reboot left the have running, so a unit that failed to start at boot was never started again and nothing said so. Baseline now asks the machine; the judge's look calls Agent.Reconcile once per down edge, leaving held and unwanted modules alone.

cmd/mesh-host/main.go

@@ -1046,15 +1046,7 @@ func runLink(ctx context.Context, opts options) error {
 			say(err.Error())
 		}
 		windows := apply.WindowsIn(filepath.Dir(opts.state))
-		j.HeldNow = func(now time.Time) map[string]bool {
-			held := map[string]bool{}
-			for _, w := range windows.Open(now) {
-				for _, name := range w.Holds {
-					held[name] = true
-				}
-			}
-			return held
-		}
+		j.HeldNow = heldBy(windows)
 		judging = j
 		// And which units its service managers say failed, and whose each is (novox/hq issue 315).
 		unitJudge = units.New(units.Exec{Run: apply.ExecRunner, System: builtFor})
@@ -1173,6 +1165,10 @@ func runLink(ctx context.Context, opts options) error {
 			if ag := current.get(); ag != nil {
 				ag.Retry(modules...)
 			}
+		}, func(modules ...string) {
+			if ag := current.get(); ag != nil {
+				ag.Reconcile(modules...)
+			}
 		})
 	}
 	// The node follows its wants: one module at a time from the composition its want names (novox/hq ADR 0539), and the
@@ -1336,8 +1332,8 @@ const (
 // the bus when a state changed, again every minute while one is not healthy, and every five minutes
 // anyway. A statement the bus did not take is said at the next look. It reads; it never acts (ADR 0240
 // rule 6) — but for going on with a take act stopped at its bound (ADR 0328 §7), which runs beside it.
-func judgeWhatRuns(ctx context.Context, j *liveness.Judge, queue *link.Session, say link.Announce, reapply func(modules ...string)) {
-	var missingBefore map[string]bool
+func judgeWhatRuns(ctx context.Context, j *liveness.Judge, queue *link.Session, say link.Announce, reapply, reconcile func(modules ...string)) {
+	var missingBefore, inactiveBefore map[string]bool
 	ticker := time.NewTicker(liveness.LookEvery)
 	defer ticker.Stop()
 	var lastSaid time.Time
@@ -1439,6 +1435,14 @@ func judgeWhatRuns(ctx context.Context, j *liveness.Judge, queue *link.Session,
 				say(fmt.Sprintf("applying %s again: a resource it declares is not on this machine", strings.Join(m, ", ")))
 				reapply(m...)
 			}
+			// A resource its module declares running that is down but still known to the manager is started by the same
+			// apply, once each time it goes down, never on a clock (ADR 0568 §2): the judge reads, the agent acts.
+			var inactive []string
+			inactive, inactiveBefore = newlyInactive(inactiveBefore, st)
+			if len(inactive) > 0 && reconcile != nil {
+				say(fmt.Sprintf("applying %s again: a resource it declares running is not active", strings.Join(inactive, ", ")))
+				reconcile(inactive...)
+			}
 		}
 		if queue.SayHealth(ctx, *withAccesses(withFoundDirectories(withAccounts(withUnits(healthAsReported(st, netSt),
 			unitSt), accountSt), found), accessSt)) {
@@ -2134,6 +2138,32 @@ func newlyMissing(last map[string]bool, st liveness.Statement) ([]string, map[st
 	return missingModules(fresh), now
 }
 
+// newlyInactive are the modules of the resources a statement says down (not running after their grace, but still known to
+// their manager, and not restarting) that the last statement acted on did not, each once, sorted, and the resources down
+// now: a resource still down is not acted on again, and one that came back and went down again is (ADR 0568 §2).
+func newlyInactive(last map[string]bool, st liveness.Statement) ([]string, map[string]bool) {
+	now := map[string]bool{}
+	var fresh liveness.Statement
+	for _, r := range st.Resources {
+		if r.State != liveness.Unhealthy || r.Reason != liveness.ReasonDown || r.Missing {
+			continue
+		}
+		now[r.ID] = true
+		if !last[r.ID] {
+			fresh.Resources = append(fresh.Resources, r)
+		}
+	}
+	return missingModules(withMissing(fresh)), now
+}
+
+// withMissing is the statement with each of its resources marked missing, so missingModules lists their modules.
+func withMissing(st liveness.Statement) liveness.Statement {
+	for i := range st.Resources {
+		st.Resources[i].Missing = true
+	}
+	return st
+}
+
 // recordKeys reads the public halves of the node's keys and the certificate the mesh issued for its internal name from
 // its state directory (novox/hq ADR 0555 §3): what the node's record says of them. A key that cannot be read is left
 // empty, never invented; the certificate is there only after the first declaration that carried it.