Reconcile a module from the machine's facts at engine start and at each edge of a declared-running resource going down (issue 1957, ADR 0568)

A reboot left the have running, so a unit that failed to start at boot was never started again and nothing said so. Baseline now asks the machine; the judge's look calls Agent.Reconcile once per down edge, leaving held and unwanted modules alone.

internal/agent/agent.go

@@ -118,6 +118,11 @@ type Env struct {
 	// Recorded says the engine holds a record of every resource of the composition, so it was applied and not removed
 	// since; nil says yes. A composition the kept declaration or a have says runs is applied again when it says no.
 	Recorded func(ctx context.Context, c delivery.Composed) bool
+	// Inactive says a resource the composition declares running is not active on the machine now (a unit stopped, a
+	// container not running), read from the machine and not from any record (ADR 0568 §1). A have or a kept declaration
+	// that says the module runs is believed at the agent's start only when this says no; nil says no, and an unreadable
+	// machine says no (unknown is not inactive).
+	Inactive func(ctx context.Context, c delivery.Composed) bool
 	// Revisions are a composition key's revisions the bucket keeps, oldest first; nil reads none. The composition a want
 	// names is taken from them when a newer one replaced it under the same key (ADR 0539 clause 1).
 	Revisions func(ctx context.Context, key string) ([]Update, error)
@@ -167,6 +172,8 @@ type Agent struct {
 	// composition is held out of the node's own apply until a composition of its own is handed (heldOut).
 	heldAlone map[string]bool
 	retry     chan []string
+	// reconcile carries the judge's edges: a resource found not active (ADR 0568 §2).
+	reconcile chan []string
 	// tries counts the transient failures in a row of each module; again carries a retry whose wait ran out, and
 	// transient asks for every transiently failed module at once. stopped is closed when Run returns.
 	tries     map[string]int
@@ -213,7 +220,7 @@ func New(env Env) *Agent {
 	}
 	return &Agent{env: env, wants: map[string]wantNow{}, comps: map[string]compNow{}, running: map[string]delivery.Have{}, applied: map[string]compNow{},
 		attempt: map[string]attempt{}, checked: map[string]bool{}, prior: map[string]bool{}, said: map[string]string{}, notComposed: map[string]notComposedMark{},
-		retry: make(chan []string, 8), tries: map[string]int{}, again: make(chan againAsk, 64),
+		retry: make(chan []string, 8), reconcile: make(chan []string, 64), tries: map[string]int{}, again: make(chan againAsk, 64),
 		transient: make(chan struct{}, 1), stopped: make(chan struct{})}
 }
 
@@ -245,6 +252,17 @@ func (a *Agent) Run(ctx context.Context) error {
 				dirty[m] = true
 			}
 			a.mu.Unlock()
+		case modules := <-a.reconcile:
+			a.mu.Lock()
+			for _, m := range modules {
+				if w := a.wants[m]; !w.Present || w.Want.Removed || a.attempt[m].Held {
+					continue
+				}
+				delete(a.running, m)
+				delete(a.attempt, m)
+				dirty[m] = true
+			}
+			a.mu.Unlock()
 		case ask := <-a.again:
 			// The wait of a transiently failed apply ran out: tried again from the keys it has, if nothing tried it since.
 			a.mu.Lock()
@@ -303,6 +321,16 @@ func (a *Agent) Retry(modules ...string) {
 	}
 }
 
+// Reconcile asks for the modules to be applied again because a resource each declares running was found not active, once
+// per edge (the judge's look, ADR 0568 §2). It is Retry, but a module whose apply is held (a failed backup) is left
+// alone, as is one the node wants no more (ADR 0568 §4). It never blocks.
+func (a *Agent) Reconcile(modules ...string) {
+	select {
+	case a.reconcile <- modules:
+	default:
+	}
+}
+
 // RetryTransient asks for every module whose last apply failed transiently to be applied again now: the engine says it
 // when it knows the bus answers again (after it wrote the bus's user list and the bus reloaded its accounts). It never
 // blocks.
@@ -1010,6 +1038,18 @@ func (a *Agent) baseline(ctx context.Context, module string, w wantNow, comp com
 			a.mu.Lock()
 			a.prior[module] = true
 			a.mu.Unlock()
+			if h.Phase == string(delivery.PhaseHeld) {
+				// Held for a failed backup, and the operator's answer is still to come: the hold is kept, nothing is
+				// backed up again or applied, and the next revision of the want is the answer (ADR 0568 §4).
+				a.mu.Lock()
+				a.attempt[module] = attempt{FP: w.Want.FP, Composed: w.Want.Composed, WantRev: w.Rev, CompRev: comp.Rev, Held: true,
+					AnsweredRev: w.Rev}
+				if h.Composed != "" {
+					a.running[module] = delivery.Have{FP: h.FP, Composed: h.Composed, Phase: delivery.PhaseRunning}
+				}
+				a.mu.Unlock()
+				return true
+			}
 			if h.Composed != "" {
 				// This node-engine wrote it: what it says is what runs, and the kept declaration says nothing more. A
 				// failed apply or a different composition is applied again.
@@ -1022,6 +1062,9 @@ func (a *Agent) baseline(ctx context.Context, module string, w wantNow, comp com
 				}
 				if h.FP == w.Want.FP && h.Composed == w.Want.Composed &&
 					(h.Phase == "" || h.Phase == string(delivery.PhaseRunning) || marked) {
+					if a.inactiveAtStart(ctx, module, comp) {
+						return false
+					}
 					a.mu.Lock()
 					a.running[module] = delivery.Have{FP: h.FP, Composed: h.Composed, Phase: delivery.PhaseRunning}
 					a.mu.Unlock()
@@ -1047,6 +1090,9 @@ func (a *Agent) baseline(ctx context.Context, module string, w wantNow, comp com
 	if !ok || !SameResources(kept, comp.C.Resources) {
 		return false
 	}
+	if a.inactiveAtStart(ctx, module, comp) {
+		return false
+	}
 	a.mu.Lock()
 	a.running[module] = delivery.Have{FP: w.Want.FP, Composed: w.Want.Composed, Phase: delivery.PhaseRunning}
 	a.mu.Unlock()
@@ -1054,6 +1100,20 @@ func (a *Agent) baseline(ctx context.Context, module string, w wantNow, comp com
 	return true
 }
 
+// inactiveAtStart says the machine shows a resource the module declares running not active, so what a have or the kept
+// declaration says runs is not believed and the module is applied once, as for a changed want (ADR 0568 §1).
+func (a *Agent) inactiveAtStart(ctx context.Context, module string, comp compNow) bool {
+	if a.env.Inactive == nil || !a.env.Inactive(ctx, comp.C) {
+		return false
+	}
+	a.mu.Lock()
+	a.prior[module] = true
+	a.sayOnce("inactive:"+module+":"+comp.Dig, fmt.Sprintf("%s is applied again: a resource it declares running is not "+
+		"active on this machine", module))
+	a.mu.Unlock()
+	return true
+}
+
 func (a *Agent) apply(ctx context.Context, module string, w wantNow, comp compNow, others []delivery.Composed) {
 	fp, dig := w.Want.FP, w.Want.Composed
 	a.mu.Lock()