Apply a module again when the engine holds no record of its resources or its health says one is missing, and move a module with nothing to back up without a backup (issue 1944, issue 1945)

cmd/mesh-host/main.go

@@ -1166,7 +1166,11 @@ func runLink(ctx context.Context, opts options) error {
 		netJudge.set(network.New(network.Machine{Run: apply.ExecRunner, Linked: queue.Linked},
 			hostOf(mine.Membership.Broker)))
 		go judging.Probe(aside)
-		go judgeWhatRuns(aside, judging, queue, say)
+		go judgeWhatRuns(aside, judging, queue, say, func(modules ...string) {
+			if ag := current.get(); ag != nil {
+				ag.Retry(modules...)
+			}
+		})
 	}
 	// The node follows its wants: one module at a time from the composition its want names (novox/hq ADR 0539), and the
 	// busy containers are asked for again soon (ADR 0360 §11).
@@ -1329,7 +1333,7 @@ const (
 // the bus when a state changed, again every minute while one is not healthy, and every five minutes
 // anyway. A statement the bus did not take is said at the next look. It reads; it never acts (ADR 0240
 // rule 6) — but for going on with a take act stopped at its bound (ADR 0328 §7), which runs beside it.
-func judgeWhatRuns(ctx context.Context, j *liveness.Judge, queue *link.Session, say link.Announce) {
+func judgeWhatRuns(ctx context.Context, j *liveness.Judge, queue *link.Session, say link.Announce, reapply func(modules ...string)) {
 	ticker := time.NewTicker(liveness.LookEvery)
 	defer ticker.Stop()
 	var lastSaid time.Time
@@ -1415,6 +1419,12 @@ func judgeWhatRuns(ctx context.Context, j *liveness.Judge, queue *link.Session,
 					say(unhealthyWords(r))
 				}
 			}
+			// A resource the machine does not have at all is made again: its module's composition applied anew (issue
+			// 1945), once each time the statement changes, never on a clock.
+			if m := missingModules(st); len(m) > 0 && reapply != nil {
+				say(fmt.Sprintf("applying %s again: a resource it declares is not on this machine", strings.Join(m, ", ")))
+				reapply(m...)
+			}
 		}
 		if queue.SayHealth(ctx, *withAccesses(withFoundDirectories(withAccounts(withUnits(healthAsReported(st, netSt),
 			unitSt), accountSt), found), accessSt)) {
@@ -2077,3 +2087,17 @@ func untakenModules(d *declaration.Declaration) map[string][]string {
 	}
 	return d.Adoption.Untaken
 }
+
+// missingModules are the modules of the resources a statement says missing, each once, sorted.
+func missingModules(st liveness.Statement) []string {
+	seen := map[string]bool{}
+	var out []string
+	for _, r := range st.Resources {
+		if r.Missing && r.Module != "" && !seen[r.Module] {
+			seen[r.Module] = true
+			out = append(out, r.Module)
+		}
+	}
+	sort.Strings(out)
+	return out
+}