Restart a service only after every file it names is written (hq issue 260)

A service was restarted where it was declared, so one declared ahead of a
file in its restart-on was restarted before that file existed (the
resolver's zones file, a fact appended after its module's resources), and
a later change to such a file was never acted on. Each service is now
applied right after the last resource it names under restart-on or
reload-on; nothing else moves.

internal/apply/apply.go

@@ -339,21 +339,20 @@ func ApplyMindingWindows(
 		}
 		orphans = append(orphans, orphan)
 	}
-	ordered := d.Resources
+	ordered := afterWhatTheyRead(d.Resources)
 	guardFirst := 0
 	if d.Adoption != nil {
-		ordered = nil
+		// Each part ordered on its own, so a guard service never leaves the part applied first.
+		var guard, rest []declaration.Resource
 		for _, r := range d.Resources {
 			if strings.HasPrefix(r.Identity(), guardPrefix) {
-				ordered = append(ordered, r)
-			}
-		}
-		guardFirst = len(ordered)
-		for _, r := range d.Resources {
-			if !strings.HasPrefix(r.Identity(), guardPrefix) {
-				ordered = append(ordered, r)
+				guard = append(guard, r)
+			} else {
+				rest = append(rest, r)
 			}
 		}
+		ordered = append(afterWhatTheyRead(guard), afterWhatTheyRead(rest)...)
+		guardFirst = len(guard)
 	}
 	orphansRemoved := false
 	removeOrphans := func() error {
@@ -775,6 +774,69 @@ func ApplyMindingWindows(
 	return report, known, nil
 }
 
+// afterWhatTheyRead is the resources in declared order, except that **a service comes after every
+// resource it names under restart-on or reload-on** (novox/hq issue 260).
+//
+// A service is started, restarted or reloaded where it is reached, and what it reads must be on the
+// machine by then. Declared ahead of one of its files, it was restarted for the file before it while
+// the other did not exist yet — the resolver, restarted on its configuration before the zones file
+// that configuration names was written, failed its first start on every machine. And a change to a
+// file applied after its service was never acted on at all: what moved is only known within one
+// apply, and the service had already been passed.
+//
+// Only a service moves, and only as far as the last of what it names; everything else keeps its
+// declared place. A name not in the list is ignored, as it is when the restart is decided. Only
+// what is declared after a service is waited for, so no ring can form and nothing is left out.
+func afterWhatTheyRead(resources []declaration.Resource) []declaration.Resource {
+	at := map[string]int{}
+	for i, r := range resources {
+		at[r.Identity()] = i
+	}
+	// What each service still waits for: the resources it names that come after it.
+	waits := map[int]map[string]bool{}
+	for i, r := range resources {
+		svc, ok := r.(*declaration.Service)
+		if !ok {
+			continue
+		}
+		for _, id := range append(append([]string{}, svc.RestartOn...), svc.ReloadOn...) {
+			if j, declared := at[id]; declared && j > i {
+				if waits[i] == nil {
+					waits[i] = map[string]bool{}
+				}
+				waits[i][id] = true
+			}
+		}
+	}
+	if len(waits) == 0 {
+		return resources
+	}
+	ordered := make([]declaration.Resource, 0, len(resources))
+	placed := map[int]bool{}
+	var place func(i int)
+	place = func(i int) {
+		placed[i] = true
+		ordered = append(ordered, resources[i])
+		id := resources[i].Identity()
+		// Whatever was waiting only for this comes now, in declared order.
+		for j := range resources {
+			if w := waits[j]; w != nil && w[id] {
+				delete(w, id)
+				if len(w) == 0 && !placed[j] {
+					delete(waits, j)
+					place(j)
+				}
+			}
+		}
+	}
+	for i := range resources {
+		if !placed[i] && len(waits[i]) == 0 {
+			place(i)
+		}
+	}
+	return ordered
+}
+
 // guardPrefix is the ids of the mesh's guard on an adopted node: its package, table, unit and
 // service (novox/hq ADR 0100).
 const guardPrefix = declaration.AdoptionPrefix + "guard"