Give each pending assignment its own condition, and keep a raised row until it clears

Second review of #150: one key per machine and module let a newer failure
be cleared in the tick that raised it, pruning could orphan an open
condition, and a build no longer waited for read as never asked although it
may still run.

cmd/mesh-controller/pending.go

@@ -281,8 +281,11 @@ func whyNotBuilt(r inventory.RequestOutcome, now time.Time) string {
 		return fmt.Sprintf("%s; the merge that added it (%s) could not ask for its build: %s", where,
 			short(r.Commit), firstLine(r.NotAsked))
 	case r.NotAsked != "":
-		return fmt.Sprintf("%s; build %s, asked by %s, was not handed over or not waited for: %s", where, r.ID,
+		return fmt.Sprintf("%s; build %s, asked by %s, was not handed over: %s", where, r.ID,
 			askerOr(r.For), firstLine(r.NotAsked))
+	case r.OutcomeUnknown != "":
+		return fmt.Sprintf("%s; build %s was asked at %s, its asker stopped waiting (%s), and no outcome has "+
+			"been heard in %s", where, r.ID, clock(r.At), firstLine(r.OutcomeUnknown), now.Sub(r.At).Round(time.Minute))
 	default:
 		return fmt.Sprintf("%s; build %s was asked at %s, and no outcome has been heard in %s", where, r.ID,
 			clock(r.At), now.Sub(r.At).Round(time.Minute))
@@ -545,7 +548,7 @@ func settlePending(ctx context.Context, open *stores, now time.Time) []string {
 		switch {
 		case r == nil && now.Sub(p.Since) > buildRequestBound:
 			why = fmt.Sprintf("build %s is no longer on record", p.Build)
-		case r != nil && !stillComing(*r, now) && (r.Heard || r.NotAsked != ""):
+		case r != nil && !stillComing(*r, now) && (r.Heard || r.NotAsked != "" || r.OutcomeUnknown != ""):
 			why = whyNotBuilt(*r, now)
 		case r != nil && !stillComing(*r, now):
 			why = fmt.Sprintf("no outcome of build %s was heard within %s of asking", p.Build, buildRequestBound)
@@ -576,9 +579,10 @@ func containsString(xs []string, x string) bool {
 	return false
 }
 
-// pendingObservation is the condition for a pending assignment that was not made.
+// pendingObservation is the condition for a pending assignment that was not made: one per pending
+// assignment, its row's id the last part of its id, so one row's condition is never another's.
 func pendingObservation(p inventory.PendingAssignment) conditions.Observation {
-	return conditions.Observation{Scope: conditions.ScopeMachine, ID: p.Node + "." + p.Module,
+	return conditions.Observation{Scope: conditions.ScopeMachine, ID: fmt.Sprintf("%s.%s.%d", p.Node, p.Module, p.ID),
 		Token: kindPendingEnded, Kind: kindPendingEnded, Machine: p.Node, Severity: conditions.Warning,
 		Resolver: conditions.ResolverOperator, Source: "pending assignments",
 		Summary: p.Note}
@@ -628,7 +632,7 @@ func raisePendingEnded(ctx context.Context, inv *inventory.Inventory) []string {
 			said = append(said, fmt.Sprintf("the condition for %s on %s was cleared and not recorded: %v", p.Module, p.Node, err))
 		}
 	}
-	return said
+	return append(said, clearOrphaned(ctx, keeper, inv)...)
 }
 
 // pendingAnswered says whether a pending assignment that was not made has been answered since, and how.
@@ -647,14 +651,60 @@ func pendingAnswered(ctx context.Context, inv *inventory.Inventory, p inventory.
 	if err != nil {
 		return "", false, err
 	}
+	// A newer pending assignment answers it only while it is open or once it was made: one that itself
+	// ended unmade is its own condition, and answers nothing.
 	for _, r := range rows {
-		if r.ID != p.ID && r.Since.After(p.Since) {
+		if r.ID != p.ID && r.Since.After(p.Since) && (r.Open() || r.State == inventory.PendingApplied) {
 			return "assigned again, pending on another build", true, nil
 		}
 	}
 	return "", false, nil
 }
 
+// clearOrphaned clears every open condition of an assignment not made whose pending assignment is no longer
+// on record: its machine was removed, which takes its pending assignments with it.
+func clearOrphaned(ctx context.Context, keeper *conditions.Keeper, inv *inventory.Inventory) []string {
+	open, err := keeper.Open(ctx)
+	if err != nil {
+		return []string{fmt.Sprintf("the open conditions could not be read to clear orphaned ones: %v", err)}
+	}
+	byID := map[int64]string{}
+	var ids []int64
+	for _, c := range open {
+		if c.Kind != kindPendingEnded {
+			continue
+		}
+		// `<scope>.<node>.<module>.<row>.<kind>`: the row is the part before the kind.
+		parts := strings.Split(c.Key, ".")
+		if len(parts) < 2 {
+			continue
+		}
+		var id int64
+		if _, err := fmt.Sscan(parts[len(parts)-2], &id); err != nil {
+			continue
+		}
+		byID[id] = c.Key
+		ids = append(ids, id)
+	}
+	if len(ids) == 0 {
+		return nil
+	}
+	known, err := inv.PendingKnown(ctx, ids)
+	if err != nil {
+		return []string{fmt.Sprintf("the pending assignments could not be read to clear orphaned conditions: %v", err)}
+	}
+	var said []string
+	for id, key := range byID {
+		if known[id] {
+			continue
+		}
+		if _, err := keeper.Clear(ctx, key, "its pending assignment is no longer on record: its machine was removed"); err != nil {
+			said = append(said, fmt.Sprintf("the condition %s could not be cleared: %v", key, err))
+		}
+	}
+	return said
+}
+
 // withdrawPending is `unassign` of a module only pending on a machine, under the machine's hold: a waiting
 // pending assignment is withdrawn; one being made now is refused, never overridden; one that expired or was
 // refused is taken back, which answers its condition. False when there is none.
@@ -663,6 +713,7 @@ func withdrawPending(ctx context.Context, inv *inventory.Inventory, node, module
 	if err != nil {
 		return "", false, err
 	}
+	var takenBack []string
 	for _, p := range rows {
 		switch p.State {
 		case inventory.PendingApplying:
@@ -687,9 +738,12 @@ func withdrawPending(ctx context.Context, inv *inventory.Inventory, node, module
 			if err := inv.MarkPending(ctx, p.ID, "acknowledged"); err != nil {
 				return "", false, err
 			}
-			return fmt.Sprintf("the pending assignment of %s to %s, which %s, is taken back; its condition clears",
-				module, node, p.State), true, nil
+			takenBack = append(takenBack, fmt.Sprintf("build %s (%s)", p.Build, p.State))
 		}
 	}
+	if len(takenBack) > 0 {
+		return fmt.Sprintf("the pending assignment(s) of %s to %s that were not made are taken back: %s; their "+
+			"conditions clear", module, node, strings.Join(takenBack, ", ")), true, nil
+	}
 	return "", false, nil
 }