Give each pending assignment its own condition, and keep a raised row until it clears

Second review of #150: one key per machine and module let a newer failure
be cleared in the tick that raised it, pruning could orphan an open
condition, and a build no longer waited for read as never asked although it
may still run.

cmd/mesh-controller/pending_test.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"encoding/json"
 	"errors"
+	"fmt"
 	"slices"
 	"strings"
 	"testing"
@@ -15,6 +16,39 @@ import (
 	"github.com/novox/mesh-controller/internal/link"
 )
 
+// failedBuild settles the merge's build of modules/sensors as failed.
+func failedBuild(t *testing.T, open *stores, id string) {
+	t.Helper()
+	if err := (builds{open.inventory, open}).Built(t.Context(), outcome(id, "modules/sensors", "3da80a4b00aa",
+		"", "boom")); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func conditionOpen(t *testing.T, p inventory.PendingAssignment) bool {
+	t.Helper()
+	_, found, err := conditionsFrom.Get(t.Context(), pendingObservation(p).Key())
+	if err != nil {
+		t.Fatal(err)
+	}
+	return found
+}
+
+func rowOf(t *testing.T, open *stores, id int64) inventory.PendingAssignment {
+	t.Helper()
+	rows, err := open.inventory.Pending(t.Context(), time.Unix(0, 0))
+	if err != nil {
+		t.Fatal(err)
+	}
+	for _, r := range rows {
+		if r.ID == id {
+			return r
+		}
+	}
+	t.Fatalf("pending assignment %d is not on record", id)
+	return inventory.PendingAssignment{}
+}
+
 // novox/hq issue 325: an assignment of a module the catalogue does not hold says which case it is in — a
 // build in flight (kept pending), known and not built (said, with build), or unknown (refused, with the
 // closest names) — and a pending assignment is made when its build registers the module, or ends with why.
@@ -370,7 +404,7 @@ func TestAFailedAskIsNotABuildInFlight(t *testing.T) {
 		t.Fatal(err)
 	}
 	_, err = assign(ctx, open, "laptop", "gauges")
-	if err == nil || !strings.Contains(err.Error(), "build build-waited, asked by build, was not handed over or not waited for") {
+	if err == nil || !strings.Contains(err.Error(), "build build-waited, asked by build, was not handed over") {
 		t.Fatalf("a build not handed over: %v", err)
 	}
 	// Heard first, the outcome stands: marking it afterwards changes nothing.
@@ -602,7 +636,8 @@ func TestAPendingAssignmentNeverWaitsSilently(t *testing.T) {
 		t.Fatalf("a module registered meanwhile: %+v", p)
 	}
 
-	// Ended rows are deleted after KeptFor (review point 7); open ones never.
+	// Ended rows are deleted after KeptFor (review point 7); open ones never, nor one whose condition is
+	// still open (second review, bug B).
 	if _, err := inv.RecordPending(ctx, inventory.PendingAssignment{Node: "anchor", Module: "lost", Build: "build-lost"}); err != nil {
 		t.Fatal(err)
 	}
@@ -612,10 +647,13 @@ func TestAPendingAssignmentNeverWaitsSilently(t *testing.T) {
 		t.Fatal(err)
 	}
 	for _, r := range rows {
-		if !r.Open() {
-			t.Fatalf("an ended pending assignment outlived %s: %+v (%v)", inventory.KeptFor, r, said)
+		if !r.Open() && (r.Raised == nil || r.Cleared != nil) {
+			t.Fatalf("an ended pending assignment with no open condition outlived %s: %+v (%v)", inventory.KeptFor, r, said)
 		}
 	}
+	if !strings.Contains(strings.Join(said, "\n"), "deleted 1 pending assignment(s)") {
+		t.Fatalf("the applied row was not deleted: %v", said)
+	}
 }
 
 // The seat's assign passes build on; unassign takes none.
@@ -641,3 +679,186 @@ func TestAnAssignmentNotMadeIsSaidPlainly(t *testing.T) {
 		t.Fatalf("headline %q", w.Headline)
 	}
 }
+
+// **One row's condition is never another's** (second review, bug A). The sequence: the first pending
+// assignment's build fails and the tick raises it; the person assigns again with build "true", a second
+// pending assignment; its build fails too before the next tick. That tick raises the second, and must not
+// clear it by judging the first answered by the second: each row has its own condition, and a newer row that
+// itself ended unmade answers nothing.
+func TestASecondAssignmentNotMadeKeepsItsCondition(t *testing.T) {
+	open := aCatalogueMesh(t)
+	ctx := t.Context()
+	asked := 0
+	was := askABuild
+	askABuild = func(context.Context, buildSource, string, string) (string, error) {
+		asked++
+		return fmt.Sprintf("b-%d", asked), nil
+	}
+	t.Cleanup(func() { askABuild = was })
+	mergeAdding(t, open, "modules/sensors", "3da80a4b00aa")
+	if _, err := assign(ctx, open, "laptop", "sensors"); err != nil {
+		t.Fatal(err)
+	}
+	failedBuild(t, open, "b-1")
+	settlePending(ctx, open, time.Now())
+	first := pendingOf(t, open)[0]
+	if !conditionOpen(t, first) {
+		t.Fatal("the first assignment not made raised nothing")
+	}
+
+	if _, err := assignWith(ctx, open, "laptop", assignOptions{Build: true}, "sensors"); err != nil {
+		t.Fatal(err)
+	}
+	failedBuild(t, open, "b-2")
+	settlePending(ctx, open, time.Now())
+
+	var second inventory.PendingAssignment
+	for _, p := range pendingOf(t, open) {
+		if p.Build == "b-2" {
+			second = p
+		}
+	}
+	if second.State != inventory.PendingExpired {
+		t.Fatalf("the second: %+v", second)
+	}
+	if !conditionOpen(t, second) {
+		t.Fatal("the second assignment's condition was cleared in the tick that raised it")
+	}
+	if !conditionOpen(t, rowOf(t, open, first.ID)) {
+		t.Fatal("the first was judged answered by a second that itself was not made")
+	}
+	// unassign takes both back, and the next tick clears both.
+	if said, err := unassign(ctx, open, "laptop", "sensors"); err != nil || !strings.Contains(said, "b-1") ||
+		!strings.Contains(said, "b-2") {
+		t.Fatalf("unassign: %q %v", said, err)
+	}
+	settlePending(ctx, open, time.Now())
+	if conditionOpen(t, first) || conditionOpen(t, second) {
+		t.Fatal("a condition stayed open after both were taken back")
+	}
+}
+
+// **A raised row outlives the pruning until its condition clears** (second review, bug B); and a machine's
+// removal, which takes its rows with it, clears their conditions at the next tick.
+func TestARaisedRowIsKeptUntilItsConditionClears(t *testing.T) {
+	open := aCatalogueMesh(t)
+	ctx := t.Context()
+	inv := open.inventory
+	asksWithPaths(t)
+	mergeAdding(t, open, "modules/sensors", "3da80a4b00aa")
+	if _, err := assign(ctx, open, "laptop", "sensors"); err != nil {
+		t.Fatal(err)
+	}
+	failedBuild(t, open, "b-modules/sensors")
+	settlePending(ctx, open, time.Now())
+	row := pendingOf(t, open)[0]
+
+	settlePending(ctx, open, time.Now().Add(inventory.KeptFor+time.Hour))
+	if rowOf(t, open, row.ID).Raised == nil || !conditionOpen(t, row) {
+		t.Fatal("a raised row was pruned, or its condition closed, while the condition was open")
+	}
+	if _, err := unassign(ctx, open, "laptop", "sensors"); err != nil {
+		t.Fatal(err)
+	}
+	settlePending(ctx, open, time.Now())
+	settlePending(ctx, open, time.Now().Add(inventory.KeptFor+time.Hour))
+	if known, err := inv.PendingKnown(ctx, []int64{row.ID}); err != nil || known[row.ID] {
+		t.Fatalf("a cleared row outlived %s: %v", inventory.KeptFor, err)
+	}
+
+	// On anchor, then anchor removed: the row goes, and its condition with it at the next tick.
+	if _, err := assign(ctx, open, "anchor", "sensors"); err == nil {
+		t.Fatal("known and not built was not refused")
+	}
+	q, err := inv.RecordPending(ctx, inventory.PendingAssignment{Node: "anchor", Module: "sensors", Build: "b-x"})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if _, err := inv.SettlePending(ctx, q.ID, inventory.PendingWaiting, inventory.PendingExpired, "boom"); err != nil {
+		t.Fatal(err)
+	}
+	settlePending(ctx, open, time.Now())
+	q.Node = "anchor"
+	if !conditionOpen(t, q) {
+		t.Fatal("not raised")
+	}
+	if _, err := inv.RemoveNodeForTest(ctx, "anchor"); err != nil {
+		t.Fatal(err)
+	}
+	settlePending(ctx, open, time.Now())
+	if conditionOpen(t, q) {
+		t.Fatal("a removed machine's condition stayed open")
+	}
+}
+
+// **A claim left by a controller that stopped** (second review): the tick settles it from what the mesh
+// holds — back to waiting when the module is not assigned there, applied when it is — and leaves a fresh
+// claim alone.
+func TestAStaleClaimIsSettledFromWhatTheMeshHolds(t *testing.T) {
+	open := aCatalogueMesh(t)
+	ctx := t.Context()
+	inv := open.inventory
+	claimed := func(node string) inventory.PendingAssignment {
+		p, err := inv.RecordPending(ctx, inventory.PendingAssignment{Node: node, Module: "sensors", Build: "b-x"})
+		if err != nil {
+			t.Fatal(err)
+		}
+		if ok, err := inv.ClaimPending(ctx, p.ID); err != nil || !ok {
+			t.Fatalf("claim: %v %v", ok, err)
+		}
+		return p
+	}
+	notAssigned, assigned := claimed("laptop"), claimed("anchor")
+	register(t, open, catalogue.Manifest{Module: "unrelated", Version: "1"})
+	if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "sensors", Version: "1"}, inventory.Source{}); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := inv.Assign(ctx, "anchor", "sensors"); err != nil {
+		t.Fatal(err)
+	}
+
+	settlePending(ctx, open, time.Now())
+	if rowOf(t, open, notAssigned.ID).State != inventory.PendingApplying || rowOf(t, open, assigned.ID).State != inventory.PendingApplying {
+		t.Fatal("a fresh claim was settled")
+	}
+	settlePending(ctx, open, time.Now().Add(claimStaleAfter+time.Minute))
+	if got := rowOf(t, open, assigned.ID); got.State != inventory.PendingApplied {
+		t.Fatalf("a stale claim of an assigned module: %+v", got)
+	}
+	// Released to waiting, and in the same tick made, since sensors is registered now.
+	if got := rowOf(t, open, notAssigned.ID); got.State != inventory.PendingApplied ||
+		!slices.Contains(assignedTo(t, open, "laptop"), "sensors") {
+		t.Fatalf("a stale claim of a module not assigned: %+v", got)
+	}
+}
+
+// **A waited build whose asker stopped waiting is asked, outcome unknown** (second review): it may still run,
+// so it reads as in flight until its outcome or its bound, never as not asked; a build never handed over
+// is not asked.
+func TestABuildNoLongerWaitedForIsStillInFlight(t *testing.T) {
+	open := aCatalogueMesh(t)
+	ctx := t.Context()
+	inv := open.inventory
+	for _, id := range []string{"build-timeout", "build-nothandedover"} {
+		dir := "modules/" + strings.TrimPrefix(id, "build-")
+		if err := inv.RecordBuildRequest(ctx, inventory.BuildRequest{ID: id, Repository: "novox/mesh-catalog",
+			Seat: "git", Path: dir, Ref: "main", For: "build"}); err != nil {
+			t.Fatal(err)
+		}
+	}
+	markWaitFailed(ctx, "build-timeout", errors.New("no build machine answered within 10m0s"))
+	markWaitFailed(ctx, "build-nothandedover", fmt.Errorf("%w: cannot submit a build: nats: timeout", link.ErrNotHandedOver))
+
+	said, err := assign(ctx, open, "laptop", "timeout")
+	if err != nil || !strings.Contains(said, "being built") || !strings.Contains(said, "kept as pending") {
+		t.Fatalf("a build no longer waited for: %q %v", said, err)
+	}
+	if _, err := assign(ctx, open, "laptop", "nothandedover"); err == nil || !strings.Contains(err.Error(), "was not handed over") {
+		t.Fatalf("a build never handed over: %v", err)
+	}
+	settlePending(ctx, open, time.Now().Add(buildRequestBound+time.Minute))
+	rows, _ := inv.PendingFor(ctx, "laptop", "timeout")
+	if len(rows) != 1 || rows[0].State != inventory.PendingExpired || !strings.Contains(rows[0].Note, "its asker stopped waiting") {
+		t.Fatalf("past the bound: %+v", rows)
+	}
+}