Keep a replay from moving the mesh backwards, and tighten the queue's edges (review of hq ADR 0219)

A registered replay asked now outranked newer asks of its module, and a plan took any later outcome as its answer. A replay is now refused while the module is asked anywhere, a plan module asked under an id is answered by that id alone, and a rebuild of a commit asks what the module follows. An ask handed back after a restart no longer reads as dead; a cancel that meets a start is withdrawn; pause holds for an ask fetched as it lands; kill removes containers before and after the build ends and says whether its outcome went out; a holder may say only its own machine is paused.
2026-10-05 19:45:45 +02:00
parent 541603c15c
commit 9873b3bf13
13 changed files with 600 additions and 106 deletions
@@ -212,31 +212,53 @@ func TestReplayRefusesToRollAnOlderCommitOutUnlessToldTo(t *testing.T) {
 	newer := inventory.Build{ID: "build-new", Module: "a", Commit: "n3wc0mm1t", Asked: asked.Add(time.Hour)}
 	history := []inventory.Build{newer, old}

-	if err := replayRefusal(old, "a", history, false, false); err != nil {
+	if err := replayRefusal(old, "a", history, false, false, nil); err != nil {
 		t.Errorf("a dry run was refused: %v", err)
 	}
-	err := replayRefusal(old, "a", history, true, false)
+	err := replayRefusal(old, "a", history, true, false, nil)
 	if err == nil || !strings.Contains(err.Error(), "build-new") || !strings.Contains(err.Error(), "--older") {
 		t.Errorf("registering an older commit than the one registered was not refused: %v", err)
 	}
-	if err := replayRefusal(old, "a", history, true, true); err != nil {
+	if err := replayRefusal(old, "a", history, true, true, nil); err != nil {
 		t.Errorf("--register --older was refused: %v", err)
 	}
-	if err := replayRefusal(newer, "a", history, true, false); err != nil {
+	if err := replayRefusal(newer, "a", history, true, false, nil); err != nil {
 		t.Errorf("registering the newest build again was refused: %v", err)
 	}
 	// A newer build of the same commit, or one that failed, is not a newer version to roll back from.
 	same := inventory.Build{ID: "build-same", Module: "a", Commit: old.Commit, Asked: asked.Add(2 * time.Hour)}
 	broken := inventory.Build{ID: "build-broken", Module: "a", Failed: "no", Asked: asked.Add(3 * time.Hour)}
-	if err := replayRefusal(old, "a", []inventory.Build{broken, same, old}, true, false); err != nil {
+	if err := replayRefusal(old, "a", []inventory.Build{broken, same, old}, true, false, nil); err != nil {
 		t.Errorf("refused for a newer build of the same commit or a failed one: %v", err)
 	}
-	if err := replayRefusal(inventory.Build{ID: "build-x", Failed: "clone"}, "", nil, false, false); err == nil {
+	if err := replayRefusal(inventory.Build{ID: "build-x", Failed: "clone"}, "", nil, false, false, nil); err == nil {
 		t.Error("a build that recorded no commit was replayed")
 	}
-	if err := replayRefusal(old, "a", history, false, true); err == nil {
+	if err := replayRefusal(old, "a", history, false, true, nil); err == nil {
 		t.Error("--older without --register was taken")
 	}
+	// **Nothing newer outstanding**, --older or not: an ask of the module in the queue, or an open plan
+	// holding it unbuilt, would be replaced by a replay asked now (issue 219).
+	q := link.Queue{Asks: []link.QueuedAsk{
+		{ID: "build-queued", Repository: "https://forge.example/novox/a.git", State: link.AskWaiting},
+		{ID: "build-elsewhere", Repository: "https://forge.example/novox/b.git", State: link.AskWaiting},
+		{ID: "build-other-path", Repository: "https://forge.example/novox/a.git", Path: "sub", State: link.AskWaiting},
+	}}
+	plans := []inventory.Plan{
+		{ID: "plan-holds", State: inventory.PlanBuilding, Tiers: [][]string{{"x"}, {"a"}}, Modules: map[string]*inventory.PlanModule{}},
+		{ID: "plan-built", State: inventory.PlanRolling, Tiers: [][]string{{"a"}}, Modules: map[string]*inventory.PlanModule{"a": {State: "built"}}},
+		{ID: "plan-failed", State: inventory.PlanFailed, Tiers: [][]string{{"a"}}, Modules: map[string]*inventory.PlanModule{}},
+	}
+	outstanding := outstandingFor("a", "novox/a", "", q, plans)
+	if len(outstanding) != 2 || !strings.Contains(outstanding[0], "build-queued") || !strings.Contains(outstanding[1], "plan-holds") {
+		t.Fatalf("outstanding: %v", outstanding)
+	}
+	if err := replayRefusal(old, "a", history, true, true, outstanding); err == nil || !strings.Contains(err.Error(), "build-queued") {
+		t.Errorf("registered over an outstanding ask: %v", err)
+	}
+	if err := replayRefusal(old, "a", history, false, false, outstanding); err != nil {
+		t.Errorf("a dry run was refused for what is outstanding: %v", err)
+	}
 	if said := replaySaid(old, "a", "build-1", false); !strings.Contains(said, "dry run") || !strings.Contains(said, "--register") {
 		t.Errorf("a dry replay does not say what it is: %q", said)
 	}
@@ -266,6 +288,13 @@ func TestAPlanWaitingOnAPausedSeatSaysSoAndIsNotLate(t *testing.T) {
 		t.Errorf("a plan waiting on a paused seat counted late")
 	}

+	longAgo := now.Add(-25 * time.Hour)
+	forgotten := p
+	forgotten.Modules = map[string]*inventory.PlanModule{"a": {State: "asked", AskedAt: &longAgo}}
+	if line := planLineWith(forgotten, now, all); !strings.Contains(line, "PAUSED OVER A DAY") || strings.Contains(line, "LATE") {
+		t.Errorf("a plan paused over a day reads %q", line)
+	}
+
 	some := pauseOf([]string{"g14", "ace"}, map[string]link.HolderState{"ace": {Paused: true}})
 	if some.All || !reflect.DeepEqual(some.Nodes, []string{"ace"}) {
 		t.Fatalf("%+v", some)
@@ -644,3 +673,100 @@ func TestAPlanStoppedAtItsFirstMachineIsRetried(t *testing.T) {
 		t.Fatalf("asked %v", *asked)
 	}
 }
+
+// A plan module asked under an id is settled by that id's outcome alone: a replay or a rebuild beside
+// the plan, asked later, never answers it (novox/hq ADR 0219).
+func TestAPlanIsAnsweredOnlyByTheBuildItAskedFor(t *testing.T) {
+	open := aMesh(t)
+	ctx := t.Context()
+	asked := time.Now().UTC().Add(-time.Minute)
+	plan := inventory.Plan{ID: "plan-own", Repository: "novox/a", Commit: "c0ffee", Created: asked,
+		State: inventory.PlanBuilding, Tiers: [][]string{{"a"}},
+		Modules: map[string]*inventory.PlanModule{"a": {State: "asked", AskedAt: &asked, Build: "build-own"}}}
+	if err := open.inventory.SavePlan(ctx, plan); err != nil {
+		t.Fatal(err)
+	}
+	planBuilt(ctx, open, "a", "0ldc0mm1t", "", time.Now().UTC(), "build-replay")
+	p, _ := open.inventory.PlanByID(ctx, plan.ID)
+	if p.Modules["a"].State != "asked" {
+		t.Fatalf("a replay asked after the plan settled it: %+v", p.Modules["a"])
+	}
+	// From the records too: a later build of the module recorded is not the plan's.
+	recorded := map[string][]inventory.Build{"a": {{ID: "build-replay", Commit: "0ldc0mm1t", Asked: time.Now(), At: time.Now()}}}
+	if settleFromRecords(&p, p.Tiers[0], recorded, nil) {
+		t.Fatalf("the records settled it with another build: %+v", p.Modules["a"])
+	}
+	planBuilt(ctx, open, "a", "c0ffee", "", asked, "build-own")
+	if p, _ = open.inventory.PlanByID(ctx, plan.ID); p.Modules["a"].State != "built" || p.Modules["a"].Commit != "c0ffee" {
+		t.Fatalf("its own build did not settle it: %+v", p.Modules["a"])
+	}
+}
+
+// rebuild of a build made at a commit asks what the module follows now, never the commit.
+func TestARebuildOfACommitAsksWhatTheModuleFollows(t *testing.T) {
+	open := aMesh(t)
+	ctx := t.Context()
+	asked := asksRecorded(t)
+	twoTiers(t, open)
+	var refs []string
+	was := askABuild
+	askABuild = func(c context.Context, source buildSource, path, ref string) (string, error) {
+		refs = append(refs, ref)
+		return was(c, source, path, ref)
+	}
+	if err := open.inventory.RecordBuild(ctx, inventory.Build{ID: "build-at-commit", Repository: "novox/a",
+		Ref: "0123456789abcdef0123456789abcdef01234567", Module: "a", Commit: "0123456789abcdef0123456789abcdef01234567"}); err != nil {
+		t.Fatal(err)
+	}
+	if err := rebuildCommand(ctx, []string{"build-at-commit"}); err != nil {
+		t.Fatal(err)
+	}
+	if len(refs) != 1 || refs[0] != "main" || len(*asked) != 1 {
+		t.Fatalf("asked %v at %v", *asked, refs)
+	}
+}
+
+// An ask the worker counts out that no machine said it started is cancelled only if no start comes
+// while a holder looks: one that does withdraws the cancel, and kill is what ends it.
+func TestCancelOfAnAskNobodySaidIsWithdrawnWhenItStarts(t *testing.T) {
+	js, ask := aBuildQueue(t)
+	open := aMesh(t)
+	ctx := t.Context()
+	was := holderLooks
+	holderLooks = 300 * time.Millisecond
+	t.Cleanup(func() { holderLooks = was })
+	id := link.NewBuildID(time.Now())
+	ask(id, "https://forge.example/novox/a.git")
+	worker, _ := broker.HolderConsumerFor("", "", broker.DeclaredSeat{Name: link.TheBuildMachine, Accepts: []string{"build"}})
+	sub, err := js.Context().PullSubscribe(worker.Filters[0], worker.Name, nats.Bind(worker.Stream, worker.Name), nats.ManualAck())
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer func() { _ = sub.Unsubscribe() }()
+	if _, err := sub.Fetch(1, nats.MaxWait(3*time.Second)); err != nil {
+		t.Fatal(err)
+	}
+	q, err := link.ReadQueue(ctx, js, link.TheBuildMachine)
+	if err != nil {
+		t.Fatal(err)
+	}
+	a, _ := q.Find(id)
+	if a.State != link.AskInFlight || a.On != "" {
+		t.Fatalf("the taken ask reads %+v", a)
+	}
+	// The holder that took it says it started, while the cancel waits.
+	go func() {
+		time.Sleep(100 * time.Millisecond)
+		started, _ := json.Marshal(link.BuildStart{ID: id, On: "ace", At: time.Now().UTC().Format(time.RFC3339Nano)})
+		_, _ = js.Context().Publish(link.BuildStarted(), started)
+	}()
+	if _, err := cancelAsk(ctx, js, open, link.TheBuildMachine, a); err == nil || !strings.Contains(err.Error(), "started on ace") {
+		t.Fatalf("a build that started was cancelled: %v", err)
+	}
+	if cancelled, _ := link.IsCancelled(js.Conn(), link.TheBuildMachine, id); cancelled {
+		t.Error("the withdrawn cancel is still marked")
+	}
+	if _, found, _ := open.inventory.BuildByID(ctx, id); found {
+		t.Error("a withdrawn cancel recorded an outcome")
+	}
+}