package link import ( "context" "encoding/json" "testing" "time" "github.com/nats-io/nats.go" "github.com/novox/mesh-controller/internal/broker" ) // The build queue read and changed by hand (novox/hq ADR 0219). // Which ask is waiting, in flight and dead, from the stream, the worker and the seat's events. func TestTheQueueTellsWaitingInFlightAndDeadApart(t *testing.T) { at := func(m int) string { return time.Date(2026, 10, 5, 12, m, 0, 0, time.UTC).Format(time.RFC3339Nano) } asks := []QueuedAsk{ {Seq: 1, ID: "dead-1"}, // handed out five times, its machine moved on: dead {Seq: 2, ID: "running-g"}, // g14's latest start, no outcome: in flight {Seq: 3, ID: "running-a"}, // ace's latest start, no outcome: in flight {Seq: 4, ID: "unsaid"}, // taken, not yet said: in flight while the worker counts it {Seq: 5, ID: "waiting-1"}, // after the worker's last delivery {Seq: 6, ID: "waiting-2"}, } heard := BuildHeard{ Started: map[string]BuildStart{ "dead-1": {ID: "dead-1", On: "g14", At: at(1)}, "running-g": {ID: "running-g", On: "g14", At: at(5)}, "running-a": {ID: "running-a", On: "ace", At: at(6)}, "done": {ID: "done", On: "novox", At: at(7)}, }, Outcomes: map[string]bool{"done": true}, } got := ClassifyQueue(asks, 4, 3, heard) want := map[string]string{"dead-1": AskDead, "running-g": AskInFlight, "running-a": AskInFlight, "unsaid": AskInFlight, "waiting-1": AskWaiting, "waiting-2": AskWaiting} for _, a := range got { if a.State != want[a.ID] { t.Errorf("%s is %s, want %s", a.ID, a.State, want[a.ID]) } } q := Queue{Asks: got} if a, _ := q.Find("running-a"); a.On != "ace" || a.Started.IsZero() { t.Errorf("an ask in flight does not say where: %+v", a) } // **The worker's count bounds it**: with two pending, the machine whose start is oldest died with // its ask, and the ask nobody said is not in flight either. got = ClassifyQueue(asks, 4, 2, heard) q = Queue{Asks: got} if a, _ := q.Find("running-g"); a.State != AskInFlight { t.Errorf("running-g is %s", a.State) } if a, _ := q.Find("unsaid"); a.State != AskDead { t.Errorf("an ask no start explains, past the worker's count, is %s", a.State) } // None pending: everything behind the worker is dead, and names no machine. for _, a := range ClassifyQueue(asks, 4, 0, heard) { if a.Seq <= 4 && (a.State != AskDead || a.On != "") { t.Errorf("%s with nothing pending is %s on %q", a.ID, a.State, a.On) } } } // What a direct read of the cancelled set answers. func TestACancelledSetReadSaysWhatWasCancelled(t *testing.T) { found := &nats.Msg{Header: nats.Header{"Nats-Subject": []string{"$KV.x.build-1"}}, Data: []byte(`{"why":"cancelled by hand"}`)} if c, err := cancelledFrom(found); err != nil || !c { t.Errorf("a value read as %v %v", c, err) } missing := &nats.Msg{Header: nats.Header{"Status": []string{"404"}}} if c, err := cancelledFrom(missing); err != nil || c { t.Errorf("not found read as %v %v", c, err) } deleted := &nats.Msg{Header: nats.Header{"KV-Operation": []string{"DEL"}}} if c, err := cancelledFrom(deleted); err != nil || c { t.Errorf("a deletion marker read as %v %v", c, err) } broken := &nats.Msg{Header: nats.Header{"Status": []string{"408"}, "Description": []string{"Request Timeout"}}} if _, err := cancelledFrom(broken); err == nil { t.Error("an error answer read as an answer") } if c, err := IsCancelled(nil, TheBuildMachine, "a.b.>"); err != nil || c { t.Error("a pattern was looked up as a key") } } // --- against a real server ---------------------------------------------------------------------- func aBusWithACancelledSet(t *testing.T) *broker.JetStream { t.Helper() js := aBusWithTheBuildRole(t) seat := broker.DeclaredSeat{Name: TheBuildMachine, Accepts: []string{"build"}} if err := broker.RaiseCancelledSets(js, []broker.DeclaredSeat{seat}); err != nil { t.Fatal(err) } t.Cleanup(func() { _ = js.Context().DeleteKeyValue(broker.CancelledSetName(TheBuildMachine)) }) return js } // **The race a delete alone leaves open**: an ask fetched in the moment it was cancelled is ended by // the holder that took it — terminated, never built, its outcome said as failed. func TestNatsAnAskCancelledAsItWasTakenIsNotBuilt(t *testing.T) { js := aBusWithACancelledSet(t) ctx, stop := context.WithCancel(context.Background()) defer stop() if err := MarkCancelled(ctx, js, TheBuildMachine, "build-cancelled"); err != nil { t.Fatal(err) } outcomes, err := js.Conn().SubscribeSync(BuildOutcome()) if err != nil { t.Fatal(err) } defer func() { _ = outcomes.Unsubscribe() }() _ = js.Conn().Flush() for _, id := range []string{"build-cancelled", "build-kept"} { body, _ := json.Marshal(BuildRequest{ID: id, Repository: "/r"}) if _, err := js.Context().Publish(BuildWork(), body); err != nil { t.Fatal(err) } } built := make(chan string, 2) machine := MachineOverNATS(js, "anchor") defer machine.Close() go func() { _ = machine.Take(ctx, func(ctx context.Context, work Build) { built <- work.Request().ID _ = work.Announce(ctx, BuildResult{ID: work.Request().ID, On: "anchor", Failed: "no"}) _ = work.Done() }) }() select { case id := <-built: if id != "build-kept" { t.Fatalf("%s was built", id) } case <-time.After(10 * time.Second): t.Fatal("the ask that was not cancelled was never built") } msg, err := outcomes.NextMsg(5 * time.Second) if err != nil { t.Fatal(err) } var said BuildResult if err := json.Unmarshal(msg.Data, &said); err != nil || said.ID != "build-cancelled" || said.Failed != CancelledByHand || said.On != "anchor" { t.Fatalf("the cancelled ask's outcome is %+v (%v)", said, err) } deadline := time.Now().Add(5 * time.Second) for time.Now().Before(deadline) { if info, err := js.Context().StreamInfo("SEAT_NODE_BUILD_AGENT"); err == nil && info.State.Msgs == 0 { return } time.Sleep(20 * time.Millisecond) } t.Fatal("the cancelled ask is still queued: terminated, it would not be") } // A paused holder takes nothing; resumed, it takes what waited. func TestNatsAPausedHolderTakesNothingNew(t *testing.T) { js := aBusWithACancelledSet(t) ctx, stop := context.WithCancel(context.Background()) defer stop() paused := make(chan bool, 1) paused <- true isPaused := func() bool { p := <-paused paused <- p return p } took := make(chan string, 1) machine := MachineOverNATSWith(js, "anchor", TheBuildMachine, MachineOptions{Paused: isPaused}) defer machine.Close() go func() { _ = machine.Take(ctx, func(ctx context.Context, work Build) { took <- work.Request().ID _ = work.Announce(ctx, BuildResult{ID: work.Request().ID, On: "anchor", Failed: "no"}) _ = work.Done() }) }() body, _ := json.Marshal(BuildRequest{ID: "build-waits", Repository: "/r"}) if _, err := js.Context().Publish(BuildWork(), body); err != nil { t.Fatal(err) } select { case id := <-took: t.Fatalf("a paused holder took %s", id) case <-time.After(3 * time.Second): } q, err := ReadQueue(ctx, js, TheBuildMachine) if err != nil { t.Fatal(err) } if len(q.Asks) != 1 || q.Asks[0].State != AskWaiting { t.Fatalf("while paused the queue reads %+v", q.Asks) } <-paused paused <- false select { case id := <-took: if id != "build-waits" { t.Fatalf("took %s", id) } case <-time.After(10 * time.Second): t.Fatal("resumed, the holder never took what waited") } } // What a holder last said about taking work is read back per machine. func TestNatsAHoldersPausedStateIsReadBack(t *testing.T) { js := aBusWithTheBuildRole(t) for _, s := range []HolderState{{On: "ace", Paused: true}, {On: "g14", Paused: true}, {On: "g14", Paused: false}} { body, _ := json.Marshal(s) if _, err := js.Context().Publish(BuildPausedOf(TheBuildMachine, s.On), body); err != nil { t.Fatal(err) } } said, err := PausedSaid(js, TheBuildMachine, []string{"ace", "g14", "novox"}) if err != nil { t.Fatal(err) } if !said["ace"].Paused || said["g14"].Paused || len(said) != 2 { t.Fatalf("read back %+v", said) } } // A `build` waiting on its outcome hears a cancel — which publishes no outcome — from the cancelled // set, and a kill from the outcome the holder announces (novox/hq ADR 0219). func TestNatsAWaitingAskerHearsItsAskCancelledOrKilled(t *testing.T) { js := aBusWithACancelledSet(t) was := cancelLook cancelLook = 200 * time.Millisecond t.Cleanup(func() { cancelLook = was }) ask := &natsBuilds{js: js, seat: TheBuildMachine} go func() { time.Sleep(500 * time.Millisecond) _ = MarkCancelled(context.Background(), js, TheBuildMachine, "build-waited-cancelled") }() result, err := ask.Submit(context.Background(), BuildRequest{ID: "build-waited-cancelled", Repository: "/r"}, 10*time.Second) if err != nil || result.Failed != CancelledByHand || result.ID != "build-waited-cancelled" { t.Fatalf("the waiter heard %+v (%v)", result, err) } // Killed: the holder announces the failure as any outcome, and the waiter has it. ctx, stop := context.WithCancel(context.Background()) defer stop() machine := MachineOverNATS(js, "ace") defer machine.Close() go func() { _ = machine.Take(ctx, func(ctx context.Context, work Build) { r := work.Request() _ = work.Announce(ctx, BuildResult{ID: r.ID, Repository: r.Repository, On: "ace", Failed: KilledByHand}) _ = work.Done() }) }() result, err = ask.Submit(context.Background(), BuildRequest{ID: "build-waited-killed", Repository: "/r"}, 10*time.Second) if err != nil || result.Failed != KilledByHand || result.On != "ace" { t.Fatalf("the waiter heard %+v (%v)", result, err) } }