493 lines
18 KiB
Go
493 lines
18 KiB
Go
//go:build integration
|
|
|
|
package main
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"fmt"
|
|
"reflect"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/nats-io/nats.go"
|
|
"github.com/novox/mesh-controller/internal/broker"
|
|
"github.com/novox/mesh-controller/internal/inventory"
|
|
"github.com/novox/mesh-controller/internal/link"
|
|
"github.com/novox/mesh-controller/internal/testbus"
|
|
)
|
|
|
|
func TestAFailedPlanIsRetriedAndGoesOnThroughItsLaterTiers(t *testing.T) {
|
|
open := aMesh(t)
|
|
ctx := t.Context()
|
|
asked := asksRecorded(t)
|
|
twoTiers(t, open)
|
|
before := time.Now().UTC().Add(-time.Hour)
|
|
failed := inventory.Plan{ID: "plan-retry", Repository: "novox/a", Branch: "main", Commit: "c0ffee",
|
|
Created: before, State: inventory.PlanFailed, Tier: 0, Tiers: [][]string{{"a"}, {"b"}},
|
|
Note: "a failed to build in tier 0",
|
|
Modules: map[string]*inventory.PlanModule{"a": {State: "failed", AskedAt: &before, Build: "build-1",
|
|
Why: link.KilledByHand}}}
|
|
if err := open.inventory.SavePlan(ctx, &failed); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
said, err := retryPlan(ctx, open, failed.ID)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
p, err := open.inventory.PlanByID(ctx, failed.ID)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
a := p.Modules["a"]
|
|
if p.State != inventory.PlanBuilding || a.State != "asked" || a.Build == "" || a.Build == "build-1" || a.Why != "" {
|
|
t.Fatalf("after retry the plan is %s and a is %+v", p.State, a)
|
|
}
|
|
if !strings.Contains(said, a.Build) {
|
|
t.Errorf("retry does not say the new id: %q", said)
|
|
}
|
|
if len(*asked) != 1 {
|
|
t.Fatalf("asked %v", *asked)
|
|
}
|
|
|
|
// The killed build's own late outcome is not this ask's; the new one's is, and tier 1 follows.
|
|
planBuilt(ctx, open, "a", "c0ffee", link.KilledByHand, before, "build-1")
|
|
if p, _ = open.inventory.PlanByID(ctx, failed.ID); p.State != inventory.PlanBuilding {
|
|
t.Fatalf("the old ask's outcome failed the retried plan: %s %q", p.State, p.Note)
|
|
}
|
|
asking, _ := link.BuildAskedAt(a.Build)
|
|
planBuilt(ctx, open, "a", "c0ffee", "", asking, a.Build)
|
|
p, err = open.inventory.PlanByID(ctx, failed.ID)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if p.Tier != 1 || p.Modules["b"] == nil || p.Modules["b"].State != "asked" || p.Modules["b"].Build == "" {
|
|
t.Fatalf("the retried plan did not go on to tier 1: tier %d, %s, b %+v", p.Tier, p.State, p.Modules["b"])
|
|
}
|
|
if len(*asked) != 2 {
|
|
t.Fatalf("asked %v", *asked)
|
|
}
|
|
}
|
|
|
|
func TestARebuildJoinsThePlanHoldingTheModule(t *testing.T) {
|
|
open := aMesh(t)
|
|
ctx := t.Context()
|
|
asked := asksRecorded(t)
|
|
twoTiers(t, open)
|
|
before := time.Now().UTC().Add(-time.Hour)
|
|
failed := inventory.Plan{ID: "plan-join", Repository: "novox/a", Branch: "main", Commit: "c0ffee",
|
|
Created: before, State: inventory.PlanFailed, Tiers: [][]string{{"a"}, {"b"}},
|
|
Note: "a failed to build in tier 0",
|
|
Modules: map[string]*inventory.PlanModule{"a": {State: "failed", AskedAt: &before, Build: "build-1", Why: link.CancelledByHand}}}
|
|
if err := open.inventory.SavePlan(ctx, &failed); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := rebuildCommand(ctx, []string{"a"}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
p, err := open.inventory.PlanByID(ctx, failed.ID)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if p.State != inventory.PlanBuilding || p.Modules["a"].State != "asked" || p.Modules["a"].Build == "build-1" {
|
|
t.Fatalf("the rebuild did not join the failed plan: %s %+v", p.State, p.Modules["a"])
|
|
}
|
|
if len(*asked) != 1 {
|
|
t.Fatalf("asked %v", *asked)
|
|
}
|
|
// b is in a tier not yet reached: nothing holds it in its current tier, so it is asked alone.
|
|
if err := rebuildCommand(ctx, []string{"b"}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if p, _ = open.inventory.PlanByID(ctx, failed.ID); p.Modules["b"] != nil {
|
|
t.Fatalf("a module the plan has not reached joined it: %+v", p.Modules["b"])
|
|
}
|
|
if len(*asked) != 2 {
|
|
t.Fatalf("asked %v", *asked)
|
|
}
|
|
}
|
|
|
|
func aBuildQueue(t *testing.T) (*broker.JetStream, func(id, repository string) link.BuildRequest) {
|
|
t.Helper()
|
|
url := testbus.URL(t)
|
|
t.Setenv(broker.NATSVar, url)
|
|
js, err := broker.Dial(url)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
t.Cleanup(js.Close)
|
|
seat := broker.DeclaredSeat{Name: link.TheBuildMachine, Accepts: []string{"build"},
|
|
Emits: []string{"started", "built", "log.*", "paused.*"}}
|
|
if err := broker.AssertMeshStreams(js); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
_ = js.Context().DeleteStream("SEAT_NODE_BUILD_AGENT")
|
|
if err := broker.RaiseSeats(js, []broker.DeclaredSeat{seat}, map[string]broker.Holder{
|
|
link.TheBuildMachine: {Node: "anchor", Module: "build-agent"}}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := broker.RaiseCancelledSets(js, []broker.DeclaredSeat{seat}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
t.Cleanup(func() {
|
|
_ = js.Context().DeleteStream("SEAT_NODE_BUILD_AGENT")
|
|
_ = js.Context().DeleteKeyValue(broker.CancelledSetName(link.TheBuildMachine))
|
|
_ = js.Context().PurgeStream(broker.EventsStream)
|
|
})
|
|
ask := func(id, repository string) link.BuildRequest {
|
|
r := link.BuildRequest{ID: id, Repository: repository, Held: map[string]string{"x/y": "secret-ish"}}
|
|
body, _ := json.Marshal(r)
|
|
if _, err := js.Context().Publish(link.BuildWork(), body); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return r
|
|
}
|
|
return js, ask
|
|
}
|
|
|
|
func TestCancelDeletesTheAskAndFailsThePlanThatAskedIt(t *testing.T) {
|
|
js, ask := aBuildQueue(t)
|
|
open := aMesh(t)
|
|
ctx := t.Context()
|
|
twoTiers(t, open)
|
|
id := link.NewBuildID(time.Now())
|
|
ask(id, "https://forge.example/novox/a.git")
|
|
asked, _ := link.BuildAskedAt(id)
|
|
plan := inventory.Plan{ID: "plan-cancel", Repository: "novox/a", Commit: "c0ffee", Created: asked,
|
|
State: inventory.PlanBuilding, Tiers: [][]string{{"a"}, {"b"}},
|
|
Modules: map[string]*inventory.PlanModule{"a": {State: "asked", AskedAt: &asked, Build: id}}}
|
|
if err := open.inventory.SavePlan(ctx, &plan); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
q, err := link.ReadQueue(ctx, js, link.TheBuildMachine)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(q.Asks) != 1 || q.Asks[0].State != link.AskWaiting || q.Asks[0].ID != id {
|
|
t.Fatalf("the queue reads %+v", q)
|
|
}
|
|
if text := queueText(q, time.Now()); !strings.Contains(text, "1 waiting, 0 in flight, 0 dead") ||
|
|
strings.Contains(text, "secret-ish") {
|
|
t.Errorf("the queue says:\n%s", text)
|
|
}
|
|
|
|
if err := cancelCommand(ctx, []string{id}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if q, _ = link.ReadQueue(ctx, js, link.TheBuildMachine); len(q.Asks) != 0 {
|
|
t.Fatalf("the ask is still queued: %+v", q.Asks)
|
|
}
|
|
if cancelled, err := link.IsCancelled(js.Conn(), link.TheBuildMachine, id); err != nil || !cancelled {
|
|
t.Errorf("the cancelled set does not hold it: %v %v", cancelled, err)
|
|
}
|
|
b, found, err := open.inventory.BuildByID(ctx, id)
|
|
if err != nil || !found || b.Failed != link.CancelledByHand {
|
|
t.Fatalf("the cancel is recorded as %+v (%v %v)", b, found, err)
|
|
}
|
|
p, err := open.inventory.PlanByID(ctx, plan.ID)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if p.State != inventory.PlanFailed || p.Modules["a"].State != "failed" || p.Modules["a"].Why != link.CancelledByHand {
|
|
t.Fatalf("the plan that asked is %s, a %+v", p.State, p.Modules["a"])
|
|
}
|
|
if err := cancelCommand(ctx, []string{id}); err == nil {
|
|
t.Error("an ask cancelled already was cancelled again")
|
|
}
|
|
}
|
|
|
|
func TestClearCancelsTheWaitingAndTheDeadOnlyWhenTold(t *testing.T) {
|
|
js, ask := aBuildQueue(t)
|
|
open := aMesh(t)
|
|
ctx := t.Context()
|
|
start := time.Now()
|
|
dead := link.NewBuildID(start)
|
|
ask(dead, "https://forge.example/novox/dead.git")
|
|
deadOne(t, js)
|
|
var waiting []string
|
|
for i := 1; i <= 2; i++ {
|
|
id := link.NewBuildID(start.Add(time.Duration(i) * time.Millisecond))
|
|
waiting = append(waiting, id)
|
|
ask(id, fmt.Sprintf("https://forge.example/novox/w%d.git", i))
|
|
}
|
|
|
|
q, err := link.ReadQueue(ctx, js, link.TheBuildMachine)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(q.Of(link.AskDead)) != 1 || len(q.Of(link.AskWaiting)) != 2 || q.MaxDeliver != 5 {
|
|
t.Fatalf("the queue reads %+v", q)
|
|
}
|
|
said, err := clearQueue(ctx, js, open, link.TheBuildMachine, false)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if !strings.Contains(said, "2 waiting ask(s) cancelled") || !strings.Contains(said, "1 dead, left") {
|
|
t.Errorf("clear said:\n%s", said)
|
|
}
|
|
q, _ = link.ReadQueue(ctx, js, link.TheBuildMachine)
|
|
if len(q.Asks) != 1 || q.Asks[0].ID != dead || q.Asks[0].State != link.AskDead {
|
|
t.Fatalf("after clear the queue is %+v", q.Asks)
|
|
}
|
|
if _, err := clearQueue(ctx, js, open, link.TheBuildMachine, true); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if q, _ = link.ReadQueue(ctx, js, link.TheBuildMachine); len(q.Asks) != 0 {
|
|
t.Fatalf("clear --dead left %+v", q.Asks)
|
|
}
|
|
for _, id := range append(waiting, dead) {
|
|
if b, found, _ := open.inventory.BuildByID(ctx, id); !found || b.Failed != link.CancelledByHand {
|
|
t.Errorf("%s is recorded as %+v", id, b)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestCancelRefusesAnAskInFlight(t *testing.T) {
|
|
js, ask := aBuildQueue(t)
|
|
open := aMesh(t)
|
|
ctx := t.Context()
|
|
id := link.NewBuildID(time.Now())
|
|
ask(id, "https://forge.example/novox/a.git")
|
|
worker, _ := broker.HolderConsumerFor("", "", broker.DeclaredSeat{Name: link.TheBuildMachine, Accepts: []string{"build"}})
|
|
sub, err := js.Context().PullSubscribe(worker.Filters[0], worker.Name, nats.Bind(worker.Stream, worker.Name), nats.ManualAck())
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer func() { _ = sub.Unsubscribe() }()
|
|
if _, err := sub.Fetch(1, nats.MaxWait(3*time.Second)); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
started, _ := json.Marshal(link.BuildStart{ID: id, On: "ace", At: time.Now().UTC().Format(time.RFC3339Nano)})
|
|
if _, err := js.Context().Publish(link.BuildStarted(), started); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
q, err := link.ReadQueue(ctx, js, link.TheBuildMachine)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
a, _ := q.Find(id)
|
|
if a.State != link.AskInFlight || a.On != "ace" {
|
|
t.Fatalf("the taken ask reads %+v", a)
|
|
}
|
|
_, err = cancelAsk(ctx, js, open, link.TheBuildMachine, a)
|
|
if err == nil || !strings.Contains(err.Error(), "kill "+id) {
|
|
t.Fatalf("an ask in flight was cancelled: %v", err)
|
|
}
|
|
if _, found, _ := open.inventory.BuildByID(ctx, id); found {
|
|
t.Error("a refused cancel recorded an outcome")
|
|
}
|
|
}
|
|
|
|
func TestKillAndPauseAskTheHolderOnTheMachine(t *testing.T) {
|
|
js, _ := aBuildQueue(t)
|
|
ctx := t.Context()
|
|
asked := map[string]string{}
|
|
handlers := map[string]link.ToolHandler{
|
|
"kill": func(_ context.Context, raw json.RawMessage) (any, error) {
|
|
var args struct{ ID string }
|
|
_ = json.Unmarshal(raw, &args)
|
|
asked["kill"] = args.ID
|
|
if args.ID != "build-running" {
|
|
return nil, fmt.Errorf("ace is not building %s", args.ID)
|
|
}
|
|
return map[string]any{"said": "killed " + args.ID}, nil
|
|
},
|
|
"pause": func(context.Context, json.RawMessage) (any, error) {
|
|
asked["pause"] = "ace"
|
|
return map[string]any{"said": "ace is paused"}, nil
|
|
},
|
|
}
|
|
stop, err := link.OverNATS{Conn: js.Conn()}.ServeNodeSeatTools(link.TheBuildMachine, "ace", handlers, nil)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer stop()
|
|
for _, id := range []string{"build-running", "build-other"} {
|
|
started, _ := json.Marshal(link.BuildStart{ID: id, On: "ace", At: time.Now().UTC().Format(time.RFC3339Nano)})
|
|
if _, err := js.Context().Publish(link.BuildStarted(), started); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
if err := killCommand(ctx, []string{"build-running"}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if asked["kill"] != "build-running" {
|
|
t.Fatalf("the holder was asked %v", asked)
|
|
}
|
|
if err := killCommand(ctx, []string{"build-other"}); err == nil {
|
|
t.Error("the holder's refusal was not the command's")
|
|
}
|
|
if err := killCommand(ctx, []string{"build-never"}); err == nil || !strings.Contains(err.Error(), "cancel build-never") {
|
|
t.Errorf("a build nobody started: %v", err)
|
|
}
|
|
if err := pauseCommand(ctx, "pause", []string{"ace"}); err != nil || asked["pause"] != "ace" {
|
|
t.Fatalf("pause: %v %v", err, asked)
|
|
}
|
|
if err := pauseCommand(ctx, "resume", []string{"g14"}); err == nil {
|
|
t.Error("a machine nothing answers on was resumed")
|
|
}
|
|
}
|
|
|
|
func TestAPlanStoppedAtItsFirstMachineIsRetried(t *testing.T) {
|
|
withTheGate(t) // the first-machine gate's own path (novox/hq ADR 0360 keeps it for earlier walks)
|
|
open := aMesh(t)
|
|
ctx := t.Context()
|
|
asked := asksRecorded(t)
|
|
twoTiers(t, open)
|
|
var sentTo [][]string
|
|
was := sendRollout
|
|
sendRollout = func(_ context.Context, _ *stores, names []string) ([]string, error) {
|
|
sentTo = append(sentTo, names)
|
|
return names, nil
|
|
}
|
|
t.Cleanup(func() { sendRollout = was })
|
|
|
|
// a records: a person's choice, since the default rolls out (novox/hq ADR 0236).
|
|
if err := open.inventory.SetUpgradeOf(ctx, "a", inventory.Upgrade{Why: "test"}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
long := time.Now().UTC().Add(-2 * time.Hour)
|
|
stopped := inventory.Plan{ID: "plan-rollout", Repository: "novox/a", Branch: "main", Commit: "c0ffee",
|
|
Created: long, State: inventory.PlanFailed, Tiers: [][]string{{"a"}, {"b"}},
|
|
Note: "a stopped at its first machine in tier 0: laptop refused what it was sent",
|
|
// Asked by the plan, as every module a plan holds built is (novox/hq issue 754).
|
|
Modules: map[string]*inventory.PlanModule{"a": {State: "built", AskedAt: &long, Build: "build-a", BuiltAt: &long,
|
|
Commit: "c0ffee", First: []string{"laptop"}, FirstAt: &long, Why: "laptop refused what it was sent"}}}
|
|
if err := open.inventory.SavePlan(ctx, &stopped); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
// A newer plan holding a refuses it: sending the older build would put it back.
|
|
newer := inventory.Plan{ID: "plan-newer", Repository: "novox/other", Commit: "d00d", Created: long.Add(time.Hour),
|
|
State: inventory.PlanDone, Tiers: [][]string{{"a"}}, Modules: map[string]*inventory.PlanModule{}}
|
|
if err := open.inventory.SavePlan(ctx, &newer); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := retryPlan(ctx, open, stopped.ID); err == nil || !strings.Contains(err.Error(), "plan-newer") {
|
|
t.Fatalf("retried under a newer plan: %v", err)
|
|
}
|
|
newer.State = inventory.PlanSuperseded
|
|
if err := open.inventory.SavePlan(ctx, &newer); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
said, err := retryPlan(ctx, open, stopped.ID)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(sentTo) != 1 || !reflect.DeepEqual(sentTo[0], []string{"laptop"}) || !strings.Contains(said, "laptop") {
|
|
t.Fatalf("sent %v; said %q", sentTo, said)
|
|
}
|
|
p, err := open.inventory.PlanByID(ctx, stopped.ID)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
a := p.Modules["a"]
|
|
if p.State != inventory.PlanRolling || a.FirstAt == nil || !a.FirstAt.After(long) || a.Why != "" {
|
|
t.Fatalf("after retry the plan is %s, a %+v", p.State, a)
|
|
}
|
|
// And it goes on: a records (its policy sends nothing more), so the next tier is asked.
|
|
advancePlans(ctx, open)
|
|
if p, _ = open.inventory.PlanByID(ctx, stopped.ID); p.Tier != 1 || p.Modules["b"] == nil || p.Modules["b"].State != "asked" {
|
|
t.Fatalf("the retried plan did not go on: tier %d %s %+v", p.Tier, p.State, p.Modules["b"])
|
|
}
|
|
if len(*asked) != 1 {
|
|
t.Fatalf("asked %v", *asked)
|
|
}
|
|
}
|
|
|
|
func TestAPlanIsAnsweredOnlyByTheBuildItAskedFor(t *testing.T) {
|
|
open := aMesh(t)
|
|
ctx := t.Context()
|
|
asked := time.Now().UTC().Add(-time.Minute)
|
|
plan := inventory.Plan{ID: "plan-own", Repository: "novox/a", Commit: "c0ffee", Created: asked,
|
|
State: inventory.PlanBuilding, Tiers: [][]string{{"a"}},
|
|
Modules: map[string]*inventory.PlanModule{"a": {State: "asked", AskedAt: &asked, Build: "build-own"}}}
|
|
if err := open.inventory.SavePlan(ctx, &plan); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
planBuilt(ctx, open, "a", "0ldc0mm1t", "", time.Now().UTC(), "build-replay")
|
|
p, _ := open.inventory.PlanByID(ctx, plan.ID)
|
|
if p.Modules["a"].State != "asked" {
|
|
t.Fatalf("a replay asked after the plan settled it: %+v", p.Modules["a"])
|
|
}
|
|
// From the records too: a later build of the module recorded is not the plan's.
|
|
recorded := map[string][]inventory.Build{"a": {{ID: "build-replay", Commit: "0ldc0mm1t", Asked: time.Now(), At: time.Now()}}}
|
|
if settleFromRecords(&p, p.Tiers[0], recorded, nil) {
|
|
t.Fatalf("the records settled it with another build: %+v", p.Modules["a"])
|
|
}
|
|
planBuilt(ctx, open, "a", "c0ffee", "", asked, "build-own")
|
|
if p, _ = open.inventory.PlanByID(ctx, plan.ID); p.Modules["a"].State != "built" || p.Modules["a"].Commit != "c0ffee" {
|
|
t.Fatalf("its own build did not settle it: %+v", p.Modules["a"])
|
|
}
|
|
}
|
|
|
|
func TestARebuildOfACommitAsksWhatTheModuleFollows(t *testing.T) {
|
|
open := aMesh(t)
|
|
ctx := t.Context()
|
|
asked := asksRecorded(t)
|
|
twoTiers(t, open)
|
|
var refs []string
|
|
was := askABuild
|
|
askABuild = func(c context.Context, source buildSource, path, ref string) (string, error) {
|
|
refs = append(refs, ref)
|
|
return was(c, source, path, ref)
|
|
}
|
|
if err := open.inventory.RecordBuild(ctx, inventory.Build{ID: "build-at-commit", Repository: "novox/a",
|
|
Ref: "0123456789abcdef0123456789abcdef01234567", Module: "a", Commit: "0123456789abcdef0123456789abcdef01234567"}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := rebuildCommand(ctx, []string{"build-at-commit"}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(refs) != 1 || refs[0] != "main" || len(*asked) != 1 {
|
|
t.Fatalf("asked %v at %v", *asked, refs)
|
|
}
|
|
}
|
|
|
|
func TestCancelOfAnAskNobodySaidIsWithdrawnWhenItStarts(t *testing.T) {
|
|
js, ask := aBuildQueue(t)
|
|
open := aMesh(t)
|
|
ctx := t.Context()
|
|
was := holderLooks
|
|
holderLooks = 300 * time.Millisecond
|
|
t.Cleanup(func() { holderLooks = was })
|
|
id := link.NewBuildID(time.Now())
|
|
ask(id, "https://forge.example/novox/a.git")
|
|
worker, _ := broker.HolderConsumerFor("", "", broker.DeclaredSeat{Name: link.TheBuildMachine, Accepts: []string{"build"}})
|
|
sub, err := js.Context().PullSubscribe(worker.Filters[0], worker.Name, nats.Bind(worker.Stream, worker.Name), nats.ManualAck())
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer func() { _ = sub.Unsubscribe() }()
|
|
if _, err := sub.Fetch(1, nats.MaxWait(3*time.Second)); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
q, err := link.ReadQueue(ctx, js, link.TheBuildMachine)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
a, _ := q.Find(id)
|
|
if a.State != link.AskInFlight || a.On != "" {
|
|
t.Fatalf("the taken ask reads %+v", a)
|
|
}
|
|
// The holder that took it says it started, while the cancel waits.
|
|
go func() {
|
|
time.Sleep(100 * time.Millisecond)
|
|
started, _ := json.Marshal(link.BuildStart{ID: id, On: "ace", At: time.Now().UTC().Format(time.RFC3339Nano)})
|
|
_, _ = js.Context().Publish(link.BuildStarted(), started)
|
|
}()
|
|
if _, err := cancelAsk(ctx, js, open, link.TheBuildMachine, a); err == nil || !strings.Contains(err.Error(), "started on ace") {
|
|
t.Fatalf("a build that started was cancelled: %v", err)
|
|
}
|
|
if cancelled, _ := link.IsCancelled(js.Conn(), link.TheBuildMachine, id); cancelled {
|
|
t.Error("the withdrawn cancel is still marked")
|
|
}
|
|
if _, found, _ := open.inventory.BuildByID(ctx, id); found {
|
|
t.Error("a withdrawn cancel recorded an outcome")
|
|
}
|
|
}
|