199 lines
7.0 KiB
Go
199 lines
7.0 KiB
Go
//go:build integration
|
|
|
|
package buildstep
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"strings"
|
|
"sync"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/nats-io/nats.go"
|
|
"github.com/nats-io/nats.go/jetstream"
|
|
|
|
sdkstep "git.novox.be/novox/mesh-delivery/sdk/buildstep"
|
|
"git.novox.be/novox/mesh-sdk/go/testbus"
|
|
)
|
|
|
|
func askBus(t *testing.T, ackWait time.Duration) jetstream.JetStream {
|
|
t.Helper()
|
|
nc, err := nats.Connect(testbus.URL(t))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
t.Cleanup(nc.Close)
|
|
js, err := jetstream.New(nc)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := DeclareAskWorkers(context.Background(), js, sdkstep.AskGo, ackWait); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return js
|
|
}
|
|
|
|
func publishAsk(t *testing.T, js jetstream.JetStream, a sdkstep.StepAsk) {
|
|
t.Helper()
|
|
raw, err := sdkstep.EncodeStepAsk(a)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
subject, _ := sdkstep.AskSubject(a.Language, a.Kind)
|
|
if _, err := js.Publish(context.Background(), subject, raw); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
|
|
// 1887/A2a, 1887/AC-order, 1887/AC-pull on the real server: a check ask published before a delivery ask runs after it;
|
|
// with capacity 2 and three asks two run, the third waits in the stream and starts when one ends.
|
|
func TestPullOrderAndCapacityOnTheBus(t *testing.T) {
|
|
js := askBus(t, 30*time.Second)
|
|
check, first, second := testAsk(sdkstep.AskCheck), testAsk(sdkstep.AskDelivery), testAsk(sdkstep.AskDelivery)
|
|
publishAsk(t, js, check)
|
|
publishAsk(t, js, first)
|
|
publishAsk(t, js, second)
|
|
release := make(chan struct{})
|
|
var mu sync.Mutex
|
|
var order []string
|
|
st := newSteps()
|
|
p := &Puller{Source: &JetStreamAsks{JS: js, Language: sdkstep.AskGo}, Language: sdkstep.AskGo, Node: "shanks", Put: st.put,
|
|
Capacity: func() int { return 2 }, PullWait: 200 * time.Millisecond,
|
|
Run: func(ctx context.Context, a sdkstep.StepAsk) (any, error) {
|
|
mu.Lock()
|
|
order = append(order, a.ID)
|
|
mu.Unlock()
|
|
<-release
|
|
return nil, nil
|
|
}}
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
defer cancel()
|
|
go p.Pull(ctx)
|
|
waitFor(t, "two in hand", func() bool { return p.InHand() == 2 })
|
|
time.Sleep(500 * time.Millisecond)
|
|
mu.Lock()
|
|
if len(order) != 2 || order[0] != first.ID || order[1] != second.ID {
|
|
t.Fatalf("ran %v, want the two delivery asks %s %s first", order, first.ID, second.ID)
|
|
}
|
|
mu.Unlock()
|
|
release <- struct{}{}
|
|
waitFor(t, "the check taken", func() bool { mu.Lock(); defer mu.Unlock(); return len(order) == 3 })
|
|
close(release)
|
|
waitFor(t, "every step ended", func() bool {
|
|
for _, a := range []sdkstep.StepAsk{check, first, second} {
|
|
key, _ := sdkstep.AskStepKey("shanks", a.ID)
|
|
if s, ok := st.get(key); !ok || s.State != sdkstep.StepPassed {
|
|
return false
|
|
}
|
|
}
|
|
return true
|
|
})
|
|
info, err := js.Stream(ctx, "SEAT_NODE_BUILD_GO")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
waitFor(t, "the stream empty (every ask acknowledged)", func() bool {
|
|
si, err := info.Info(ctx)
|
|
return err == nil && si.State.Msgs == 0
|
|
})
|
|
}
|
|
|
|
// 1887/A2c, ADR 0525 §6: a killed puller's ask is redelivered by the stream's acknowledgement wait to the other node,
|
|
// whose step key names the reference.
|
|
func TestPullRedeliversAKilledPullersAsk(t *testing.T) {
|
|
js := askBus(t, 2*time.Second)
|
|
a := testAsk(sdkstep.AskDelivery)
|
|
publishAsk(t, js, a)
|
|
st := newSteps()
|
|
taken := make(chan struct{})
|
|
dying := &Puller{Source: &JetStreamAsks{JS: js, Language: sdkstep.AskGo}, Language: sdkstep.AskGo, Node: "ace", Put: st.put,
|
|
Capacity: func() int { return 1 }, PullWait: 200 * time.Millisecond, Heartbeat: time.Hour,
|
|
Run: func(ctx context.Context, a sdkstep.StepAsk) (any, error) {
|
|
close(taken)
|
|
<-ctx.Done()
|
|
return nil, ctx.Err()
|
|
}}
|
|
kill, killed := context.WithCancel(context.Background())
|
|
go dying.Pull(kill)
|
|
<-taken
|
|
killed()
|
|
other := &Puller{Source: &JetStreamAsks{JS: js, Language: sdkstep.AskGo}, Language: sdkstep.AskGo, Node: "shanks", Put: st.put,
|
|
Capacity: func() int { return 1 }, PullWait: 200 * time.Millisecond,
|
|
Run: func(ctx context.Context, a sdkstep.StepAsk) (any, error) {
|
|
return sdkstep.BuildAnswer{Artifacts: []sdkstep.AskBuilt{{Artifact: "bundle", Digest: "sha256:00", Reference: "artifact-store://x"}}}, nil
|
|
}}
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
defer cancel()
|
|
go other.Pull(ctx)
|
|
key, _ := sdkstep.AskStepKey("shanks", a.ID)
|
|
deadline := time.Now().Add(10 * time.Second)
|
|
for {
|
|
if s, ok := st.get(key); ok && s.State == sdkstep.StepPassed {
|
|
if want := `artifact-store://x`; !contains(string(s.Result), want) {
|
|
t.Fatalf("the step's result %s names no %s", s.Result, want)
|
|
}
|
|
return
|
|
}
|
|
if time.Now().After(deadline) {
|
|
t.Fatalf("the killed puller's ask was not taken by shanks within 10 s")
|
|
}
|
|
time.Sleep(20 * time.Millisecond)
|
|
}
|
|
}
|
|
|
|
// ADR 0664 §1, §5 (issue 1748) on the real server: a run that ends on a transport fault is written again and the bus gives the
|
|
// ask again after the delay, not before, as the second run (its delivery count one higher), and that run's end is the ask's;
|
|
// a step that runs when its ask is dropped ends dropped, naming who dropped it, and the ask is not given again.
|
|
func TestAnAskThatMetTheNetworkIsGivenAgainAndADroppedOneIsEnded(t *testing.T) {
|
|
js := askBus(t, 30*time.Second)
|
|
a, doomed := testAsk(sdkstep.AskDelivery), testAsk(sdkstep.AskDelivery)
|
|
publishAsk(t, js, a)
|
|
st, kv := newSteps(), &pauseKV{}
|
|
const delay = 600 * time.Millisecond
|
|
var mu sync.Mutex
|
|
var runs []time.Time
|
|
hold := make(chan struct{})
|
|
p := &Puller{Source: &JetStreamAsks{JS: js, Language: sdkstep.AskGo}, Language: sdkstep.AskGo, Node: "ace", Put: st.put, PausePut: kv.put, Load: kv.load,
|
|
ForgetPause: kv.forget, PauseKeys: kv.keys, Capacity: func() int { return 1 }, PullWait: 100 * time.Millisecond,
|
|
Delay: func(int) time.Duration { return delay },
|
|
Run: func(ctx context.Context, got sdkstep.StepAsk) (any, error) {
|
|
if got.ID == doomed.ID {
|
|
close(hold)
|
|
<-ctx.Done()
|
|
return nil, ctx.Err()
|
|
}
|
|
mu.Lock()
|
|
runs = append(runs, time.Now())
|
|
n := len(runs)
|
|
mu.Unlock()
|
|
if n == 1 {
|
|
return nil, &sdkstep.Passing{Err: errors.New("lookup git.novox.be: i/o timeout")}
|
|
}
|
|
return nil, nil
|
|
}}
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
defer cancel()
|
|
go p.Pull(ctx)
|
|
key, _ := sdkstep.AskStepKey("ace", a.ID)
|
|
waitFor(t, "the ask's second run ended", func() bool { s, ok := st.get(key); return ok && s.State == sdkstep.StepPassed })
|
|
mu.Lock()
|
|
gap := runs[1].Sub(runs[0])
|
|
mu.Unlock()
|
|
if s, _ := st.get(key); len(runs) != 2 || gap < delay || s.Run != 2 {
|
|
t.Errorf("runs %d, %s apart (delay %s), the step's run %d: want two runs, the second after the delay, as run 2", len(runs), gap, delay, s.Run)
|
|
}
|
|
publishAsk(t, js, doomed)
|
|
<-hold
|
|
if err := p.Drop(doomed.ID, "mesh-delivery-builder", "superseded"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
dkey, _ := sdkstep.AskStepKey("ace", doomed.ID)
|
|
waitFor(t, "the dropped step dropped", func() bool { s, ok := st.get(dkey); return ok && s.State == sdkstep.StepDropped })
|
|
if s, _ := st.get(dkey); !contains(s.Reason, "dropped by mesh-delivery-builder: superseded") {
|
|
t.Errorf("the dropped step says %q", s.Reason)
|
|
}
|
|
}
|
|
|
|
func contains(s, sub string) bool { return strings.Contains(s, sub) }
|