440 lines
14 KiB
Go
440 lines
14 KiB
Go
package watch
|
|
|
|
import (
|
|
"encoding/json"
|
|
"errors"
|
|
"slices"
|
|
"testing"
|
|
"time"
|
|
|
|
"git.novox.be/novox/mesh-sdk/go/delivery"
|
|
)
|
|
|
|
type fakeOut struct {
|
|
presence map[string]bool
|
|
presenceWrite int
|
|
pulses map[string]delivery.Pulse
|
|
ttls map[string]time.Duration
|
|
pulseDeletes []string
|
|
stuck map[string]delivery.Stuck
|
|
stuckWrites int
|
|
}
|
|
|
|
func newOut() *fakeOut {
|
|
return &fakeOut{presence: map[string]bool{}, pulses: map[string]delivery.Pulse{}, ttls: map[string]time.Duration{}, stuck: map[string]delivery.Stuck{}}
|
|
}
|
|
|
|
func (o *fakeOut) PutPresence(key string, online bool) error {
|
|
o.presence[key] = online
|
|
o.presenceWrite++
|
|
return nil
|
|
}
|
|
|
|
func (o *fakeOut) PutPulse(key string, value []byte, ttl time.Duration) error {
|
|
p, err := delivery.DecodePulse(value)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
o.pulses[key], o.ttls[key] = p, ttl
|
|
return nil
|
|
}
|
|
|
|
func (o *fakeOut) DeletePulse(key string) error {
|
|
delete(o.pulses, key)
|
|
o.pulseDeletes = append(o.pulseDeletes, key)
|
|
return nil
|
|
}
|
|
|
|
func (o *fakeOut) PutStuck(key string, value []byte) error {
|
|
s, err := delivery.DecodeStuck(value)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
o.stuck[key] = s
|
|
o.stuckWrites++
|
|
return nil
|
|
}
|
|
|
|
var t0 = time.Date(2026, 10, 18, 12, 0, 0, 0, time.UTC)
|
|
|
|
type clock struct{ t time.Time }
|
|
|
|
func (c *clock) now() time.Time { return c.t }
|
|
|
|
func setup() (*Watchman, *fakeOut, *clock) {
|
|
o, c := newOut(), &clock{t: t0}
|
|
return New(o, c.now), o, c
|
|
}
|
|
|
|
// checkValue is a check's key as the controller's builder writes it (mesh-controller internal/link/checks_nats.go
|
|
// CheckRunning, kept by keepRunning on every heartbeat).
|
|
func checkValue(started time.Time, head string) []byte {
|
|
b, _ := json.Marshal(map[string]any{"head": head, "trunk": "def456", "builder": "n1", "build": "b9", "started": started})
|
|
return b
|
|
}
|
|
|
|
const (
|
|
checksKey = "mesh-controller_checks-running"
|
|
presenceNS = delivery.PresenceBucket
|
|
)
|
|
|
|
func pulseKeyOf(t *testing.T, bucket, key string) string {
|
|
t.Helper()
|
|
k, err := delivery.PulseKey(bucket, key)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return k
|
|
}
|
|
|
|
func stuckKeyOf(t *testing.T, bucket, key string) string {
|
|
t.Helper()
|
|
k, err := delivery.StuckKey(bucket, key)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return k
|
|
}
|
|
|
|
func TestPresenceFromEvents(t *testing.T) { // 1135/AC1
|
|
w, o, _ := setup()
|
|
w.Connect("node.ace")
|
|
w.Connect("ace.mesh-delivery-view")
|
|
w.Connect("ace.gitea")
|
|
want := map[string]bool{"ace": true, "part.mesh-delivery-view.ace": true}
|
|
if len(o.presence) != 2 || o.presence["ace"] != true || o.presence["part.mesh-delivery-view.ace"] != true {
|
|
t.Fatalf("presence = %v, want %v and nothing for another module's user", o.presence, want)
|
|
}
|
|
w.Disconnect("node.ace")
|
|
w.Disconnect("ace.gitea")
|
|
if online, ok := o.presence["ace"]; !ok || online {
|
|
t.Fatalf("presence = %v, want ace offline", o.presence)
|
|
}
|
|
if len(o.presence) != 2 {
|
|
t.Fatalf("another module's user wrote presence: %v", o.presence)
|
|
}
|
|
}
|
|
|
|
func TestPresenceWrittenOnlyWhenItDiffers(t *testing.T) { // 1135/AC1
|
|
w, o, _ := setup()
|
|
w.Connect("node.ace")
|
|
w.Connect("node.ace")
|
|
if o.presenceWrite != 1 {
|
|
t.Fatalf("writes = %d, want 1 (the second connection changes nothing)", o.presenceWrite)
|
|
}
|
|
w.Disconnect("node.ace")
|
|
if !o.presence["ace"] || o.presenceWrite != 1 {
|
|
t.Fatalf("one of two connections closed: ace must stay online without a write, got %v, %d writes", o.presence, o.presenceWrite)
|
|
}
|
|
w.Disconnect("node.ace")
|
|
if o.presence["ace"] || o.presenceWrite != 2 {
|
|
t.Fatalf("both closed: ace offline, got %v, %d writes", o.presence, o.presenceWrite)
|
|
}
|
|
w.Disconnect("node.ace") // a stray event
|
|
if o.presenceWrite != 2 {
|
|
t.Fatalf("a disconnect of a user already offline wrote %d times", o.presenceWrite)
|
|
}
|
|
}
|
|
|
|
func TestParseEvent(t *testing.T) { // 1135/AC1
|
|
user, err := ParseEvent([]byte(`{"type":"io.nats.server.advisory.v1.client_connect","client":{"acc":"MESH","user":"node.ace","id":4}}`))
|
|
if err != nil || user != "node.ace" {
|
|
t.Fatalf("user = %q, %v", user, err)
|
|
}
|
|
if _, err := ParseEvent([]byte(`not json`)); err == nil {
|
|
t.Fatal("an event that is not JSON must be refused")
|
|
}
|
|
if user, err := ParseEvent([]byte(`{"client":{}}`)); err != nil || user != "" {
|
|
t.Fatalf("an event with no user = %q, %v; want empty and no error", user, err)
|
|
}
|
|
}
|
|
|
|
func connzAnswer(users ...string) []byte {
|
|
var conns []map[string]string
|
|
for _, u := range users {
|
|
conns = append(conns, map[string]string{"authorized_user": u})
|
|
}
|
|
b, _ := json.Marshal(map[string]any{"data": map[string]any{"connections": conns, "total": len(conns)}})
|
|
return b
|
|
}
|
|
|
|
func TestRebuildAsksConnectionsWithAuth(t *testing.T) { // 1135/AC2
|
|
w, o, _ := setup()
|
|
var body []byte
|
|
err := w.Rebuild(func(b []byte) ([]byte, error) {
|
|
body = b
|
|
return connzAnswer("node.ace", "ace.mesh-delivery-view", "ace.mesh-delivery-view", "ace.gitea"), nil
|
|
})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if string(body) != `{"auth":true}` || string(ConnzRequest()) != `{"auth":true}` {
|
|
t.Fatalf("request body = %s, want {\"auth\":true}", body)
|
|
}
|
|
if !o.presence["ace"] || !o.presence["part.mesh-delivery-view.ace"] || len(o.presence) != 2 {
|
|
t.Fatalf("presence = %v", o.presence)
|
|
}
|
|
// Two connections of the part: one closing leaves it online.
|
|
w.Disconnect("ace.mesh-delivery-view")
|
|
if !o.presence["part.mesh-delivery-view.ace"] {
|
|
t.Fatal("the part went offline with one connection still open")
|
|
}
|
|
}
|
|
|
|
func TestRebuildTakesThePresenceOfWhoIsGone(t *testing.T) { // 1135/AC2
|
|
w, o, c := setup()
|
|
w.SeedPresence("ace", true, t0.Add(-time.Hour))
|
|
w.SeedPresence("g14", true, t0.Add(-time.Hour))
|
|
if err := w.Rebuild(func([]byte) ([]byte, error) { return connzAnswer("node.g14"), nil }); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if online, ok := o.presence["ace"]; !ok || online {
|
|
t.Fatalf("ace was online in the bucket and is not among the connections: %v", o.presence)
|
|
}
|
|
if _, ok := o.presence["g14"]; ok {
|
|
t.Fatalf("g14 is online and was online: nothing to write, got %v", o.presence)
|
|
}
|
|
k := pulseKeyOf(t, presenceNS, "ace")
|
|
if o.ttls[k] != 120*time.Second {
|
|
t.Fatalf("ttl of the pulse of ace gone = %v, want 120s", o.ttls[k])
|
|
}
|
|
_ = c
|
|
if err := w.Rebuild(func([]byte) ([]byte, error) { return nil, errors.New("no answer") }); err == nil {
|
|
t.Fatal("a connections request that fails must be an error")
|
|
}
|
|
}
|
|
|
|
func TestRebuildPulsesFromCurrentKeys(t *testing.T) { // 1135/AC2
|
|
w, o, c := setup()
|
|
c.t = t0.Add(30 * time.Second)
|
|
w.Check("o.r.7", checkValue(t0, "abc"), t0.Add(10*time.Second), Put)
|
|
w.SeedPresence("ace", false, t0)
|
|
w.Reconcile()
|
|
if err := w.Rebuild(func([]byte) ([]byte, error) { return connzAnswer(), nil }); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
for _, k := range []string{pulseKeyOf(t, checksKey, "o.r.7"), pulseKeyOf(t, presenceNS, "ace")} {
|
|
if _, ok := o.pulses[k]; !ok {
|
|
t.Errorf("no pulse %s; have %v", k, o.pulses)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestReconcileDropsPulsesNothingMirrorsAnyMore(t *testing.T) { // 1135/AC2
|
|
w, o, _ := setup()
|
|
stale := pulseKeyOf(t, checksKey, "o.r.gone")
|
|
live := pulseKeyOf(t, checksKey, "o.r.7")
|
|
w.SeedPulse(stale)
|
|
w.SeedPulse(live)
|
|
w.Check("o.r.7", checkValue(t0, "abc"), t0, Put)
|
|
w.Reconcile()
|
|
if !slices.Equal(o.pulseDeletes, []string{stale}) {
|
|
t.Fatalf("deleted = %v, want only %s", o.pulseDeletes, stale)
|
|
}
|
|
if len(o.stuck) != 0 {
|
|
t.Fatalf("a stale pulse must not become stuck: %v", o.stuck)
|
|
}
|
|
}
|
|
|
|
func TestPulseTimeToLiveIsTheBoundMinusTheAge(t *testing.T) { // 1135/AC3
|
|
w, o, c := setup()
|
|
c.t = t0.Add(30 * time.Second)
|
|
w.Check("o.r.7", checkValue(t0, "abc"), t0, Put) // the key found at start, last renewed 30s ago
|
|
k := pulseKeyOf(t, checksKey, "o.r.7")
|
|
if o.ttls[k] != 90*time.Second {
|
|
t.Fatalf("ttl = %v, want 90s (last renewed 30s ago)", o.ttls[k])
|
|
}
|
|
if p := o.pulses[k]; p.Bucket != checksKey || p.Key != "o.r.7" {
|
|
t.Fatalf("pulse = %+v", p)
|
|
}
|
|
w.Connect("node.ace")
|
|
w.Disconnect("node.ace")
|
|
if got := o.ttls[pulseKeyOf(t, presenceNS, "ace")]; got != 120*time.Second {
|
|
t.Fatalf("presence just gone offline: ttl = %v, want 120s", got)
|
|
}
|
|
}
|
|
|
|
func TestAMoveAtTheBoundIsStuckAtOnce(t *testing.T) { // 1135/AC3
|
|
w, o, c := setup()
|
|
c.t = t0.Add(120 * time.Second)
|
|
w.Check("o.r.7", checkValue(t0, "abc"), t0, Put)
|
|
if len(o.pulses) != 0 {
|
|
t.Fatalf("a pulse was written for a heartbeat 120s old: %v", o.pulses)
|
|
}
|
|
s, ok := o.stuck[stuckKeyOf(t, checksKey, "o.r.7")]
|
|
if !ok {
|
|
t.Fatalf("no stuck key; have %v", o.stuck)
|
|
}
|
|
if s.Moved || !s.Since.Equal(t0) || s.Waited != "the check of o.r.7 to end" {
|
|
t.Fatalf("stuck = %+v", s)
|
|
}
|
|
}
|
|
|
|
func TestWaitedInWords(t *testing.T) { // 1135/AC3
|
|
w, o, c := setup()
|
|
c.t = t0.Add(200 * time.Second)
|
|
w.Check("o.r.7", checkValue(t0, "abc"), t0, Put)
|
|
w.SeedPresence("ace", false, t0)
|
|
w.SeedPresence("part.mesh-delivery-view.ace", false, t0)
|
|
w.Reconcile()
|
|
_ = w.Rebuild(func([]byte) ([]byte, error) { return connzAnswer(), nil })
|
|
want := map[string]string{
|
|
stuckKeyOf(t, checksKey, "o.r.7"): "the check of o.r.7 to end",
|
|
stuckKeyOf(t, presenceNS, "ace"): "ace to come back online",
|
|
stuckKeyOf(t, presenceNS, "part.mesh-delivery-view.ace"): "mesh-delivery-view on ace to come back online",
|
|
}
|
|
for k, waited := range want {
|
|
if o.stuck[k].Waited != waited {
|
|
t.Errorf("%s waited = %q, want %q", k, o.stuck[k].Waited, waited)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestEveryHeartbeatRenewsThePulse(t *testing.T) { // 1135/AC3
|
|
w, o, c := setup()
|
|
k := pulseKeyOf(t, checksKey, "o.r.7")
|
|
w.Check("o.r.7", checkValue(t0, "abc"), t0, Put)
|
|
if o.ttls[k] != 120*time.Second {
|
|
t.Fatalf("ttl = %v, want the full 120s", o.ttls[k])
|
|
}
|
|
o.ttls[k] = 0
|
|
c.t = t0.Add(100 * time.Second)
|
|
w.Check("o.r.7", checkValue(t0, "abc"), c.t, Put) // the builder re-puts the key: same check, started long ago
|
|
if o.ttls[k] != 120*time.Second {
|
|
t.Fatalf("a heartbeat left the pulse as it was: ttl = %v, want the full 120s again", o.ttls[k])
|
|
}
|
|
if len(o.stuck) != 0 {
|
|
t.Fatalf("a check that heartbeats is stuck: %v", o.stuck)
|
|
}
|
|
// The same revision heard twice changes nothing.
|
|
o.ttls[k] = 0
|
|
w.Check("o.r.7", checkValue(t0, "abc"), c.t, Put)
|
|
if o.ttls[k] != 0 {
|
|
t.Fatal("one revision rewrote the pulse twice")
|
|
}
|
|
}
|
|
|
|
func TestMirroredMovesDeleteThePulse(t *testing.T) { // 1135/AC4
|
|
w, o, c := setup()
|
|
c.t = t0.Add(10 * time.Second)
|
|
w.Check("o.r.7", checkValue(t0, "abc"), t0, Put)
|
|
w.Connect("node.ace")
|
|
w.Disconnect("node.ace")
|
|
w.Check("o.r.7", nil, c.t, Delete) // the check ended
|
|
w.Connect("node.ace")
|
|
if len(o.pulses) != 0 {
|
|
t.Fatalf("pulses left: %v", o.pulses)
|
|
}
|
|
if len(o.pulseDeletes) != 2 {
|
|
t.Fatalf("deletes = %v, want 2", o.pulseDeletes)
|
|
}
|
|
if len(o.stuck) != 0 {
|
|
t.Fatalf("a move before expiry wrote stuck: %v", o.stuck)
|
|
}
|
|
}
|
|
|
|
func TestACheckWhoseBuilderStopsIsStuckFromItsLastHeartbeat(t *testing.T) { // 1135/AC4
|
|
w, o, c := setup()
|
|
w.Check("o.r.7", checkValue(t0, "abc"), t0, Put)
|
|
c.t = t0.Add(50 * time.Second)
|
|
w.Check("o.r.7", checkValue(t0, "abc"), c.t, Put) // the last heartbeat
|
|
c.t = t0.Add(170 * time.Second)
|
|
w.Pulse(pulseKeyOf(t, checksKey, "o.r.7"), Expired, c.t)
|
|
s := o.stuck[stuckKeyOf(t, checksKey, "o.r.7")]
|
|
if !s.Since.Equal(t0.Add(50 * time.Second)) {
|
|
t.Fatalf("stuck = %+v, want since the last heartbeat", s)
|
|
}
|
|
}
|
|
|
|
func TestAStuckKeyIsWrittenMovedNeverDeleted(t *testing.T) { // 1135/AC4
|
|
w, o, c := setup()
|
|
w.Check("o.r.7", checkValue(t0, "abc"), t0, Put)
|
|
pk := pulseKeyOf(t, checksKey, "o.r.7")
|
|
c.t = t0.Add(120 * time.Second)
|
|
w.Pulse(pk, Expired, c.t)
|
|
sk := stuckKeyOf(t, checksKey, "o.r.7")
|
|
if o.stuck[sk].Moved {
|
|
t.Fatal("stuck written as moved at expiry")
|
|
}
|
|
c.t = t0.Add(300 * time.Second)
|
|
w.Check("o.r.7", nil, c.t, Delete)
|
|
s := o.stuck[sk]
|
|
if !s.Moved || !s.Since.Equal(t0) || s.Waited != "the check of o.r.7 to end" {
|
|
t.Fatalf("stuck after the move = %+v, want the same key moved", s)
|
|
}
|
|
n := o.stuckWrites
|
|
w.Check("o.r.7", nil, c.t, Delete)
|
|
if o.stuckWrites != n {
|
|
t.Fatal("a stuck key already moved was written again")
|
|
}
|
|
}
|
|
|
|
func TestAHeartbeatAfterStuckMovesIt(t *testing.T) { // 1135/AC4
|
|
w, o, c := setup()
|
|
w.Check("o.r.7", checkValue(t0, "abc"), t0, Put)
|
|
c.t = t0.Add(120 * time.Second)
|
|
w.Pulse(pulseKeyOf(t, checksKey, "o.r.7"), Expired, c.t)
|
|
w.Check("o.r.7", checkValue(t0, "abc"), c.t, Put) // the builder is back
|
|
if !o.stuck[stuckKeyOf(t, checksKey, "o.r.7")].Moved {
|
|
t.Fatal("a check heartbeating again is still stuck")
|
|
}
|
|
}
|
|
|
|
func TestPresenceStuckThenBackOnline(t *testing.T) { // 1135/AC4
|
|
w, o, c := setup()
|
|
w.Connect("node.ace")
|
|
w.Disconnect("node.ace")
|
|
c.t = t0.Add(120 * time.Second)
|
|
w.Pulse(pulseKeyOf(t, presenceNS, "ace"), Expired, c.t)
|
|
sk := stuckKeyOf(t, presenceNS, "ace")
|
|
if s := o.stuck[sk]; s.Moved || !s.Since.Equal(t0) {
|
|
t.Fatalf("stuck = %+v", s)
|
|
}
|
|
w.Connect("node.ace")
|
|
if !o.stuck[sk].Moved {
|
|
t.Fatalf("stuck = %+v, want moved once ace is back", o.stuck[sk])
|
|
}
|
|
}
|
|
|
|
func TestExpiryMarkerWritesStuck(t *testing.T) { // 1135/AC5
|
|
w, o, c := setup()
|
|
w.Check("o.r.7", checkValue(t0, "abc"), t0, Put)
|
|
c.t = t0.Add(120 * time.Second)
|
|
w.Pulse(pulseKeyOf(t, checksKey, "o.r.7"), Expired, c.t)
|
|
s, ok := o.stuck[stuckKeyOf(t, checksKey, "o.r.7")]
|
|
if !ok {
|
|
t.Fatalf("no stuck key; have %v", o.stuck)
|
|
}
|
|
if s.Bucket != checksKey || s.Key != "o.r.7" || s.Waited != "the check of o.r.7 to end" || !s.Since.Equal(t0) || s.Moved {
|
|
t.Fatalf("stuck = %+v", s)
|
|
}
|
|
}
|
|
|
|
func TestOwnDeleteOfAPulseWritesNothing(t *testing.T) { // 1135/AC5
|
|
w, o, c := setup()
|
|
w.Check("o.r.7", checkValue(t0, "abc"), t0, Put)
|
|
pk := pulseKeyOf(t, checksKey, "o.r.7")
|
|
w.Check("o.r.7", nil, c.t, Delete)
|
|
w.Pulse(pk, Delete, c.t) // the delete the watchman made, heard on its own watch
|
|
w.Pulse(pk, Expired, c.t)
|
|
w.Pulse("mesh-delivery-watchman_pulse.unknown.key", Expired, c.t)
|
|
if len(o.stuck) != 0 {
|
|
t.Fatalf("stuck written for a pulse the watchman deleted or never made: %v", o.stuck)
|
|
}
|
|
}
|
|
|
|
func TestTheBoundCanBeShortenedForATest(t *testing.T) { // 1135/AC8
|
|
w, o, c := setup()
|
|
w.SetBound(2 * time.Second)
|
|
c.t = t0.Add(500 * time.Millisecond)
|
|
w.Check("o.r.7", checkValue(t0, "abc"), t0, Put)
|
|
if got := o.ttls[pulseKeyOf(t, checksKey, "o.r.7")]; got != 1500*time.Millisecond {
|
|
t.Fatalf("ttl = %v, want 1.5s", got)
|
|
}
|
|
c.t = t0.Add(3 * time.Second)
|
|
w.Check("o.r.8", checkValue(t0, "abc"), t0, Put)
|
|
if _, ok := o.stuck[stuckKeyOf(t, checksKey, "o.r.8")]; !ok {
|
|
t.Fatalf("a heartbeat older than the bound is stuck at once; have %v", o.stuck)
|
|
}
|
|
}
|