Its states presence, pulse and stuck and its first views were named from the constant WatchmanModule; they are now `<cred.Module>_<state>` (watch.BucketsOf), and the watchman's stuck keys of presence take its own presence bucket's name. Builds: design 47 G A-296 Claude-Session: https://claude.ai/code/session_01NcbgfGKaxw984XyAPfLm4h
759 lines
26 KiB
Go
759 lines
26 KiB
Go
package watch
|
|
|
|
import (
|
|
"encoding/json"
|
|
"errors"
|
|
"slices"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshwatchdogsclient"
|
|
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
|
|
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshwantsclient"
|
|
)
|
|
|
|
type fakeOut struct {
|
|
presence map[string]bool
|
|
presenceWrite int
|
|
pulses map[string]meshwatchdogsclient.Pulse
|
|
ttls map[string]time.Duration
|
|
pulseDeletes []string
|
|
stuck map[string]meshwatchdogsclient.Stuck
|
|
stuckWrites int
|
|
findings map[string]Finding
|
|
findingWrites int
|
|
said []said
|
|
}
|
|
|
|
type said struct{ subject, id string }
|
|
|
|
func (o *fakeOut) Say(subject string, _ []byte, id string) error {
|
|
o.said = append(o.said, said{subject, id})
|
|
return nil
|
|
}
|
|
|
|
func newOut() *fakeOut {
|
|
return &fakeOut{presence: map[string]bool{}, pulses: map[string]meshwatchdogsclient.Pulse{}, ttls: map[string]time.Duration{}, stuck: map[string]meshwatchdogsclient.Stuck{},
|
|
findings: map[string]Finding{}}
|
|
}
|
|
|
|
func (o *fakeOut) PutFinding(key string, value []byte) error {
|
|
f, err := DecodeFinding(value)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if f.Key() != key {
|
|
return errors.New("a finding written under another key than its own: " + key)
|
|
}
|
|
o.findings[key] = f
|
|
o.findingWrites++
|
|
return nil
|
|
}
|
|
|
|
func (o *fakeOut) DeleteFinding(key string) error {
|
|
delete(o.findings, key)
|
|
return nil
|
|
}
|
|
|
|
func (o *fakeOut) PutPresence(key string, online bool) error {
|
|
o.presence[key] = online
|
|
o.presenceWrite++
|
|
return nil
|
|
}
|
|
|
|
func (o *fakeOut) PutPulse(key string, value []byte, ttl time.Duration) error {
|
|
p, err := meshwatchdogsclient.DecodePulse(value)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
o.pulses[key], o.ttls[key] = p, ttl
|
|
return nil
|
|
}
|
|
|
|
func (o *fakeOut) DeletePulse(key string) error {
|
|
delete(o.pulses, key)
|
|
o.pulseDeletes = append(o.pulseDeletes, key)
|
|
return nil
|
|
}
|
|
|
|
func (o *fakeOut) PutStuck(key string, value []byte) error {
|
|
s, err := meshwatchdogsclient.DecodeStuck(value)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
o.stuck[key] = s
|
|
o.stuckWrites++
|
|
return nil
|
|
}
|
|
|
|
var t0 = time.Date(2026, 10, 18, 12, 0, 0, 0, time.UTC)
|
|
|
|
type clock struct{ t time.Time }
|
|
|
|
func (c *clock) now() time.Time { return c.t }
|
|
|
|
func setup() (*Watchman, *fakeOut, *clock) {
|
|
o, c := newOut(), &clock{t: t0}
|
|
w := New(testModule, o, c.now)
|
|
w.SetStepBound(meshwatchdogsclient.PulseMoveBound) // the older tests move a step as they move any key: stuck at the pulse's bound
|
|
return w, o, c
|
|
}
|
|
|
|
// checkValue is a check's key as the controller's builder writes it (mesh-controller internal/link/checks_nats.go
|
|
// CheckRunning, kept by keepRunning on every heartbeat).
|
|
func checkValue(started time.Time, head string) []byte {
|
|
b, _ := json.Marshal(map[string]any{"head": head, "trunk": "def456", "builder": "n1", "build": "b9", "started": started})
|
|
return b
|
|
}
|
|
|
|
const (
|
|
checksKey = "SEAT_NODE_BUILD_GO"
|
|
testModule = "a-watchman"
|
|
presenceNS = testModule + "_presence"
|
|
)
|
|
|
|
// askBody is the body of a build seat's step-progress, ask-taken or step-ended for an ask at a time.
|
|
func askBody(id string, at time.Time) []byte {
|
|
b, err := json.Marshal(map[string]any{"ask": id, "asker": "a1", "at": at, "phase": "build"})
|
|
if err != nil {
|
|
panic(err)
|
|
}
|
|
return b
|
|
}
|
|
|
|
// askTaken, askProgress and askEnded are what the holder of the Go build seat says on node n1 (design 47 G §496, §602).
|
|
func askTaken(w *Watchman, id string, at time.Time) {
|
|
w.AskTaken("node-build-go", "n1", askBody(id, at))
|
|
}
|
|
func askProgress(w *Watchman, id string, at time.Time) {
|
|
w.StepProgress("node-build-go", "n1", askBody(id, at))
|
|
}
|
|
func askEnded(w *Watchman, id string) { w.AskEnded("node-build-go", askBody(id, t0)) }
|
|
|
|
func pulseKeyOf(t *testing.T, bucket, key string) string {
|
|
t.Helper()
|
|
k, err := meshwatchdogsclient.PulseKey(bucket, key)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return k
|
|
}
|
|
|
|
func stuckKeyOf(t *testing.T, bucket, key string) string {
|
|
t.Helper()
|
|
k, err := meshwatchdogsclient.StuckKey(bucket, key)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return k
|
|
}
|
|
|
|
func TestPresenceFromEventsWrittenOnlyWhenItDiffers(t *testing.T) { // 1135/AC1
|
|
t.Run("from events", presenceFromEvents)
|
|
t.Run("written only when it differs", presenceWrittenOnlyWhenItDiffers)
|
|
}
|
|
|
|
func presenceFromEvents(t *testing.T) { // 1135/AC1
|
|
w, o, _ := setup()
|
|
w.Connect("node.ace")
|
|
w.Connect("ace.mesh-delivery-view")
|
|
w.Connect("ace.gitea")
|
|
want := map[string]bool{"ace": true, "part.mesh-delivery-view.ace": true}
|
|
if len(o.presence) != 2 || o.presence["ace"] != true || o.presence["part.mesh-delivery-view.ace"] != true {
|
|
t.Fatalf("presence = %v, want %v and nothing for another module's user", o.presence, want)
|
|
}
|
|
w.Disconnect("node.ace")
|
|
w.Disconnect("ace.gitea")
|
|
if online, ok := o.presence["ace"]; !ok || online {
|
|
t.Fatalf("presence = %v, want ace offline", o.presence)
|
|
}
|
|
if len(o.presence) != 2 {
|
|
t.Fatalf("another module's user wrote presence: %v", o.presence)
|
|
}
|
|
}
|
|
|
|
func presenceWrittenOnlyWhenItDiffers(t *testing.T) { // 1135/AC1
|
|
w, o, _ := setup()
|
|
w.Connect("node.ace")
|
|
w.Connect("node.ace")
|
|
if o.presenceWrite != 1 {
|
|
t.Fatalf("writes = %d, want 1 (the second connection changes nothing)", o.presenceWrite)
|
|
}
|
|
w.Disconnect("node.ace")
|
|
if !o.presence["ace"] || o.presenceWrite != 1 {
|
|
t.Fatalf("one of two connections closed: ace must stay online without a write, got %v, %d writes", o.presence, o.presenceWrite)
|
|
}
|
|
w.Disconnect("node.ace")
|
|
if o.presence["ace"] || o.presenceWrite != 2 {
|
|
t.Fatalf("both closed: ace offline, got %v, %d writes", o.presence, o.presenceWrite)
|
|
}
|
|
w.Disconnect("node.ace") // a stray event
|
|
if o.presenceWrite != 2 {
|
|
t.Fatalf("a disconnect of a user already offline wrote %d times", o.presenceWrite)
|
|
}
|
|
}
|
|
|
|
func TestParseEvent(t *testing.T) { // 1135/AC1
|
|
user, err := ParseEvent([]byte(`{"type":"io.nats.server.advisory.v1.client_connect","client":{"acc":"MESH","user":"node.ace","id":4}}`))
|
|
if err != nil || user != "node.ace" {
|
|
t.Fatalf("user = %q, %v", user, err)
|
|
}
|
|
if _, err := ParseEvent([]byte(`not json`)); err == nil {
|
|
t.Fatal("an event that is not JSON must be refused")
|
|
}
|
|
if user, err := ParseEvent([]byte(`{"client":{}}`)); err != nil || user != "" {
|
|
t.Fatalf("an event with no user = %q, %v; want empty and no error", user, err)
|
|
}
|
|
}
|
|
|
|
func connzAnswer(users ...string) []byte {
|
|
var conns []map[string]string
|
|
for _, u := range users {
|
|
conns = append(conns, map[string]string{"authorized_user": u})
|
|
}
|
|
b, _ := json.Marshal(map[string]any{"data": map[string]any{"connections": conns, "total": len(conns)}})
|
|
return b
|
|
}
|
|
|
|
func TestRebuildAsksConnectionsAndTakesWhoIsGone(t *testing.T) { // 1135/AC2
|
|
t.Run("asks the connections with auth", rebuildAsksConnectionsWithAuth)
|
|
t.Run("takes the presence of who is gone", rebuildTakesThePresenceOfWhoIsGone)
|
|
}
|
|
|
|
func rebuildAsksConnectionsWithAuth(t *testing.T) { // 1135/AC2
|
|
w, o, _ := setup()
|
|
var body []byte
|
|
err := w.Rebuild(func(b []byte) ([]byte, error) {
|
|
body = b
|
|
return connzAnswer("node.ace", "ace.mesh-delivery-view", "ace.mesh-delivery-view", "ace.gitea"), nil
|
|
})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if string(body) != `{"auth":true}` || string(ConnzRequest()) != `{"auth":true}` {
|
|
t.Fatalf("request body = %s, want {\"auth\":true}", body)
|
|
}
|
|
if !o.presence["ace"] || !o.presence["part.mesh-delivery-view.ace"] || len(o.presence) != 2 {
|
|
t.Fatalf("presence = %v", o.presence)
|
|
}
|
|
// Two connections of the part: one closing leaves it online.
|
|
w.Disconnect("ace.mesh-delivery-view")
|
|
if !o.presence["part.mesh-delivery-view.ace"] {
|
|
t.Fatal("the part went offline with one connection still open")
|
|
}
|
|
}
|
|
|
|
func rebuildTakesThePresenceOfWhoIsGone(t *testing.T) { // 1135/AC2
|
|
w, o, c := setup()
|
|
w.SeedPresence("ace", true, t0.Add(-time.Hour))
|
|
w.SeedPresence("g14", true, t0.Add(-time.Hour))
|
|
if err := w.Rebuild(func([]byte) ([]byte, error) { return connzAnswer("node.g14"), nil }); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if online, ok := o.presence["ace"]; !ok || online {
|
|
t.Fatalf("ace was online in the bucket and is not among the connections: %v", o.presence)
|
|
}
|
|
if _, ok := o.presence["g14"]; ok {
|
|
t.Fatalf("g14 is online and was online: nothing to write, got %v", o.presence)
|
|
}
|
|
k := pulseKeyOf(t, presenceNS, "ace")
|
|
if o.ttls[k] != 120*time.Second {
|
|
t.Fatalf("ttl of the pulse of ace gone = %v, want 120s", o.ttls[k])
|
|
}
|
|
_ = c
|
|
if err := w.Rebuild(func([]byte) ([]byte, error) { return nil, errors.New("no answer") }); err == nil {
|
|
t.Fatal("a connections request that fails must be an error")
|
|
}
|
|
}
|
|
|
|
func TestRebuildPulsesFromCurrentKeys(t *testing.T) { // 1135/AC2
|
|
w, o, c := setup()
|
|
c.t = t0.Add(30 * time.Second)
|
|
askProgress(w, "askid1", t0.Add(10*time.Second))
|
|
w.SeedPresence("ace", false, t0)
|
|
w.Reconcile()
|
|
if err := w.Rebuild(func([]byte) ([]byte, error) { return connzAnswer(), nil }); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
for _, k := range []string{pulseKeyOf(t, checksKey, "askid1"), pulseKeyOf(t, presenceNS, "ace")} {
|
|
if _, ok := o.pulses[k]; !ok {
|
|
t.Errorf("no pulse %s; have %v", k, o.pulses)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestReconcileDropsPulsesNothingMirrorsAnyMore(t *testing.T) { // 1135/AC2
|
|
w, o, _ := setup()
|
|
stale := pulseKeyOf(t, checksKey, "gone")
|
|
live := pulseKeyOf(t, checksKey, "askid1")
|
|
w.SeedPulse(stale)
|
|
w.SeedPulse(live)
|
|
askProgress(w, "askid1", t0)
|
|
w.Reconcile()
|
|
if !slices.Equal(o.pulseDeletes, []string{stale}) {
|
|
t.Fatalf("deleted = %v, want only %s", o.pulseDeletes, stale)
|
|
}
|
|
if len(o.stuck) != 0 {
|
|
t.Fatalf("a stale pulse must not become stuck: %v", o.stuck)
|
|
}
|
|
}
|
|
|
|
func TestPulseTimeToLiveIsTheBoundMinusTheAge(t *testing.T) { // 1135/AC3
|
|
w, o, c := setup()
|
|
c.t = t0.Add(30 * time.Second)
|
|
askProgress(w, "askid1", t0) // the key found at start, last renewed 30s ago
|
|
k := pulseKeyOf(t, checksKey, "askid1")
|
|
if o.ttls[k] != 90*time.Second {
|
|
t.Fatalf("ttl = %v, want 90s (last renewed 30s ago)", o.ttls[k])
|
|
}
|
|
if p := o.pulses[k]; p.Bucket != checksKey || p.Key != "askid1" {
|
|
t.Fatalf("pulse = %+v", p)
|
|
}
|
|
w.Connect("node.ace")
|
|
w.Disconnect("node.ace")
|
|
if got := o.ttls[pulseKeyOf(t, presenceNS, "ace")]; got != 120*time.Second {
|
|
t.Fatalf("presence just gone offline: ttl = %v, want 120s", got)
|
|
}
|
|
}
|
|
|
|
func TestAMoveAtTheBoundIsStuckAtOnce(t *testing.T) { // 1135/AC3
|
|
w, o, c := setup()
|
|
c.t = t0.Add(120 * time.Second)
|
|
askProgress(w, "askid1", t0)
|
|
if len(o.pulses) != 0 {
|
|
t.Fatalf("a pulse was written for a heartbeat 120s old: %v", o.pulses)
|
|
}
|
|
s, ok := o.stuck[stuckKeyOf(t, checksKey, "askid1")]
|
|
if !ok {
|
|
t.Fatalf("no stuck key; have %v", o.stuck)
|
|
}
|
|
if s.Moved || !s.Since.Equal(t0) || s.Waited != "the ask askid1 to end on n1" {
|
|
t.Fatalf("stuck = %+v", s)
|
|
}
|
|
}
|
|
|
|
func TestWaitedInWords(t *testing.T) { // 1135/AC3
|
|
w, o, c := setup()
|
|
c.t = t0.Add(200 * time.Second)
|
|
askProgress(w, "askid1", t0)
|
|
w.SeedPresence("ace", false, t0)
|
|
w.SeedPresence("part.mesh-delivery-view.ace", false, t0)
|
|
w.Reconcile()
|
|
_ = w.Rebuild(func([]byte) ([]byte, error) { return connzAnswer(), nil })
|
|
want := map[string]string{
|
|
stuckKeyOf(t, checksKey, "askid1"): "the ask askid1 to end on n1",
|
|
stuckKeyOf(t, presenceNS, "ace"): "ace to come back online",
|
|
stuckKeyOf(t, presenceNS, "part.mesh-delivery-view.ace"): "mesh-delivery-view on ace to come back online",
|
|
}
|
|
for k, waited := range want {
|
|
if o.stuck[k].Waited != waited {
|
|
t.Errorf("%s waited = %q, want %q", k, o.stuck[k].Waited, waited)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestEveryHeartbeatRenewsThePulse(t *testing.T) { // 1135/AC3
|
|
w, o, c := setup()
|
|
k := pulseKeyOf(t, checksKey, "askid1")
|
|
askProgress(w, "askid1", t0)
|
|
if o.ttls[k] != 120*time.Second {
|
|
t.Fatalf("ttl = %v, want the full 120s", o.ttls[k])
|
|
}
|
|
o.ttls[k] = 0
|
|
c.t = t0.Add(100 * time.Second)
|
|
askProgress(w, "askid1", c.t) // the builder re-puts the key: same check, started long ago
|
|
if o.ttls[k] != 120*time.Second {
|
|
t.Fatalf("a heartbeat left the pulse as it was: ttl = %v, want the full 120s again", o.ttls[k])
|
|
}
|
|
if len(o.stuck) != 0 {
|
|
t.Fatalf("a check that heartbeats is stuck: %v", o.stuck)
|
|
}
|
|
// The same revision heard twice changes nothing.
|
|
o.ttls[k] = 0
|
|
askProgress(w, "askid1", c.t)
|
|
if o.ttls[k] != 0 {
|
|
t.Fatal("one revision rewrote the pulse twice")
|
|
}
|
|
}
|
|
|
|
func TestMirroredMovesDeleteThePulse(t *testing.T) { // 1135/AC4
|
|
w, o, c := setup()
|
|
c.t = t0.Add(10 * time.Second)
|
|
askProgress(w, "askid1", t0)
|
|
w.Connect("node.ace")
|
|
w.Disconnect("node.ace")
|
|
askEnded(w, "askid1") // the check ended
|
|
w.Connect("node.ace")
|
|
if len(o.pulses) != 0 {
|
|
t.Fatalf("pulses left: %v", o.pulses)
|
|
}
|
|
if len(o.pulseDeletes) != 2 {
|
|
t.Fatalf("deletes = %v, want 2", o.pulseDeletes)
|
|
}
|
|
if len(o.stuck) != 0 {
|
|
t.Fatalf("a move before expiry wrote stuck: %v", o.stuck)
|
|
}
|
|
}
|
|
|
|
func TestACheckWhoseBuilderStopsIsStuckFromItsLastHeartbeat(t *testing.T) { // 1135/AC4
|
|
w, o, c := setup()
|
|
askProgress(w, "askid1", t0)
|
|
c.t = t0.Add(50 * time.Second)
|
|
askProgress(w, "askid1", c.t) // the last heartbeat
|
|
c.t = t0.Add(170 * time.Second)
|
|
w.Pulse(pulseKeyOf(t, checksKey, "askid1"), Expired, c.t)
|
|
s := o.stuck[stuckKeyOf(t, checksKey, "askid1")]
|
|
if !s.Since.Equal(t0.Add(50 * time.Second)) {
|
|
t.Fatalf("stuck = %+v, want since the last heartbeat", s)
|
|
}
|
|
}
|
|
|
|
func TestAStuckKeyIsWrittenMovedNeverDeleted(t *testing.T) { // 1135/AC4
|
|
w, o, c := setup()
|
|
askProgress(w, "askid1", t0)
|
|
pk := pulseKeyOf(t, checksKey, "askid1")
|
|
c.t = t0.Add(120 * time.Second)
|
|
w.Pulse(pk, Expired, c.t)
|
|
sk := stuckKeyOf(t, checksKey, "askid1")
|
|
if o.stuck[sk].Moved {
|
|
t.Fatal("stuck written as moved at expiry")
|
|
}
|
|
c.t = t0.Add(300 * time.Second)
|
|
askEnded(w, "askid1")
|
|
s := o.stuck[sk]
|
|
if !s.Moved || !s.Since.Equal(t0) || s.Waited != "the ask askid1 to end on n1" {
|
|
t.Fatalf("stuck after the move = %+v, want the same key moved", s)
|
|
}
|
|
n := o.stuckWrites
|
|
askEnded(w, "askid1")
|
|
if o.stuckWrites != n {
|
|
t.Fatal("a stuck key already moved was written again")
|
|
}
|
|
}
|
|
|
|
func TestAnAskEndedAfterStuckMovesIt(t *testing.T) { // 1135/AC4
|
|
w, o, c := setup()
|
|
askProgress(w, "askid1", t0)
|
|
c.t = t0.Add(120 * time.Second)
|
|
w.Pulse(pulseKeyOf(t, checksKey, "askid1"), Expired, c.t)
|
|
askEnded(w, "askid1") // the ask ended at last
|
|
if !o.stuck[stuckKeyOf(t, checksKey, "askid1")].Moved {
|
|
t.Fatal("an ask taken after it was stuck is still stuck")
|
|
}
|
|
}
|
|
|
|
func TestPresenceStuckThenBackOnline(t *testing.T) { // 1135/AC4
|
|
w, o, c := setup()
|
|
w.Connect("node.ace")
|
|
w.Disconnect("node.ace")
|
|
c.t = t0.Add(120 * time.Second)
|
|
w.Pulse(pulseKeyOf(t, presenceNS, "ace"), Expired, c.t)
|
|
sk := stuckKeyOf(t, presenceNS, "ace")
|
|
if s := o.stuck[sk]; s.Moved || !s.Since.Equal(t0) {
|
|
t.Fatalf("stuck = %+v", s)
|
|
}
|
|
w.Connect("node.ace")
|
|
if !o.stuck[sk].Moved {
|
|
t.Fatalf("stuck = %+v, want moved once ace is back", o.stuck[sk])
|
|
}
|
|
}
|
|
|
|
func TestOnlyTheExpiryOfItsOwnPulseWritesStuck(t *testing.T) { // 1135/AC5
|
|
t.Run("the expiry marker writes stuck", expiryMarkerWritesStuck)
|
|
t.Run("its own delete of a pulse writes nothing", ownDeleteOfAPulseWritesNothing)
|
|
}
|
|
|
|
func expiryMarkerWritesStuck(t *testing.T) { // 1135/AC5
|
|
w, o, c := setup()
|
|
askProgress(w, "askid1", t0)
|
|
c.t = t0.Add(120 * time.Second)
|
|
w.Pulse(pulseKeyOf(t, checksKey, "askid1"), Expired, c.t)
|
|
s, ok := o.stuck[stuckKeyOf(t, checksKey, "askid1")]
|
|
if !ok {
|
|
t.Fatalf("no stuck key; have %v", o.stuck)
|
|
}
|
|
if s.Bucket != checksKey || s.Key != "askid1" || s.Waited != "the ask askid1 to end on n1" || !s.Since.Equal(t0) || s.Moved {
|
|
t.Fatalf("stuck = %+v", s)
|
|
}
|
|
}
|
|
|
|
func ownDeleteOfAPulseWritesNothing(t *testing.T) { // 1135/AC5
|
|
w, o, c := setup()
|
|
askProgress(w, "askid1", t0)
|
|
pk := pulseKeyOf(t, checksKey, "askid1")
|
|
askEnded(w, "askid1")
|
|
w.Pulse(pk, Delete, c.t) // the delete the watchman made, heard on its own watch
|
|
w.Pulse(pk, Expired, c.t)
|
|
w.Pulse("mesh-delivery-watchman_pulse.unknown.key", Expired, c.t)
|
|
if len(o.stuck) != 0 {
|
|
t.Fatalf("stuck written for a pulse the watchman deleted or never made: %v", o.stuck)
|
|
}
|
|
}
|
|
|
|
func TestTheBoundCanBeShortenedForATest(t *testing.T) { // 1135/AC8
|
|
w, o, c := setup()
|
|
w.SetBound(2 * time.Second)
|
|
c.t = t0.Add(500 * time.Millisecond)
|
|
askProgress(w, "askid1", t0)
|
|
if got := o.ttls[pulseKeyOf(t, checksKey, "askid1")]; got != 1500*time.Millisecond {
|
|
t.Fatalf("ttl = %v, want 1.5s", got)
|
|
}
|
|
c.t = t0.Add(3 * time.Second)
|
|
askProgress(w, "askid2", t0)
|
|
if _, ok := o.stuck[stuckKeyOf(t, checksKey, "askid2")]; !ok {
|
|
t.Fatalf("a heartbeat older than the bound is stuck at once; have %v", o.stuck)
|
|
}
|
|
}
|
|
|
|
func TestAnAskTakenByANodeHasAPulseFromWhenItStartedAndEndsWithTheStep(t *testing.T) { // 1908, 1909
|
|
w, o, c := setup()
|
|
c.t = t0.Add(10 * time.Second)
|
|
askProgress(w, "askid1", t0)
|
|
if len(o.pulses) != 1 {
|
|
t.Fatalf("pulses = %v, want the taken ask's", o.pulses)
|
|
}
|
|
w2, o2, c2 := setup()
|
|
c2.t = t0.Add(10 * time.Second)
|
|
askTaken(w2, "askid1", t0)
|
|
askEnded(w2, "askid1")
|
|
if len(o2.pulses) != 0 && o2.pulses[pulseKeyOf(t, checksKey, "askid1")].Key != "" {
|
|
t.Errorf("the pulse of an ended ask stands: %v", o2.pulses)
|
|
}
|
|
_ = c2
|
|
_ = c
|
|
}
|
|
|
|
func TestARunningAskFoundAtStartGetsItsPulse(t *testing.T) { // 1909
|
|
w, o, c := setup()
|
|
c.t = t0.Add(30 * time.Second)
|
|
askProgress(w, "askid1", t0) // what the steps held when the watchman started
|
|
p, ok := o.pulses[pulseKeyOf(t, checksKey, "askid1")]
|
|
if !ok || p.Key != "askid1" {
|
|
t.Fatalf("pulses = %v", o.pulses)
|
|
}
|
|
if got := o.ttls[pulseKeyOf(t, checksKey, "askid1")]; got <= 0 || got >= 120*time.Second {
|
|
t.Errorf("ttl = %v, want the bound less its 30s", got)
|
|
}
|
|
}
|
|
|
|
func TestAnEventThatDoesNotReadOrIsNoBuildSeatsIsLeft(t *testing.T) { // 1908
|
|
w, o, _ := setup()
|
|
w.StepProgress("node-build-go", "n1", []byte(`{"v":99}`))
|
|
w.StepProgress("node-build-go", "n1", askBody("", t0))
|
|
w.AskTaken("node-build-go", "n1", []byte(`nope`))
|
|
w.AskTaken("mesh-wants", "n1", askBody("askid1", t0))
|
|
if len(o.pulses) != 0 || len(o.stuck) != 0 {
|
|
t.Errorf("pulses %v stuck %v", o.pulses, o.stuck)
|
|
}
|
|
}
|
|
|
|
func TestARunningStepThatSaysItsProgressIsNeverStuckAndOneWhoseHolderStopsIs(t *testing.T) { // design 47 G §496, §602
|
|
o, c := newOut(), &clock{t: t0}
|
|
w := New(testModule, o, c.now)
|
|
askTaken(w, "askid1", t0)
|
|
k := pulseKeyOf(t, checksKey, "askid1")
|
|
if got := o.ttls[k]; got != ProgressBound {
|
|
t.Errorf("ttl = %v, want the progress bound %v", got, ProgressBound)
|
|
}
|
|
// a 20-minute step: its progress each minute renews the clock
|
|
for m := 1; m <= 20; m++ {
|
|
c.t = t0.Add(time.Duration(m) * time.Minute)
|
|
askProgress(w, "askid1", c.t)
|
|
if o.ttls[k] != ProgressBound {
|
|
t.Fatalf("minute %d: ttl = %v, want the full bound again", m, o.ttls[k])
|
|
}
|
|
}
|
|
if len(o.stuck) != 0 {
|
|
t.Fatalf("a step saying its progress is stuck: %v", o.stuck)
|
|
}
|
|
// its holder stops: the pulse expires, and the ask is stuck from its last progress
|
|
c.t = t0.Add(26 * time.Minute)
|
|
w.Pulse(k, Expired, c.t)
|
|
s, ok := o.stuck[stuckKeyOf(t, checksKey, "askid1")]
|
|
if !ok || s.Moved || !s.Since.Equal(t0.Add(20*time.Minute)) {
|
|
t.Fatalf("stuck = %+v, %v", s, ok)
|
|
}
|
|
askEnded(w, "askid1")
|
|
if !o.stuck[stuckKeyOf(t, checksKey, "askid1")].Moved {
|
|
t.Fatal("the end of the ask left it stuck")
|
|
}
|
|
}
|
|
|
|
func TestARunningStepPastItsBoundIsStuck(t *testing.T) { // 1908
|
|
o, c := newOut(), &clock{t: t0}
|
|
w := New(testModule, o, c.now)
|
|
askProgress(w, "askid1", t0)
|
|
c.t = t0.Add(6 * time.Minute)
|
|
w.Pulse(pulseKeyOf(t, checksKey, "askid1"), Expired, c.t)
|
|
s, ok := o.stuck[stuckKeyOf(t, checksKey, "askid1")]
|
|
if !ok || s.Moved || !s.Since.Equal(t0) {
|
|
t.Fatalf("stuck = %+v, %v", s, ok)
|
|
}
|
|
// and one found past its bound at the watchman's start is stuck at once
|
|
o2, c2 := newOut(), &clock{t: t0.Add(time.Hour)}
|
|
askProgress(New(testModule, o2, c2.now), "askid1", t0)
|
|
if _, ok := o2.stuck[stuckKeyOf(t, checksKey, "askid1")]; !ok || len(o2.pulses) != 0 {
|
|
t.Errorf("stuck %v pulses %v", o2.stuck, o2.pulses)
|
|
}
|
|
}
|
|
|
|
// 1906/D6 (ADR 0539 §19): a want is a pulse owed by the node's agent from when the assigner wrote it, until the have names
|
|
// its fingerprint.
|
|
func wantValue(t *testing.T, fp string, removed bool) []byte {
|
|
t.Helper()
|
|
w := meshwantsclient.Want{Fp: fp, Composed: "sha256:" + strings.Repeat("c", 64), Removed: removed}
|
|
if removed {
|
|
w.Composed = ""
|
|
}
|
|
b, err := meshwantsclient.EncodeWant(w)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return b
|
|
}
|
|
|
|
// runningBody is the node-engine's `running` for a build of fingerprint fp (design 47 G §541).
|
|
func runningBody(fp string) []byte { return []byte(`{"fingerprint":"` + fp + `"}`) }
|
|
|
|
var (
|
|
fpOld = meshcatalogclient.SourcePrefix2 + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
|
|
fpNew = meshcatalogclient.SourcePrefix2 + "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"
|
|
)
|
|
|
|
func TestAWantIsAPulseOwedByTheNodeAndTheHaveNamingItEndsIt(t *testing.T) {
|
|
w, o, c := setup()
|
|
c.t = t0.Add(10 * time.Second)
|
|
w.Want("want.ace.web", wantValue(t, fpNew, false), Put, t0)
|
|
pk := pulseKeyOf(t, AssignerWantsBucket, "want.ace.web")
|
|
p, ok := o.pulses[pk]
|
|
if !ok || p.Key != "want.ace.web" || !strings.Contains(p.Owes, "ace") || !strings.Contains(p.Owes, "web") {
|
|
t.Fatalf("pulses = %v", o.pulses)
|
|
}
|
|
w.EngineRunning("ace", "web", runningBody(fpOld)) // the node still runs the old one
|
|
if _, ok := o.pulses[pk]; !ok {
|
|
t.Fatalf("a have of another fingerprint ended the pulse")
|
|
}
|
|
w.EngineRunning("ace", "web", runningBody(fpNew))
|
|
if _, ok := o.pulses[pk]; ok {
|
|
t.Errorf("the have naming the want's fingerprint left the pulse standing: %v", o.pulses)
|
|
}
|
|
if len(o.stuck) != 0 {
|
|
t.Errorf("stuck = %v", o.stuck)
|
|
}
|
|
}
|
|
|
|
func TestAHaveThatCameBeforeTheWantEndsItAtOnceAndARemovedWantHasNoPulse(t *testing.T) {
|
|
w, o, c := setup()
|
|
c.t = t0.Add(time.Second)
|
|
w.EngineRunning("ace", "web", runningBody(fpNew))
|
|
w.Want("want.ace.web", wantValue(t, fpNew, false), Put, t0)
|
|
w.Want("want.ace.mail", wantValue(t, "", true), Put, t0)
|
|
if len(o.pulses) != 0 {
|
|
t.Errorf("pulses = %v", o.pulses)
|
|
}
|
|
// the want moving on to a new fingerprint makes a new pulse; the key `want.ace.node` and a held key are left
|
|
w.Want("want.ace.web", wantValue(t, fpOld, false), Put, t0.Add(time.Second))
|
|
w.Want("want.ace.node", []byte(`{}`), Put, t0)
|
|
w.Want("held.web.bbbb", []byte(`{}`), Put, t0)
|
|
if len(o.pulses) != 1 {
|
|
t.Errorf("pulses = %v, want the web want's only", o.pulses)
|
|
}
|
|
}
|
|
|
|
func TestAWantNotAppliedByItsBoundIsStuckAndAKeyDeletedEndsIt(t *testing.T) {
|
|
o, c := newOut(), &clock{t: t0}
|
|
w := New(testModule, o, c.now)
|
|
w.Want("want.ace.web", wantValue(t, fpNew, false), Put, t0)
|
|
pk := pulseKeyOf(t, AssignerWantsBucket, "want.ace.web")
|
|
c.t = t0.Add(31 * time.Minute)
|
|
w.Pulse(pk, Expired, c.t)
|
|
if s, ok := o.stuck[stuckKeyOf(t, AssignerWantsBucket, "want.ace.web")]; !ok || s.Moved || !s.Since.Equal(t0) {
|
|
t.Fatalf("stuck = %v", o.stuck)
|
|
}
|
|
w2, o2, c2 := setup()
|
|
c2.t = t0.Add(time.Second)
|
|
w2.Want("want.ace.web", wantValue(t, fpNew, false), Put, t0)
|
|
w2.Want("want.ace.web", nil, Delete, t0)
|
|
if len(o2.pulses) != 0 {
|
|
t.Errorf("a deleted want left its pulse: %v", o2.pulses)
|
|
}
|
|
}
|
|
|
|
// Sweep 2026-10-30: a watched key that aged out of its bucket or was deleted while nobody heard leaves its stuck key; at the
|
|
// start's end the watchman closes it (moved, its finding cleared), but only for a bucket it read whole at that start.
|
|
func TestAStuckKeyWhoseKeyIsGoneIsClosedAtTheStart(t *testing.T) {
|
|
wantKey := "want.n1.mod-a"
|
|
for _, row := range []struct {
|
|
name string
|
|
bucket string
|
|
key string
|
|
standing func(w *Watchman) // what the start reads standing
|
|
readWhole bool
|
|
closed bool
|
|
late bool // read after the start's end (Awaits, ReadLate)
|
|
}{
|
|
{"a want deleted while nobody heard", AssignerWantsBucket, wantKey, func(*Watchman) {}, true, true, false},
|
|
{"a want still standing", AssignerWantsBucket, wantKey, func(w *Watchman) { w.Want(wantKey, wantValue(t, fpNew, false), Put, t0) }, true, false, false},
|
|
// DLV-M5 (ADR 0615): the wants are read from the assigner's first view, which may come after the start's end; what
|
|
// they hold waits for it, and is then reconciled as at the start.
|
|
{"a want deleted while nobody heard, read from a late first view", AssignerWantsBucket, wantKey, func(*Watchman) {}, true, true, true},
|
|
{"a want still standing, read from a late first view", AssignerWantsBucket, wantKey, func(w *Watchman) { w.Want(wantKey, wantValue(t, fpNew, false), Put, t0) }, true, false, true},
|
|
} {
|
|
t.Run(row.name, func(t *testing.T) {
|
|
w, o, c := setup()
|
|
c.t = t0.Add(9 * 24 * time.Hour)
|
|
sk := stuckKeyOf(t, row.bucket, row.key)
|
|
v, _ := meshwatchdogsclient.EncodeStuck(meshwatchdogsclient.Stuck{Bucket: row.bucket, Key: row.key, Waited: "a move", Since: t0})
|
|
w.SeedStuck(sk, v)
|
|
if _, ok := o.findings[FindingKey(ScopeDelivery, sk, KindStuck)]; !ok {
|
|
t.Fatal("the seeded stuck key raised no finding")
|
|
}
|
|
if row.late {
|
|
// A pulse of another key of the bucket, and a finding the bucket is the source of, stood at the start.
|
|
pk := pulseKeyOf(t, row.bucket, "want.n9.gone")
|
|
w.SeedPulse(pk)
|
|
fv, err := EncodeFinding(Finding{Scope: ScopeModule, ID: "gone", Kind: KindManifestRefused, Severity: Warning, Source: row.bucket, Summary: "refused", Since: t0})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
fk := FindingKey(ScopeModule, "gone", KindManifestRefused)
|
|
o.findings[fk], _ = DecodeFinding(fv)
|
|
w.SeedFinding(fk, fv)
|
|
w.Awaits(row.bucket)
|
|
w.Reconcile()
|
|
_, open := o.findings[FindingKey(ScopeDelivery, sk, KindStuck)]
|
|
_, seeded := o.findings[fk]
|
|
if o.stuck[sk].Moved || !open || !seeded || slices.Contains(o.pulseDeletes, pk) {
|
|
t.Fatalf("the start's end took what an awaited bucket holds: moved %v, stuck open %v, finding %v, pulses deleted %v", o.stuck[sk].Moved, open, seeded, o.pulseDeletes)
|
|
}
|
|
row.standing(w)
|
|
w.ReadLate(row.bucket)
|
|
if _, seeded := o.findings[fk]; seeded || !slices.Contains(o.pulseDeletes, pk) {
|
|
t.Errorf("the late read kept the finding (%v) or the pulse (deleted %v) nothing made again", seeded, o.pulseDeletes)
|
|
}
|
|
} else {
|
|
row.standing(w)
|
|
if row.readWhole {
|
|
w.ReadWhole(row.bucket)
|
|
}
|
|
w.Reconcile()
|
|
}
|
|
_, open := o.findings[FindingKey(ScopeDelivery, sk, KindStuck)]
|
|
if got := o.stuck[sk].Moved; got != row.closed || open == row.closed {
|
|
t.Errorf("moved %v, finding open %v; want closed %v", got, open, row.closed)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
// design 47 G §7, §428: no part's code names another module: a bus user is said by its own module's name on its machine, the tool
|
|
// runner's no differently from any module's.
|
|
func TestABusUserIsSaidByItsModuleWithoutAModulesNameInTheCode(t *testing.T) {
|
|
for user, want := range map[string]string{"ace.node-tools": "node-tools on ace", "ace.web": "web on ace", "node.ace": "the node-engine on ace"} {
|
|
if got, _ := BusUserWords(user); got != want {
|
|
t.Errorf("BusUserWords(%q) = %q, want %q", user, got, want)
|
|
}
|
|
}
|
|
}
|