Files
mesh-delivery/modules/mesh-delivery-watchman/internal/watch/watch_test.go
T
jschoubben e8d1bf96be Names N3 (watchman): the watchman names its own buckets from its credential's module
Its states presence, pulse and stuck and its first views were named from the constant
WatchmanModule; they are now `<cred.Module>_<state>` (watch.BucketsOf), and the watchman's
stuck keys of presence take its own presence bucket's name.

Builds: design 47 G A-296

Claude-Session: https://claude.ai/code/session_01NcbgfGKaxw984XyAPfLm4h
2026-11-12 19:09:46 +01:00

759 lines
26 KiB
Go

package watch
import (
"encoding/json"
"errors"
"slices"
"strings"
"testing"
"time"
"git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshwatchdogsclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshwantsclient"
)
type fakeOut struct {
presence map[string]bool
presenceWrite int
pulses map[string]meshwatchdogsclient.Pulse
ttls map[string]time.Duration
pulseDeletes []string
stuck map[string]meshwatchdogsclient.Stuck
stuckWrites int
findings map[string]Finding
findingWrites int
said []said
}
type said struct{ subject, id string }
func (o *fakeOut) Say(subject string, _ []byte, id string) error {
o.said = append(o.said, said{subject, id})
return nil
}
func newOut() *fakeOut {
return &fakeOut{presence: map[string]bool{}, pulses: map[string]meshwatchdogsclient.Pulse{}, ttls: map[string]time.Duration{}, stuck: map[string]meshwatchdogsclient.Stuck{},
findings: map[string]Finding{}}
}
func (o *fakeOut) PutFinding(key string, value []byte) error {
f, err := DecodeFinding(value)
if err != nil {
return err
}
if f.Key() != key {
return errors.New("a finding written under another key than its own: " + key)
}
o.findings[key] = f
o.findingWrites++
return nil
}
func (o *fakeOut) DeleteFinding(key string) error {
delete(o.findings, key)
return nil
}
func (o *fakeOut) PutPresence(key string, online bool) error {
o.presence[key] = online
o.presenceWrite++
return nil
}
func (o *fakeOut) PutPulse(key string, value []byte, ttl time.Duration) error {
p, err := meshwatchdogsclient.DecodePulse(value)
if err != nil {
return err
}
o.pulses[key], o.ttls[key] = p, ttl
return nil
}
func (o *fakeOut) DeletePulse(key string) error {
delete(o.pulses, key)
o.pulseDeletes = append(o.pulseDeletes, key)
return nil
}
func (o *fakeOut) PutStuck(key string, value []byte) error {
s, err := meshwatchdogsclient.DecodeStuck(value)
if err != nil {
return err
}
o.stuck[key] = s
o.stuckWrites++
return nil
}
var t0 = time.Date(2026, 10, 18, 12, 0, 0, 0, time.UTC)
type clock struct{ t time.Time }
func (c *clock) now() time.Time { return c.t }
func setup() (*Watchman, *fakeOut, *clock) {
o, c := newOut(), &clock{t: t0}
w := New(testModule, o, c.now)
w.SetStepBound(meshwatchdogsclient.PulseMoveBound) // the older tests move a step as they move any key: stuck at the pulse's bound
return w, o, c
}
// checkValue is a check's key as the controller's builder writes it (mesh-controller internal/link/checks_nats.go
// CheckRunning, kept by keepRunning on every heartbeat).
func checkValue(started time.Time, head string) []byte {
b, _ := json.Marshal(map[string]any{"head": head, "trunk": "def456", "builder": "n1", "build": "b9", "started": started})
return b
}
const (
checksKey = "SEAT_NODE_BUILD_GO"
testModule = "a-watchman"
presenceNS = testModule + "_presence"
)
// askBody is the body of a build seat's step-progress, ask-taken or step-ended for an ask at a time.
func askBody(id string, at time.Time) []byte {
b, err := json.Marshal(map[string]any{"ask": id, "asker": "a1", "at": at, "phase": "build"})
if err != nil {
panic(err)
}
return b
}
// askTaken, askProgress and askEnded are what the holder of the Go build seat says on node n1 (design 47 G §496, §602).
func askTaken(w *Watchman, id string, at time.Time) {
w.AskTaken("node-build-go", "n1", askBody(id, at))
}
func askProgress(w *Watchman, id string, at time.Time) {
w.StepProgress("node-build-go", "n1", askBody(id, at))
}
func askEnded(w *Watchman, id string) { w.AskEnded("node-build-go", askBody(id, t0)) }
func pulseKeyOf(t *testing.T, bucket, key string) string {
t.Helper()
k, err := meshwatchdogsclient.PulseKey(bucket, key)
if err != nil {
t.Fatal(err)
}
return k
}
func stuckKeyOf(t *testing.T, bucket, key string) string {
t.Helper()
k, err := meshwatchdogsclient.StuckKey(bucket, key)
if err != nil {
t.Fatal(err)
}
return k
}
func TestPresenceFromEventsWrittenOnlyWhenItDiffers(t *testing.T) { // 1135/AC1
t.Run("from events", presenceFromEvents)
t.Run("written only when it differs", presenceWrittenOnlyWhenItDiffers)
}
func presenceFromEvents(t *testing.T) { // 1135/AC1
w, o, _ := setup()
w.Connect("node.ace")
w.Connect("ace.mesh-delivery-view")
w.Connect("ace.gitea")
want := map[string]bool{"ace": true, "part.mesh-delivery-view.ace": true}
if len(o.presence) != 2 || o.presence["ace"] != true || o.presence["part.mesh-delivery-view.ace"] != true {
t.Fatalf("presence = %v, want %v and nothing for another module's user", o.presence, want)
}
w.Disconnect("node.ace")
w.Disconnect("ace.gitea")
if online, ok := o.presence["ace"]; !ok || online {
t.Fatalf("presence = %v, want ace offline", o.presence)
}
if len(o.presence) != 2 {
t.Fatalf("another module's user wrote presence: %v", o.presence)
}
}
func presenceWrittenOnlyWhenItDiffers(t *testing.T) { // 1135/AC1
w, o, _ := setup()
w.Connect("node.ace")
w.Connect("node.ace")
if o.presenceWrite != 1 {
t.Fatalf("writes = %d, want 1 (the second connection changes nothing)", o.presenceWrite)
}
w.Disconnect("node.ace")
if !o.presence["ace"] || o.presenceWrite != 1 {
t.Fatalf("one of two connections closed: ace must stay online without a write, got %v, %d writes", o.presence, o.presenceWrite)
}
w.Disconnect("node.ace")
if o.presence["ace"] || o.presenceWrite != 2 {
t.Fatalf("both closed: ace offline, got %v, %d writes", o.presence, o.presenceWrite)
}
w.Disconnect("node.ace") // a stray event
if o.presenceWrite != 2 {
t.Fatalf("a disconnect of a user already offline wrote %d times", o.presenceWrite)
}
}
func TestParseEvent(t *testing.T) { // 1135/AC1
user, err := ParseEvent([]byte(`{"type":"io.nats.server.advisory.v1.client_connect","client":{"acc":"MESH","user":"node.ace","id":4}}`))
if err != nil || user != "node.ace" {
t.Fatalf("user = %q, %v", user, err)
}
if _, err := ParseEvent([]byte(`not json`)); err == nil {
t.Fatal("an event that is not JSON must be refused")
}
if user, err := ParseEvent([]byte(`{"client":{}}`)); err != nil || user != "" {
t.Fatalf("an event with no user = %q, %v; want empty and no error", user, err)
}
}
func connzAnswer(users ...string) []byte {
var conns []map[string]string
for _, u := range users {
conns = append(conns, map[string]string{"authorized_user": u})
}
b, _ := json.Marshal(map[string]any{"data": map[string]any{"connections": conns, "total": len(conns)}})
return b
}
func TestRebuildAsksConnectionsAndTakesWhoIsGone(t *testing.T) { // 1135/AC2
t.Run("asks the connections with auth", rebuildAsksConnectionsWithAuth)
t.Run("takes the presence of who is gone", rebuildTakesThePresenceOfWhoIsGone)
}
func rebuildAsksConnectionsWithAuth(t *testing.T) { // 1135/AC2
w, o, _ := setup()
var body []byte
err := w.Rebuild(func(b []byte) ([]byte, error) {
body = b
return connzAnswer("node.ace", "ace.mesh-delivery-view", "ace.mesh-delivery-view", "ace.gitea"), nil
})
if err != nil {
t.Fatal(err)
}
if string(body) != `{"auth":true}` || string(ConnzRequest()) != `{"auth":true}` {
t.Fatalf("request body = %s, want {\"auth\":true}", body)
}
if !o.presence["ace"] || !o.presence["part.mesh-delivery-view.ace"] || len(o.presence) != 2 {
t.Fatalf("presence = %v", o.presence)
}
// Two connections of the part: one closing leaves it online.
w.Disconnect("ace.mesh-delivery-view")
if !o.presence["part.mesh-delivery-view.ace"] {
t.Fatal("the part went offline with one connection still open")
}
}
func rebuildTakesThePresenceOfWhoIsGone(t *testing.T) { // 1135/AC2
w, o, c := setup()
w.SeedPresence("ace", true, t0.Add(-time.Hour))
w.SeedPresence("g14", true, t0.Add(-time.Hour))
if err := w.Rebuild(func([]byte) ([]byte, error) { return connzAnswer("node.g14"), nil }); err != nil {
t.Fatal(err)
}
if online, ok := o.presence["ace"]; !ok || online {
t.Fatalf("ace was online in the bucket and is not among the connections: %v", o.presence)
}
if _, ok := o.presence["g14"]; ok {
t.Fatalf("g14 is online and was online: nothing to write, got %v", o.presence)
}
k := pulseKeyOf(t, presenceNS, "ace")
if o.ttls[k] != 120*time.Second {
t.Fatalf("ttl of the pulse of ace gone = %v, want 120s", o.ttls[k])
}
_ = c
if err := w.Rebuild(func([]byte) ([]byte, error) { return nil, errors.New("no answer") }); err == nil {
t.Fatal("a connections request that fails must be an error")
}
}
func TestRebuildPulsesFromCurrentKeys(t *testing.T) { // 1135/AC2
w, o, c := setup()
c.t = t0.Add(30 * time.Second)
askProgress(w, "askid1", t0.Add(10*time.Second))
w.SeedPresence("ace", false, t0)
w.Reconcile()
if err := w.Rebuild(func([]byte) ([]byte, error) { return connzAnswer(), nil }); err != nil {
t.Fatal(err)
}
for _, k := range []string{pulseKeyOf(t, checksKey, "askid1"), pulseKeyOf(t, presenceNS, "ace")} {
if _, ok := o.pulses[k]; !ok {
t.Errorf("no pulse %s; have %v", k, o.pulses)
}
}
}
func TestReconcileDropsPulsesNothingMirrorsAnyMore(t *testing.T) { // 1135/AC2
w, o, _ := setup()
stale := pulseKeyOf(t, checksKey, "gone")
live := pulseKeyOf(t, checksKey, "askid1")
w.SeedPulse(stale)
w.SeedPulse(live)
askProgress(w, "askid1", t0)
w.Reconcile()
if !slices.Equal(o.pulseDeletes, []string{stale}) {
t.Fatalf("deleted = %v, want only %s", o.pulseDeletes, stale)
}
if len(o.stuck) != 0 {
t.Fatalf("a stale pulse must not become stuck: %v", o.stuck)
}
}
func TestPulseTimeToLiveIsTheBoundMinusTheAge(t *testing.T) { // 1135/AC3
w, o, c := setup()
c.t = t0.Add(30 * time.Second)
askProgress(w, "askid1", t0) // the key found at start, last renewed 30s ago
k := pulseKeyOf(t, checksKey, "askid1")
if o.ttls[k] != 90*time.Second {
t.Fatalf("ttl = %v, want 90s (last renewed 30s ago)", o.ttls[k])
}
if p := o.pulses[k]; p.Bucket != checksKey || p.Key != "askid1" {
t.Fatalf("pulse = %+v", p)
}
w.Connect("node.ace")
w.Disconnect("node.ace")
if got := o.ttls[pulseKeyOf(t, presenceNS, "ace")]; got != 120*time.Second {
t.Fatalf("presence just gone offline: ttl = %v, want 120s", got)
}
}
func TestAMoveAtTheBoundIsStuckAtOnce(t *testing.T) { // 1135/AC3
w, o, c := setup()
c.t = t0.Add(120 * time.Second)
askProgress(w, "askid1", t0)
if len(o.pulses) != 0 {
t.Fatalf("a pulse was written for a heartbeat 120s old: %v", o.pulses)
}
s, ok := o.stuck[stuckKeyOf(t, checksKey, "askid1")]
if !ok {
t.Fatalf("no stuck key; have %v", o.stuck)
}
if s.Moved || !s.Since.Equal(t0) || s.Waited != "the ask askid1 to end on n1" {
t.Fatalf("stuck = %+v", s)
}
}
func TestWaitedInWords(t *testing.T) { // 1135/AC3
w, o, c := setup()
c.t = t0.Add(200 * time.Second)
askProgress(w, "askid1", t0)
w.SeedPresence("ace", false, t0)
w.SeedPresence("part.mesh-delivery-view.ace", false, t0)
w.Reconcile()
_ = w.Rebuild(func([]byte) ([]byte, error) { return connzAnswer(), nil })
want := map[string]string{
stuckKeyOf(t, checksKey, "askid1"): "the ask askid1 to end on n1",
stuckKeyOf(t, presenceNS, "ace"): "ace to come back online",
stuckKeyOf(t, presenceNS, "part.mesh-delivery-view.ace"): "mesh-delivery-view on ace to come back online",
}
for k, waited := range want {
if o.stuck[k].Waited != waited {
t.Errorf("%s waited = %q, want %q", k, o.stuck[k].Waited, waited)
}
}
}
func TestEveryHeartbeatRenewsThePulse(t *testing.T) { // 1135/AC3
w, o, c := setup()
k := pulseKeyOf(t, checksKey, "askid1")
askProgress(w, "askid1", t0)
if o.ttls[k] != 120*time.Second {
t.Fatalf("ttl = %v, want the full 120s", o.ttls[k])
}
o.ttls[k] = 0
c.t = t0.Add(100 * time.Second)
askProgress(w, "askid1", c.t) // the builder re-puts the key: same check, started long ago
if o.ttls[k] != 120*time.Second {
t.Fatalf("a heartbeat left the pulse as it was: ttl = %v, want the full 120s again", o.ttls[k])
}
if len(o.stuck) != 0 {
t.Fatalf("a check that heartbeats is stuck: %v", o.stuck)
}
// The same revision heard twice changes nothing.
o.ttls[k] = 0
askProgress(w, "askid1", c.t)
if o.ttls[k] != 0 {
t.Fatal("one revision rewrote the pulse twice")
}
}
func TestMirroredMovesDeleteThePulse(t *testing.T) { // 1135/AC4
w, o, c := setup()
c.t = t0.Add(10 * time.Second)
askProgress(w, "askid1", t0)
w.Connect("node.ace")
w.Disconnect("node.ace")
askEnded(w, "askid1") // the check ended
w.Connect("node.ace")
if len(o.pulses) != 0 {
t.Fatalf("pulses left: %v", o.pulses)
}
if len(o.pulseDeletes) != 2 {
t.Fatalf("deletes = %v, want 2", o.pulseDeletes)
}
if len(o.stuck) != 0 {
t.Fatalf("a move before expiry wrote stuck: %v", o.stuck)
}
}
func TestACheckWhoseBuilderStopsIsStuckFromItsLastHeartbeat(t *testing.T) { // 1135/AC4
w, o, c := setup()
askProgress(w, "askid1", t0)
c.t = t0.Add(50 * time.Second)
askProgress(w, "askid1", c.t) // the last heartbeat
c.t = t0.Add(170 * time.Second)
w.Pulse(pulseKeyOf(t, checksKey, "askid1"), Expired, c.t)
s := o.stuck[stuckKeyOf(t, checksKey, "askid1")]
if !s.Since.Equal(t0.Add(50 * time.Second)) {
t.Fatalf("stuck = %+v, want since the last heartbeat", s)
}
}
func TestAStuckKeyIsWrittenMovedNeverDeleted(t *testing.T) { // 1135/AC4
w, o, c := setup()
askProgress(w, "askid1", t0)
pk := pulseKeyOf(t, checksKey, "askid1")
c.t = t0.Add(120 * time.Second)
w.Pulse(pk, Expired, c.t)
sk := stuckKeyOf(t, checksKey, "askid1")
if o.stuck[sk].Moved {
t.Fatal("stuck written as moved at expiry")
}
c.t = t0.Add(300 * time.Second)
askEnded(w, "askid1")
s := o.stuck[sk]
if !s.Moved || !s.Since.Equal(t0) || s.Waited != "the ask askid1 to end on n1" {
t.Fatalf("stuck after the move = %+v, want the same key moved", s)
}
n := o.stuckWrites
askEnded(w, "askid1")
if o.stuckWrites != n {
t.Fatal("a stuck key already moved was written again")
}
}
func TestAnAskEndedAfterStuckMovesIt(t *testing.T) { // 1135/AC4
w, o, c := setup()
askProgress(w, "askid1", t0)
c.t = t0.Add(120 * time.Second)
w.Pulse(pulseKeyOf(t, checksKey, "askid1"), Expired, c.t)
askEnded(w, "askid1") // the ask ended at last
if !o.stuck[stuckKeyOf(t, checksKey, "askid1")].Moved {
t.Fatal("an ask taken after it was stuck is still stuck")
}
}
func TestPresenceStuckThenBackOnline(t *testing.T) { // 1135/AC4
w, o, c := setup()
w.Connect("node.ace")
w.Disconnect("node.ace")
c.t = t0.Add(120 * time.Second)
w.Pulse(pulseKeyOf(t, presenceNS, "ace"), Expired, c.t)
sk := stuckKeyOf(t, presenceNS, "ace")
if s := o.stuck[sk]; s.Moved || !s.Since.Equal(t0) {
t.Fatalf("stuck = %+v", s)
}
w.Connect("node.ace")
if !o.stuck[sk].Moved {
t.Fatalf("stuck = %+v, want moved once ace is back", o.stuck[sk])
}
}
func TestOnlyTheExpiryOfItsOwnPulseWritesStuck(t *testing.T) { // 1135/AC5
t.Run("the expiry marker writes stuck", expiryMarkerWritesStuck)
t.Run("its own delete of a pulse writes nothing", ownDeleteOfAPulseWritesNothing)
}
func expiryMarkerWritesStuck(t *testing.T) { // 1135/AC5
w, o, c := setup()
askProgress(w, "askid1", t0)
c.t = t0.Add(120 * time.Second)
w.Pulse(pulseKeyOf(t, checksKey, "askid1"), Expired, c.t)
s, ok := o.stuck[stuckKeyOf(t, checksKey, "askid1")]
if !ok {
t.Fatalf("no stuck key; have %v", o.stuck)
}
if s.Bucket != checksKey || s.Key != "askid1" || s.Waited != "the ask askid1 to end on n1" || !s.Since.Equal(t0) || s.Moved {
t.Fatalf("stuck = %+v", s)
}
}
func ownDeleteOfAPulseWritesNothing(t *testing.T) { // 1135/AC5
w, o, c := setup()
askProgress(w, "askid1", t0)
pk := pulseKeyOf(t, checksKey, "askid1")
askEnded(w, "askid1")
w.Pulse(pk, Delete, c.t) // the delete the watchman made, heard on its own watch
w.Pulse(pk, Expired, c.t)
w.Pulse("mesh-delivery-watchman_pulse.unknown.key", Expired, c.t)
if len(o.stuck) != 0 {
t.Fatalf("stuck written for a pulse the watchman deleted or never made: %v", o.stuck)
}
}
func TestTheBoundCanBeShortenedForATest(t *testing.T) { // 1135/AC8
w, o, c := setup()
w.SetBound(2 * time.Second)
c.t = t0.Add(500 * time.Millisecond)
askProgress(w, "askid1", t0)
if got := o.ttls[pulseKeyOf(t, checksKey, "askid1")]; got != 1500*time.Millisecond {
t.Fatalf("ttl = %v, want 1.5s", got)
}
c.t = t0.Add(3 * time.Second)
askProgress(w, "askid2", t0)
if _, ok := o.stuck[stuckKeyOf(t, checksKey, "askid2")]; !ok {
t.Fatalf("a heartbeat older than the bound is stuck at once; have %v", o.stuck)
}
}
func TestAnAskTakenByANodeHasAPulseFromWhenItStartedAndEndsWithTheStep(t *testing.T) { // 1908, 1909
w, o, c := setup()
c.t = t0.Add(10 * time.Second)
askProgress(w, "askid1", t0)
if len(o.pulses) != 1 {
t.Fatalf("pulses = %v, want the taken ask's", o.pulses)
}
w2, o2, c2 := setup()
c2.t = t0.Add(10 * time.Second)
askTaken(w2, "askid1", t0)
askEnded(w2, "askid1")
if len(o2.pulses) != 0 && o2.pulses[pulseKeyOf(t, checksKey, "askid1")].Key != "" {
t.Errorf("the pulse of an ended ask stands: %v", o2.pulses)
}
_ = c2
_ = c
}
func TestARunningAskFoundAtStartGetsItsPulse(t *testing.T) { // 1909
w, o, c := setup()
c.t = t0.Add(30 * time.Second)
askProgress(w, "askid1", t0) // what the steps held when the watchman started
p, ok := o.pulses[pulseKeyOf(t, checksKey, "askid1")]
if !ok || p.Key != "askid1" {
t.Fatalf("pulses = %v", o.pulses)
}
if got := o.ttls[pulseKeyOf(t, checksKey, "askid1")]; got <= 0 || got >= 120*time.Second {
t.Errorf("ttl = %v, want the bound less its 30s", got)
}
}
func TestAnEventThatDoesNotReadOrIsNoBuildSeatsIsLeft(t *testing.T) { // 1908
w, o, _ := setup()
w.StepProgress("node-build-go", "n1", []byte(`{"v":99}`))
w.StepProgress("node-build-go", "n1", askBody("", t0))
w.AskTaken("node-build-go", "n1", []byte(`nope`))
w.AskTaken("mesh-wants", "n1", askBody("askid1", t0))
if len(o.pulses) != 0 || len(o.stuck) != 0 {
t.Errorf("pulses %v stuck %v", o.pulses, o.stuck)
}
}
func TestARunningStepThatSaysItsProgressIsNeverStuckAndOneWhoseHolderStopsIs(t *testing.T) { // design 47 G §496, §602
o, c := newOut(), &clock{t: t0}
w := New(testModule, o, c.now)
askTaken(w, "askid1", t0)
k := pulseKeyOf(t, checksKey, "askid1")
if got := o.ttls[k]; got != ProgressBound {
t.Errorf("ttl = %v, want the progress bound %v", got, ProgressBound)
}
// a 20-minute step: its progress each minute renews the clock
for m := 1; m <= 20; m++ {
c.t = t0.Add(time.Duration(m) * time.Minute)
askProgress(w, "askid1", c.t)
if o.ttls[k] != ProgressBound {
t.Fatalf("minute %d: ttl = %v, want the full bound again", m, o.ttls[k])
}
}
if len(o.stuck) != 0 {
t.Fatalf("a step saying its progress is stuck: %v", o.stuck)
}
// its holder stops: the pulse expires, and the ask is stuck from its last progress
c.t = t0.Add(26 * time.Minute)
w.Pulse(k, Expired, c.t)
s, ok := o.stuck[stuckKeyOf(t, checksKey, "askid1")]
if !ok || s.Moved || !s.Since.Equal(t0.Add(20*time.Minute)) {
t.Fatalf("stuck = %+v, %v", s, ok)
}
askEnded(w, "askid1")
if !o.stuck[stuckKeyOf(t, checksKey, "askid1")].Moved {
t.Fatal("the end of the ask left it stuck")
}
}
func TestARunningStepPastItsBoundIsStuck(t *testing.T) { // 1908
o, c := newOut(), &clock{t: t0}
w := New(testModule, o, c.now)
askProgress(w, "askid1", t0)
c.t = t0.Add(6 * time.Minute)
w.Pulse(pulseKeyOf(t, checksKey, "askid1"), Expired, c.t)
s, ok := o.stuck[stuckKeyOf(t, checksKey, "askid1")]
if !ok || s.Moved || !s.Since.Equal(t0) {
t.Fatalf("stuck = %+v, %v", s, ok)
}
// and one found past its bound at the watchman's start is stuck at once
o2, c2 := newOut(), &clock{t: t0.Add(time.Hour)}
askProgress(New(testModule, o2, c2.now), "askid1", t0)
if _, ok := o2.stuck[stuckKeyOf(t, checksKey, "askid1")]; !ok || len(o2.pulses) != 0 {
t.Errorf("stuck %v pulses %v", o2.stuck, o2.pulses)
}
}
// 1906/D6 (ADR 0539 §19): a want is a pulse owed by the node's agent from when the assigner wrote it, until the have names
// its fingerprint.
func wantValue(t *testing.T, fp string, removed bool) []byte {
t.Helper()
w := meshwantsclient.Want{Fp: fp, Composed: "sha256:" + strings.Repeat("c", 64), Removed: removed}
if removed {
w.Composed = ""
}
b, err := meshwantsclient.EncodeWant(w)
if err != nil {
t.Fatal(err)
}
return b
}
// runningBody is the node-engine's `running` for a build of fingerprint fp (design 47 G §541).
func runningBody(fp string) []byte { return []byte(`{"fingerprint":"` + fp + `"}`) }
var (
fpOld = meshcatalogclient.SourcePrefix2 + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
fpNew = meshcatalogclient.SourcePrefix2 + "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"
)
func TestAWantIsAPulseOwedByTheNodeAndTheHaveNamingItEndsIt(t *testing.T) {
w, o, c := setup()
c.t = t0.Add(10 * time.Second)
w.Want("want.ace.web", wantValue(t, fpNew, false), Put, t0)
pk := pulseKeyOf(t, AssignerWantsBucket, "want.ace.web")
p, ok := o.pulses[pk]
if !ok || p.Key != "want.ace.web" || !strings.Contains(p.Owes, "ace") || !strings.Contains(p.Owes, "web") {
t.Fatalf("pulses = %v", o.pulses)
}
w.EngineRunning("ace", "web", runningBody(fpOld)) // the node still runs the old one
if _, ok := o.pulses[pk]; !ok {
t.Fatalf("a have of another fingerprint ended the pulse")
}
w.EngineRunning("ace", "web", runningBody(fpNew))
if _, ok := o.pulses[pk]; ok {
t.Errorf("the have naming the want's fingerprint left the pulse standing: %v", o.pulses)
}
if len(o.stuck) != 0 {
t.Errorf("stuck = %v", o.stuck)
}
}
func TestAHaveThatCameBeforeTheWantEndsItAtOnceAndARemovedWantHasNoPulse(t *testing.T) {
w, o, c := setup()
c.t = t0.Add(time.Second)
w.EngineRunning("ace", "web", runningBody(fpNew))
w.Want("want.ace.web", wantValue(t, fpNew, false), Put, t0)
w.Want("want.ace.mail", wantValue(t, "", true), Put, t0)
if len(o.pulses) != 0 {
t.Errorf("pulses = %v", o.pulses)
}
// the want moving on to a new fingerprint makes a new pulse; the key `want.ace.node` and a held key are left
w.Want("want.ace.web", wantValue(t, fpOld, false), Put, t0.Add(time.Second))
w.Want("want.ace.node", []byte(`{}`), Put, t0)
w.Want("held.web.bbbb", []byte(`{}`), Put, t0)
if len(o.pulses) != 1 {
t.Errorf("pulses = %v, want the web want's only", o.pulses)
}
}
func TestAWantNotAppliedByItsBoundIsStuckAndAKeyDeletedEndsIt(t *testing.T) {
o, c := newOut(), &clock{t: t0}
w := New(testModule, o, c.now)
w.Want("want.ace.web", wantValue(t, fpNew, false), Put, t0)
pk := pulseKeyOf(t, AssignerWantsBucket, "want.ace.web")
c.t = t0.Add(31 * time.Minute)
w.Pulse(pk, Expired, c.t)
if s, ok := o.stuck[stuckKeyOf(t, AssignerWantsBucket, "want.ace.web")]; !ok || s.Moved || !s.Since.Equal(t0) {
t.Fatalf("stuck = %v", o.stuck)
}
w2, o2, c2 := setup()
c2.t = t0.Add(time.Second)
w2.Want("want.ace.web", wantValue(t, fpNew, false), Put, t0)
w2.Want("want.ace.web", nil, Delete, t0)
if len(o2.pulses) != 0 {
t.Errorf("a deleted want left its pulse: %v", o2.pulses)
}
}
// Sweep 2026-10-30: a watched key that aged out of its bucket or was deleted while nobody heard leaves its stuck key; at the
// start's end the watchman closes it (moved, its finding cleared), but only for a bucket it read whole at that start.
func TestAStuckKeyWhoseKeyIsGoneIsClosedAtTheStart(t *testing.T) {
wantKey := "want.n1.mod-a"
for _, row := range []struct {
name string
bucket string
key string
standing func(w *Watchman) // what the start reads standing
readWhole bool
closed bool
late bool // read after the start's end (Awaits, ReadLate)
}{
{"a want deleted while nobody heard", AssignerWantsBucket, wantKey, func(*Watchman) {}, true, true, false},
{"a want still standing", AssignerWantsBucket, wantKey, func(w *Watchman) { w.Want(wantKey, wantValue(t, fpNew, false), Put, t0) }, true, false, false},
// DLV-M5 (ADR 0615): the wants are read from the assigner's first view, which may come after the start's end; what
// they hold waits for it, and is then reconciled as at the start.
{"a want deleted while nobody heard, read from a late first view", AssignerWantsBucket, wantKey, func(*Watchman) {}, true, true, true},
{"a want still standing, read from a late first view", AssignerWantsBucket, wantKey, func(w *Watchman) { w.Want(wantKey, wantValue(t, fpNew, false), Put, t0) }, true, false, true},
} {
t.Run(row.name, func(t *testing.T) {
w, o, c := setup()
c.t = t0.Add(9 * 24 * time.Hour)
sk := stuckKeyOf(t, row.bucket, row.key)
v, _ := meshwatchdogsclient.EncodeStuck(meshwatchdogsclient.Stuck{Bucket: row.bucket, Key: row.key, Waited: "a move", Since: t0})
w.SeedStuck(sk, v)
if _, ok := o.findings[FindingKey(ScopeDelivery, sk, KindStuck)]; !ok {
t.Fatal("the seeded stuck key raised no finding")
}
if row.late {
// A pulse of another key of the bucket, and a finding the bucket is the source of, stood at the start.
pk := pulseKeyOf(t, row.bucket, "want.n9.gone")
w.SeedPulse(pk)
fv, err := EncodeFinding(Finding{Scope: ScopeModule, ID: "gone", Kind: KindManifestRefused, Severity: Warning, Source: row.bucket, Summary: "refused", Since: t0})
if err != nil {
t.Fatal(err)
}
fk := FindingKey(ScopeModule, "gone", KindManifestRefused)
o.findings[fk], _ = DecodeFinding(fv)
w.SeedFinding(fk, fv)
w.Awaits(row.bucket)
w.Reconcile()
_, open := o.findings[FindingKey(ScopeDelivery, sk, KindStuck)]
_, seeded := o.findings[fk]
if o.stuck[sk].Moved || !open || !seeded || slices.Contains(o.pulseDeletes, pk) {
t.Fatalf("the start's end took what an awaited bucket holds: moved %v, stuck open %v, finding %v, pulses deleted %v", o.stuck[sk].Moved, open, seeded, o.pulseDeletes)
}
row.standing(w)
w.ReadLate(row.bucket)
if _, seeded := o.findings[fk]; seeded || !slices.Contains(o.pulseDeletes, pk) {
t.Errorf("the late read kept the finding (%v) or the pulse (deleted %v) nothing made again", seeded, o.pulseDeletes)
}
} else {
row.standing(w)
if row.readWhole {
w.ReadWhole(row.bucket)
}
w.Reconcile()
}
_, open := o.findings[FindingKey(ScopeDelivery, sk, KindStuck)]
if got := o.stuck[sk].Moved; got != row.closed || open == row.closed {
t.Errorf("moved %v, finding open %v; want closed %v", got, open, row.closed)
}
})
}
}
// design 47 G §7, §428: no part's code names another module: a bus user is said by its own module's name on its machine, the tool
// runner's no differently from any module's.
func TestABusUserIsSaidByItsModuleWithoutAModulesNameInTheCode(t *testing.T) {
for user, want := range map[string]string{"ace.node-tools": "node-tools on ace", "ace.web": "web on ace", "node.ace": "the node-engine on ace"} {
if got, _ := BusUserWords(user); got != want {
t.Errorf("BusUserWords(%q) = %q, want %q", user, got, want)
}
}
}