jochen
950e37a46e
mesh/delivery rejected: the gate failed or could not run, or the repository's own check failed
mesh/merge-gate out of date: judged against trunk a44d490d974c, the trunk is now b15846dd6e78; judged again on a recheck or when it is merged
mesh/repo-check out of date: judged against trunk a44d490d974c, the trunk is now b15846dd6e78; judged again on a recheck or when it is merged
A watch that panicked stopped for good, and one that could not read said so only to stderr, so a dead watch read as no slow consumers. A faulted look is now recovered and the loop goes on; every look ends with a bus.watch heartbeat saying whether the watch is healthy, which the controller raises when it stops or says it cannot see. The record is read newest first back to the last look, and closures that could not be published are forgotten after the hour.
337 lines
13 KiB
Go
337 lines
13 KiB
Go
package main
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"os"
|
|
"sort"
|
|
"strings"
|
|
"sync"
|
|
"testing"
|
|
"time"
|
|
)
|
|
|
|
// The watch (novox/hq ADR 0357, issue 270): every user's slow consumer and every connection the bus closed
|
|
// for breaking its rules, said once as an event the controller raises as S9.
|
|
|
|
var watchNow = time.Date(2026, 10, 14, 12, 0, 0, 0, time.UTC)
|
|
|
|
// closedRecord is a record of closed connections as the server serves it, with a password-shaped field the
|
|
// server never serves, to prove nothing but the user is carried.
|
|
const closedRecord = `{"connections":[
|
|
{"cid":10,"name":"g14.claude-code","authorized_user":"g14.claude-code","start":"2026-10-14T11:40:00Z","stop":"2026-10-14T11:58:00Z","reason":"Slow Consumer (Write Deadline)","pending_bytes":334000},
|
|
{"cid":11,"name":"mesh-controller verb node","authorized_user":"controller","start":"2026-10-14T11:58:00Z","stop":"2026-10-14T11:58:01Z","reason":"Client Closed"},
|
|
{"cid":12,"name":"mesh-host/g14","authorized_user":"node.g14","start":"2026-10-14T10:00:00Z","stop":"2026-10-14T11:59:00Z","reason":"Stale Connection"},
|
|
{"cid":13,"name":"g14.node-tools","authorized_user":"g14.node-tools","start":"2026-10-14T10:00:00Z","stop":"2026-10-14T11:59:10Z","reason":"Read Error"},
|
|
{"cid":14,"name":"shanks.desk","authorized_user":"shanks.desk","start":"2026-10-14T11:00:00Z","stop":"2026-10-14T11:59:20Z","reason":"Maximum Message Payload Exceeded"},
|
|
{"cid":15,"name":"","authorized_user":"","start":"2026-10-14T11:59:00Z","stop":"2026-10-14T11:59:30Z","reason":"Authentication Failure"},
|
|
{"cid":16,"name":"ace.media","authorized_user":"ace.media","start":"2026-10-14T09:00:00Z","stop":"2026-10-14T10:30:00Z","reason":"Slow Consumer (Pending Bytes)"},
|
|
{"cid":17,"name":"ace.future","authorized_user":"ace.future","start":"2026-10-14T11:00:00Z","stop":"2026-10-14T11:59:40Z","reason":"A Reason Not Yet Invented","password":"never-carried"},
|
|
{"cid":18,"name":"ace.route-proxy","authorized_user":"ace.route-proxy","start":"2026-10-14T11:00:00Z","stop":"2026-10-14T11:59:50Z","reason":"Server Shutdown"}
|
|
]}`
|
|
|
|
type published struct {
|
|
event string
|
|
body Closure
|
|
}
|
|
|
|
type recorder struct {
|
|
mu sync.Mutex
|
|
got []published
|
|
beats []Beat
|
|
fail error
|
|
}
|
|
|
|
func (r *recorder) emit(event string, body any) error {
|
|
r.mu.Lock()
|
|
defer r.mu.Unlock()
|
|
if r.fail != nil {
|
|
return r.fail
|
|
}
|
|
raw, _ := json.Marshal(body)
|
|
if event == EventWatch {
|
|
var b Beat
|
|
_ = json.Unmarshal(raw, &b)
|
|
r.beats = append(r.beats, b)
|
|
return nil
|
|
}
|
|
if strings.Contains(string(raw), "never-carried") || strings.Contains(strings.ToLower(string(raw)), "password") {
|
|
panic("a credential was published: " + string(raw))
|
|
}
|
|
var c Closure
|
|
_ = json.Unmarshal(raw, &c)
|
|
r.got = append(r.got, published{event, c})
|
|
return nil
|
|
}
|
|
|
|
func (r *recorder) said() []string {
|
|
r.mu.Lock()
|
|
defer r.mu.Unlock()
|
|
var out []string
|
|
for _, p := range r.got {
|
|
out = append(out, p.event+" "+p.body.User)
|
|
}
|
|
sort.Strings(out)
|
|
return out
|
|
}
|
|
|
|
func recordOf(doc *string) Monitor {
|
|
return func(_ context.Context, path string) ([]byte, error) {
|
|
if !strings.HasPrefix(path, "/connz?state=closed&auth=1") {
|
|
return nil, errors.New("the watch asked " + path)
|
|
}
|
|
return []byte(*doc), nil
|
|
}
|
|
}
|
|
|
|
func watchAt(doc *string, r *recorder) *Watch {
|
|
w := NewWatch(recordOf(doc), r.emit)
|
|
w.now = func() time.Time { return watchNow }
|
|
return w
|
|
}
|
|
|
|
func TestTheWatchSaysEachClosureTheServerDroppedNamingItsUserAndNothingRoutine(t *testing.T) {
|
|
doc := closedRecord
|
|
r := &recorder{}
|
|
watchAt(&doc, r).Look(context.Background())
|
|
want := []string{
|
|
"bus.connection-closed ace.future", // a reason the module does not know is said
|
|
"bus.connection-closed shanks.desk", // a limit broken
|
|
"bus.slow-consumer g14.claude-code", // the slow consumer research 036 could not name
|
|
}
|
|
if got := r.said(); strings.Join(got, "|") != strings.Join(want, "|") {
|
|
t.Fatalf("said %v, want %v (none for the client's own close, the network, a refused login, the "+
|
|
"server's shutdown, or a closure older than the hour)", got, want)
|
|
}
|
|
for _, p := range r.got {
|
|
if p.body.User == "g14.claude-code" {
|
|
if p.body.CID != 10 || p.body.Reason != "Slow Consumer (Write Deadline)" || p.body.Pending != 334000 ||
|
|
!p.body.Closed.Equal(time.Date(2026, 10, 14, 11, 58, 0, 0, time.UTC)) || p.body.Lived != 18*60 {
|
|
t.Fatalf("the slow consumer said as %+v", p.body)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestTheWatchSaysAClosureOnceWhileItRuns(t *testing.T) {
|
|
doc := closedRecord
|
|
r := &recorder{}
|
|
w := watchAt(&doc, r)
|
|
w.Look(context.Background())
|
|
w.Look(context.Background())
|
|
if got := len(r.said()); got != 3 {
|
|
t.Fatalf("said %d events over two looks at the same record, want 3: %v", got, r.said())
|
|
}
|
|
}
|
|
|
|
func TestAClosureWhosePublishFailedIsTriedAgainAtTheNextLook(t *testing.T) {
|
|
doc := closedRecord
|
|
r := &recorder{fail: errors.New("the runtime did not answer")}
|
|
w := watchAt(&doc, r)
|
|
w.Look(context.Background())
|
|
if len(r.said()) != 0 {
|
|
t.Fatal("a failed publish was counted as said")
|
|
}
|
|
if report := w.Report(); len(report["not_published"].(map[string]string)) != 3 {
|
|
t.Fatalf("the watch does not say what it could not publish: %v", report)
|
|
}
|
|
r.fail = nil
|
|
w.Look(context.Background())
|
|
if got := len(r.said()); got != 3 {
|
|
t.Fatalf("said %d after the runtime came back, want the 3 that failed", got)
|
|
}
|
|
if _, left := w.Report()["not_published"]; left {
|
|
t.Fatal("a closure published at last is still said not published")
|
|
}
|
|
}
|
|
|
|
func TestAWatchStartedAgainSaysTheLastHourAgain(t *testing.T) {
|
|
doc := closedRecord
|
|
first := &recorder{}
|
|
watchAt(&doc, first).Look(context.Background())
|
|
again := &recorder{}
|
|
watchAt(&doc, again).Look(context.Background())
|
|
if strings.Join(first.said(), "|") != strings.Join(again.said(), "|") || len(again.said()) != 3 {
|
|
t.Fatalf("a restarted watch said %v, the first %v: the last hour must be said again, for the controller "+
|
|
"to count once", again.said(), first.said())
|
|
}
|
|
}
|
|
|
|
func TestAReusedConnectionIDWithANewOpeningIsANewClosure(t *testing.T) {
|
|
doc := `{"connections":[{"cid":10,"authorized_user":"g14.claude-code","start":"2026-10-14T11:40:00Z","stop":"2026-10-14T11:50:00Z","reason":"Slow Consumer (Pending Bytes)"}]}`
|
|
r := &recorder{}
|
|
w := watchAt(&doc, r)
|
|
w.Look(context.Background())
|
|
// The server restarted and numbered connections from one again.
|
|
doc = `{"connections":[
|
|
{"cid":10,"authorized_user":"g14.claude-code","start":"2026-10-14T11:40:00Z","stop":"2026-10-14T11:50:00Z","reason":"Slow Consumer (Pending Bytes)"},
|
|
{"cid":10,"authorized_user":"g14.claude-code","start":"2026-10-14T12:00:05Z","stop":"2026-10-14T12:00:20Z","reason":"Slow Consumer (Pending Bytes)"}]}`
|
|
w.now = func() time.Time { return watchNow.Add(WatchEvery) }
|
|
w.Look(context.Background())
|
|
if got := len(r.said()); got != 2 {
|
|
t.Fatalf("said %d, want 2: the same connection id opened again is another closure", got)
|
|
}
|
|
}
|
|
|
|
func TestTheWatchSaysItCannotReadTheRecord(t *testing.T) {
|
|
r := &recorder{}
|
|
w := NewWatch(func(context.Context, string) ([]byte, error) { return nil, errors.New("connection refused") }, r.emit)
|
|
w.Look(context.Background())
|
|
if report := w.Report(); !strings.Contains(report["last_error"].(string), "connection refused") {
|
|
t.Fatalf("a record that could not be read is not said: %v", report)
|
|
}
|
|
}
|
|
|
|
// **A dead watch is heard** (review 274): every look ends with a heartbeat saying whether the watch is
|
|
// healthy; a record it cannot read for longer than three looks is said not healthy, with why, rather than read
|
|
// as no slow consumers; and it recovers with a read.
|
|
func TestAWatchThatCannotReadForThreeLooksSaysSoInItsHeartbeat(t *testing.T) {
|
|
broken := true
|
|
doc := closedRecord
|
|
clock := watchNow
|
|
r := &recorder{}
|
|
w := NewWatch(func(ctx context.Context, path string) ([]byte, error) {
|
|
if broken {
|
|
return nil, errors.New("connection refused")
|
|
}
|
|
return recordOf(&doc)(ctx, path)
|
|
}, r.emit)
|
|
w.now, w.started = func() time.Time { return clock }, clock
|
|
w.Look(context.Background())
|
|
if len(r.beats) != 1 || !r.beats[0].Healthy {
|
|
t.Fatalf("a watch that just started beat %+v", r.beats)
|
|
}
|
|
clock = clock.Add(WatchStale + time.Second)
|
|
w.Look(context.Background())
|
|
last := r.beats[len(r.beats)-1]
|
|
if last.Healthy || !strings.Contains(last.Why, "connection refused") || !last.At.Equal(clock) {
|
|
t.Fatalf("a watch that has not read for %s beat %+v", WatchStale, last)
|
|
}
|
|
broken = false
|
|
w.Look(context.Background())
|
|
if last := r.beats[len(r.beats)-1]; !last.Healthy || last.LastRead == nil {
|
|
t.Fatalf("a watch that reads again beat %+v", last)
|
|
}
|
|
}
|
|
|
|
// A closure that cannot be published for longer than three looks is not healthy either.
|
|
func TestAClosureUnpublishedForThreeLooksIsNotHealthy(t *testing.T) {
|
|
doc := closedRecord
|
|
r := &recorder{fail: errors.New("the runtime did not answer")}
|
|
clock := watchNow
|
|
w := watchAt(&doc, r)
|
|
w.now, w.started = func() time.Time { return clock }, clock
|
|
w.Look(context.Background())
|
|
clock = clock.Add(WatchStale + time.Second)
|
|
w.Look(context.Background())
|
|
if ok, why := w.Health(); ok || !strings.Contains(why, "could not publish") {
|
|
t.Fatalf("closures unpublished for %s read healthy %v: %q", WatchStale, ok, why)
|
|
}
|
|
}
|
|
|
|
// **A look that panics does not end the watch** (review 274): it is recovered and said, and the next look runs.
|
|
func TestALookThatPanicsIsRecoveredAndTheWatchGoesOn(t *testing.T) {
|
|
calls := 0
|
|
doc := closedRecord
|
|
r := &recorder{}
|
|
w := NewWatch(func(ctx context.Context, path string) ([]byte, error) {
|
|
calls++
|
|
if calls == 1 {
|
|
panic("a nil map somewhere")
|
|
}
|
|
return recordOf(&doc)(ctx, path)
|
|
}, r.emit)
|
|
w.now = func() time.Time { return watchNow }
|
|
w.Look(context.Background())
|
|
w.Look(context.Background())
|
|
if len(r.said()) != 3 {
|
|
t.Fatalf("after a look that panicked the next said %v", r.said())
|
|
}
|
|
if report := w.Report(); report["faults_recovered"] != 1 {
|
|
t.Fatalf("the recovered fault is not said: %v", report)
|
|
}
|
|
}
|
|
|
|
// **The record is read newest first, and only back to the last look** (review 274), never all 10,000 kept.
|
|
func TestALaterLookReadsOnlyWhatClosedSinceTheLastOne(t *testing.T) {
|
|
var asked []string
|
|
page := func(n int, newest time.Time) []byte {
|
|
var conns []string
|
|
for i := range n {
|
|
stop := newest.Add(-time.Duration(i) * time.Second)
|
|
conns = append(conns, fmt.Sprintf(`{"cid":%d,"authorized_user":"u","start":"%s","stop":"%s","reason":"Client Closed"}`,
|
|
1000-i, stop.Add(-time.Minute).Format(time.RFC3339), stop.Format(time.RFC3339)))
|
|
}
|
|
return []byte(`{"connections":[` + strings.Join(conns, ",") + `]}`)
|
|
}
|
|
clock := watchNow
|
|
w := NewWatch(func(_ context.Context, path string) ([]byte, error) {
|
|
asked = append(asked, path)
|
|
if !strings.Contains(path, "sort=stop") {
|
|
return nil, errors.New("not read newest first: " + path)
|
|
}
|
|
if strings.Contains(path, "offset=0") {
|
|
return page(readPage, clock), nil // a full page of the last few minutes
|
|
}
|
|
return page(readPage, clock.Add(-2*time.Hour)), nil // older than any look reaches
|
|
}, (&recorder{}).emit)
|
|
w.now = func() time.Time { return clock }
|
|
w.Look(context.Background())
|
|
first := len(asked)
|
|
asked = nil
|
|
clock = clock.Add(WatchEvery)
|
|
w.Look(context.Background())
|
|
if len(asked) != 1 || first > 2 {
|
|
t.Fatalf("the first look asked %d pages and a later one %d (%v): a later look reads back only to the last", first,
|
|
len(asked), asked)
|
|
}
|
|
}
|
|
|
|
// The failed-publish record forgets a closure older than the hour: S9 would no longer say it.
|
|
func TestAClosureThatCouldNotBePublishedIsForgottenAfterTheHour(t *testing.T) {
|
|
doc := closedRecord
|
|
r := &recorder{fail: errors.New("the runtime did not answer")}
|
|
clock := watchNow
|
|
w := watchAt(&doc, r)
|
|
w.now = func() time.Time { return clock }
|
|
w.Look(context.Background())
|
|
doc = `{"connections":[]}`
|
|
clock = clock.Add(2 * time.Hour)
|
|
w.Look(context.Background())
|
|
if _, left := w.Report()["not_published"]; left {
|
|
t.Fatal("closures older than the hour are still kept as not published")
|
|
}
|
|
}
|
|
|
|
func TestTheManifestDeclaresTheToolsServedTheWatchesEventsAndItsInterval(t *testing.T) {
|
|
raw, err := os.ReadFile("../../module.json")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var m struct {
|
|
Tools []string `json:"tools"`
|
|
Emits []string `json:"emits"`
|
|
}
|
|
if err := json.Unmarshal(raw, &m); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var served []string
|
|
for _, tool := range tools(nil, nil, nil, NewWatch(nil, nil)) {
|
|
served = append(served, tool.Name)
|
|
}
|
|
sort.Strings(served)
|
|
sort.Strings(m.Tools)
|
|
if strings.Join(served, ",") != strings.Join(m.Tools, ",") {
|
|
t.Fatalf("served %v, the manifest lists %v", served, m.Tools)
|
|
}
|
|
sort.Strings(m.Emits)
|
|
if strings.Join(m.Emits, ",") != EventConnectionClosed+","+EventSlowConsumer+","+EventWatch {
|
|
t.Fatalf("the manifest emits %v: the runtime may publish only what is declared", m.Emits)
|
|
}
|
|
if WatchEvery != 30*time.Second || WatchSaysAgain != time.Hour {
|
|
t.Fatalf("the watch looks every %s and reads back %s; ADR 0357 says 30s and an hour", WatchEvery, WatchSaysAgain)
|
|
}
|
|
}
|