A controller restart lost every call's outcome, `status` composed the mesh while its caller waited (18.6s live on 2026-10-06, past the 10s window), a repair by hand left no trace, and the core's bounds had nothing measured to be set from. - calls: kept in the controller's bucket mesh-controller_calls (last 1000 or 14 days, answers bounded to 64 KiB), read by id across a restart; a controller starting marks a stopped one's running calls abandoned; each call names its caller from the inbox its answer goes to. - status: the serving controller composes it at start, after news from a machine, a build or an acting verb, and every minute; the verb answers the last composition at once with when and how long it took. Composing resolves each machine once instead of twice. - hand-act log in mesh-controller_hand-acts: push (required through the seat), plans stop/close, broker consumer-reset and the new hand-act record take --why/--cause/--condition; `hand-acts` lists them and repeated causes; status counts the week's. - durations (migration 0066): apply (send to first report), heartbeat gap, plan tier and build, recorded as heard; `durations` summarises them. - the controller's seat row takes this binary's definition of its own verbs, so the console no longer judges calls against an older build's schema. - the controller is granted its two buckets' subjects.
153 lines
4.5 KiB
Go
153 lines
4.5 KiB
Go
package main
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"strings"
|
|
"sync/atomic"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/novox/mesh-controller/internal/link"
|
|
)
|
|
|
|
// quickly shortens the summary's clocks for one test.
|
|
func quickly(t *testing.T) {
|
|
t.Helper()
|
|
every, settle := statusEvery, statusSettle
|
|
statusEvery, statusSettle = time.Hour, 10*time.Millisecond
|
|
t.Cleanup(func() { statusEvery, statusSettle = every, settle })
|
|
}
|
|
|
|
// **`status` answers in full within ten seconds, five times in a row** (novox/hq to-be 45 Phase 0,
|
|
// D9) — however long composing it takes. On 2026-10-06 composing took eighteen seconds and the
|
|
// verb answered "still running"; from the summary it answers at once, in full, saying when.
|
|
func TestStatusAnswersAtOnceHoweverLongComposingTakes(t *testing.T) {
|
|
quickly(t)
|
|
var composed atomic.Int32
|
|
slow := make(chan struct{})
|
|
s := newStatusSummary(func(ctx context.Context) ([]byte, error) {
|
|
if composed.Add(1) > 1 {
|
|
<-slow // every composition after the first outlasts any caller
|
|
}
|
|
return []byte(`{"wrong":[],"machines":4}`), nil
|
|
})
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
defer cancel()
|
|
defer close(slow)
|
|
go s.keep(ctx)
|
|
for i := 0; i < 5; i++ {
|
|
s.nudge() // a composition is under way and does not finish
|
|
start := time.Now()
|
|
got, err := s.answer(ctx)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if took := time.Since(start); took > time.Second {
|
|
t.Fatalf("answer %d took %s", i+1, took)
|
|
}
|
|
m := got.(map[string]any)
|
|
if m["answer"].(map[string]any)["machines"] != float64(4) || m["composed"] == "" || m["ok"] != true {
|
|
t.Fatalf("answer %d was not in full: %v", i+1, m)
|
|
}
|
|
}
|
|
}
|
|
|
|
// The first composition is waited for, never past the caller's window; a controller with none yet
|
|
// says so rather than answering an empty mesh as a well one.
|
|
func TestStatusBeforeItsFirstCompositionSaysSo(t *testing.T) {
|
|
quickly(t)
|
|
was := link.AnswerWithin
|
|
link.AnswerWithin = 1100 * time.Millisecond
|
|
t.Cleanup(func() { link.AnswerWithin = was })
|
|
never := make(chan struct{})
|
|
defer close(never)
|
|
s := newStatusSummary(func(context.Context) ([]byte, error) { <-never; return nil, nil })
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
defer cancel()
|
|
go s.keep(ctx)
|
|
start := time.Now()
|
|
_, err := s.answer(ctx)
|
|
if err == nil || !strings.Contains(err.Error(), "has no status to answer yet") {
|
|
t.Fatalf("answered %v", err)
|
|
}
|
|
if took := time.Since(start); took > link.AnswerWithin {
|
|
t.Fatalf("waited %s, past the caller's window", took)
|
|
}
|
|
}
|
|
|
|
// A nudge composes it again, once for several close together; a failed composition leaves the last
|
|
// summary standing and says it is the last that could be composed.
|
|
func TestANudgeComposesAgainAndAFailureKeepsTheLastSummary(t *testing.T) {
|
|
quickly(t)
|
|
var composed atomic.Int32
|
|
fail := atomic.Bool{}
|
|
s := newStatusSummary(func(context.Context) ([]byte, error) {
|
|
n := composed.Add(1)
|
|
if fail.Load() {
|
|
return nil, errors.New("the store did not answer")
|
|
}
|
|
body, _ := json.Marshal(map[string]any{"n": n})
|
|
return body, nil
|
|
})
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
defer cancel()
|
|
go s.keep(ctx)
|
|
if _, err := s.answer(ctx); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
s.nudge()
|
|
s.nudge()
|
|
s.nudge()
|
|
waitFor(t, func() bool { return composed.Load() == 2 })
|
|
time.Sleep(50 * time.Millisecond)
|
|
if n := composed.Load(); n != 2 {
|
|
t.Fatalf("three nudges together composed %d times after the first", n-1)
|
|
}
|
|
fail.Store(true)
|
|
s.nudge()
|
|
waitFor(t, func() bool { return composed.Load() == 3 })
|
|
waitFor(t, func() bool {
|
|
got, err := s.answer(ctx)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
m := got.(map[string]any)
|
|
return m["lastAttemptFailed"] != nil && m["answer"].(map[string]any)["n"] == float64(2)
|
|
})
|
|
}
|
|
|
|
// The seat's `status` answers from the summary when this process keeps one.
|
|
func TestTheStatusVerbAnswersFromTheSummary(t *testing.T) {
|
|
quickly(t)
|
|
s := newStatusSummary(func(context.Context) ([]byte, error) { return []byte(`{"from":"summary"}`), nil })
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
defer cancel()
|
|
go s.keep(ctx)
|
|
statusFrom = s
|
|
t.Cleanup(func() { statusFrom = nil })
|
|
handlers, _, err := seatToolHandlers()
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
got, err := handlers["status"](ctx, json.RawMessage(`{}`))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if got.(map[string]any)["answer"].(map[string]any)["from"] != "summary" {
|
|
t.Fatalf("answered %v", got)
|
|
}
|
|
}
|
|
|
|
func waitFor(t *testing.T, ok func() bool) {
|
|
t.Helper()
|
|
deadline := time.Now().Add(3 * time.Second)
|
|
for !ok() {
|
|
if time.Now().After(deadline) {
|
|
t.Fatal("never happened")
|
|
}
|
|
time.Sleep(5 * time.Millisecond)
|
|
}
|
|
}
|