Keep calls and hand acts on the bus, answer status at once, record durations (hq to-be 45 Phase 0)
A controller restart lost every call's outcome, `status` composed the mesh while its caller waited (18.6s live on 2026-10-06, past the 10s window), a repair by hand left no trace, and the core's bounds had nothing measured to be set from. - calls: kept in the controller's bucket mesh-controller_calls (last 1000 or 14 days, answers bounded to 64 KiB), read by id across a restart; a controller starting marks a stopped one's running calls abandoned; each call names its caller from the inbox its answer goes to. - status: the serving controller composes it at start, after news from a machine, a build or an acting verb, and every minute; the verb answers the last composition at once with when and how long it took. Composing resolves each machine once instead of twice. - hand-act log in mesh-controller_hand-acts: push (required through the seat), plans stop/close, broker consumer-reset and the new hand-act record take --why/--cause/--condition; `hand-acts` lists them and repeated causes; status counts the week's. - durations (migration 0066): apply (send to first report), heartbeat gap, plan tier and build, recorded as heard; `durations` summarises them. - the controller's seat row takes this binary's definition of its own verbs, so the console no longer judges calls against an older build's schema. - the controller is granted its two buckets' subjects.
This commit is contained in:
jochen
@@ -0,0 +1,188 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// `status` answered from a summary the serving controller keeps current (novox/hq to-be 45 Phase 0,
|
||||
// §4's D9 and §8's health of the controller).
|
||||
//
|
||||
// **Asked, it was composed: every machine resolved, twice, while its caller waited.** On 2026-10-06
|
||||
// the verb took eighteen seconds on a mesh of four machines, so its caller read "still running" and
|
||||
// had to ask `calls` for the answer to "is the mesh alright" — the one question that must answer at
|
||||
// once, and the one a self-check and a rollout gate will ask every few minutes. So the serving
|
||||
// controller composes it in the background — at its start, after anything that changes what it says
|
||||
// (a machine's report, a build, a verb that acts), and every minute regardless — and the verb answers
|
||||
// the last composition at once, saying when it was composed and how long that took. A caller who
|
||||
// needs it newer than that reads the time and asks again; nothing is answered as current that is not.
|
||||
|
||||
// statusEvery is how often the summary is composed with nothing having nudged it; statusSettle how
|
||||
// long a nudge waits for the next, so a push answered by four machines is composed once.
|
||||
var (
|
||||
statusEvery = time.Minute
|
||||
statusSettle = 2 * time.Second
|
||||
// statusComposeWithin bounds one composition, so a store that hangs cannot stop the summary for
|
||||
// good; the attempt is said as failed, and the last summary stands with its age.
|
||||
statusComposeWithin = 2 * time.Minute
|
||||
)
|
||||
|
||||
// statusSummary is the last composed `status --json`, and when.
|
||||
type statusSummary struct {
|
||||
compose func(context.Context) ([]byte, error)
|
||||
// every, settle and within are the clocks above, read once when it is made.
|
||||
every, settle, within time.Duration
|
||||
|
||||
mu sync.Mutex
|
||||
body []byte
|
||||
composedAt time.Time
|
||||
took time.Duration
|
||||
failed string
|
||||
failedAt time.Time
|
||||
started time.Time
|
||||
first chan struct{} // closed when the first attempt ends, either way
|
||||
|
||||
nudged chan struct{}
|
||||
}
|
||||
|
||||
func newStatusSummary(compose func(context.Context) ([]byte, error)) *statusSummary {
|
||||
return &statusSummary{compose: compose, started: time.Now(), first: make(chan struct{}),
|
||||
nudged: make(chan struct{}, 1), every: statusEvery, settle: statusSettle, within: statusComposeWithin}
|
||||
}
|
||||
|
||||
// statusFrom is the serving controller's summary; nil in any other process, where `status` is
|
||||
// composed when asked, as at a shell.
|
||||
var statusFrom *statusSummary
|
||||
|
||||
// nudge asks for a composition soon. Never blocks: one pending is as good as many.
|
||||
func (s *statusSummary) nudge() {
|
||||
if s == nil {
|
||||
return
|
||||
}
|
||||
select {
|
||||
case s.nudged <- struct{}{}:
|
||||
default:
|
||||
}
|
||||
}
|
||||
|
||||
// keep composes until ctx ends: now, on a nudge once things settle, and every statusEvery.
|
||||
func (s *statusSummary) keep(ctx context.Context) {
|
||||
once := sync.Once{}
|
||||
for {
|
||||
s.composeOnce(ctx)
|
||||
once.Do(func() { close(s.first) })
|
||||
timer := time.NewTimer(s.every)
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
timer.Stop()
|
||||
return
|
||||
case <-timer.C:
|
||||
case <-s.nudged:
|
||||
timer.Stop()
|
||||
// Let what else is arriving arrive, then compose once for all of it.
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-time.After(s.settle):
|
||||
}
|
||||
select {
|
||||
case <-s.nudged:
|
||||
default:
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (s *statusSummary) composeOnce(ctx context.Context) {
|
||||
start := time.Now()
|
||||
asking, cancel := context.WithTimeout(ctx, s.within)
|
||||
body, err := s.compose(asking)
|
||||
cancel()
|
||||
took := time.Since(start)
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if err != nil {
|
||||
s.failed, s.failedAt = err.Error(), time.Now()
|
||||
fmt.Printf("status could not be composed (after %s): %v — `status` answers the last summary, "+
|
||||
"with its age\n", took.Round(time.Millisecond), err)
|
||||
return
|
||||
}
|
||||
s.body, s.composedAt, s.took, s.failed = body, start, took, ""
|
||||
}
|
||||
|
||||
// answer is what the `status` verb answers: the last summary at once, the same document `status
|
||||
// --json` prints, with when it was composed. Before the first composition has ended it waits for it,
|
||||
// but never past the caller's window; a controller that has none says so and why, rather than
|
||||
// answering an empty mesh as a well one.
|
||||
func (s *statusSummary) answer(ctx context.Context) (any, error) {
|
||||
wait := time.NewTimer(link.AnswerWithin - time.Second)
|
||||
defer wait.Stop()
|
||||
select {
|
||||
case <-s.first:
|
||||
case <-wait.C:
|
||||
case <-ctx.Done():
|
||||
}
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if s.body == nil {
|
||||
why := "its first composition has not finished"
|
||||
if s.failed != "" {
|
||||
why = "it could not be composed: " + s.failed
|
||||
}
|
||||
return nil, fmt.Errorf("this controller started %s ago and has no status to answer yet — %s. "+
|
||||
"Ask again shortly", time.Since(s.started).Round(time.Second), why)
|
||||
}
|
||||
var parsed any
|
||||
_ = json.Unmarshal(s.body, &parsed)
|
||||
out := map[string]any{
|
||||
"output": string(s.body), "ok": true, "answer": parsed,
|
||||
"composed": s.composedAt.UTC().Format(time.RFC3339),
|
||||
"age": time.Since(s.composedAt).Round(time.Second).String(),
|
||||
"composedIn": s.took.Round(time.Millisecond).String(),
|
||||
"note": "composed by the serving controller at its start, after each report, build or act, and " +
|
||||
"every minute; answered at once from the last composition",
|
||||
}
|
||||
if s.failed != "" && s.failedAt.After(s.composedAt) {
|
||||
out["lastAttemptFailed"] = fmt.Sprintf("%s: %s — this summary is the last that could be composed",
|
||||
s.failedAt.UTC().Format(time.RFC3339), s.failed)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// composeStatus is `status --json`, composed in this process against its stores.
|
||||
func composeStatus(open *stores) func(context.Context) ([]byte, error) {
|
||||
return func(ctx context.Context) ([]byte, error) {
|
||||
asked, err := theThreeQuestions(ctx, open)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return statusAsJSON(asked)
|
||||
}
|
||||
}
|
||||
|
||||
// readingVerbs are the verbs that only read; after any other, what `status` says may have changed, so the summary is
|
||||
// composed again; a verb that only reads leaves it alone, or a console polling `nodes` would keep the
|
||||
// controller composing for ever.
|
||||
var readingVerbs = map[string]bool{
|
||||
"tools": true, "calls": true, "status": true, "nodes": true, "node": true, "modules": true,
|
||||
"seats": true, "builds": true, "plan": true, "queue": true, "durations": true, "hand-acts": true,
|
||||
}
|
||||
|
||||
// nudgingListener is the enrolment, nudging the summary when a machine said something new.
|
||||
type nudgingListener struct {
|
||||
link.Enrolment
|
||||
summary *statusSummary
|
||||
}
|
||||
|
||||
func (l nudgingListener) Heard(ctx context.Context, report link.Report) (bool, error) {
|
||||
news, err := l.Enrolment.Heard(ctx, report)
|
||||
if news {
|
||||
l.summary.nudge()
|
||||
}
|
||||
return news, err
|
||||
}
|
||||
Reference in New Issue
Block a user