A controller restart lost every call's outcome, `status` composed the mesh while its caller waited (18.6s live on 2026-10-06, past the 10s window), a repair by hand left no trace, and the core's bounds had nothing measured to be set from. - calls: kept in the controller's bucket mesh-controller_calls (last 1000 or 14 days, answers bounded to 64 KiB), read by id across a restart; a controller starting marks a stopped one's running calls abandoned; each call names its caller from the inbox its answer goes to. - status: the serving controller composes it at start, after news from a machine, a build or an acting verb, and every minute; the verb answers the last composition at once with when and how long it took. Composing resolves each machine once instead of twice. - hand-act log in mesh-controller_hand-acts: push (required through the seat), plans stop/close, broker consumer-reset and the new hand-act record take --why/--cause/--condition; `hand-acts` lists them and repeated causes; status counts the week's. - durations (migration 0066): apply (send to first report), heartbeat gap, plan tier and build, recorded as heard; `durations` summarises them. - the controller's seat row takes this binary's definition of its own verbs, so the console no longer judges calls against an older build's schema. - the controller is granted its two buckets' subjects.
163 lines
4.9 KiB
Go
163 lines
4.9 KiB
Go
package link
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"os"
|
|
"sort"
|
|
"strconv"
|
|
"strings"
|
|
"sync/atomic"
|
|
"time"
|
|
|
|
"github.com/nats-io/nats.go"
|
|
"github.com/nats-io/nats.go/jetstream"
|
|
|
|
"github.com/novox/mesh-controller/internal/broker"
|
|
)
|
|
|
|
// The hand-act log (novox/hq to-be 45 §7).
|
|
//
|
|
// **A repair a person makes by hand is the record of a healer the mesh does not have yet.** Tonight's
|
|
// pushes of one machine, plans closed because a report never came, a consumer re-made because it fell
|
|
// a week behind — each was done, and the only trace was a chat. So every verb that repairs by hand
|
|
// asks why, and writes one entry: who, which verb and arguments, why, when, the condition it
|
|
// addresses if one is named, and a cause — a word that, recorded twice in a fortnight, says a healer
|
|
// is wanted (S15, from Phase 3). `hand-act record` is the same entry for an act done outside the mesh.
|
|
// The controller is the bucket's only writer; its verbs are the way in.
|
|
|
|
// HandAct is one entry.
|
|
type HandAct struct {
|
|
ID string `json:"id"`
|
|
At time.Time `json:"at"`
|
|
// By is who: the bus principal a seat call came from, or the account and machine at a shell.
|
|
By string `json:"by"`
|
|
// Verb and Args are the act as given: `push`, `plans close`, `broker consumer-reset`, or
|
|
// `hand-act record` with what was done outside the mesh.
|
|
Verb string `json:"verb"`
|
|
Args []string `json:"arguments,omitempty"`
|
|
Why string `json:"why"`
|
|
// Cause is the condition kind, or a word the person gives; the verb's own name when neither.
|
|
Cause string `json:"cause"`
|
|
// Condition is the condition's key the act addresses, when it names one.
|
|
Condition string `json:"condition,omitempty"`
|
|
}
|
|
|
|
// CallerVar carries a seat call's caller to the command the controller runs for it, so an act done
|
|
// through the console says who asked rather than "the controller".
|
|
const CallerVar = "MESH_CALLER"
|
|
|
|
// Caller is who is acting in this process: the seat call's caller when the controller ran it for
|
|
// one, otherwise the account and machine at the shell.
|
|
func Caller() string {
|
|
if c := strings.TrimSpace(os.Getenv(CallerVar)); c != "" {
|
|
return c
|
|
}
|
|
user := os.Getenv("USER")
|
|
if user == "" {
|
|
user = "an unnamed account"
|
|
}
|
|
host, _ := os.Hostname()
|
|
return fmt.Sprintf("%s at a shell on %s", user, host)
|
|
}
|
|
|
|
type callerKey struct{}
|
|
|
|
// CallerIn is the caller of the seat call ctx belongs to, empty outside one.
|
|
func CallerIn(ctx context.Context) string {
|
|
c, _ := ctx.Value(callerKey{}).(string)
|
|
return c
|
|
}
|
|
|
|
var handActSeq atomic.Uint64
|
|
|
|
// RecordHandAct writes one entry. Its key is its time and a sequence, so the bucket lists in order.
|
|
func RecordHandAct(ctx context.Context, conn *nats.Conn, act HandAct) (HandAct, error) {
|
|
if strings.TrimSpace(act.Why) == "" {
|
|
return act, errors.New("an act by hand says why: --why <text>")
|
|
}
|
|
if act.At.IsZero() {
|
|
act.At = time.Now().UTC()
|
|
}
|
|
if act.ID == "" {
|
|
act.ID = "act-" + strconv.FormatInt(act.At.UnixNano(), 10) + "-" + strconv.FormatUint(handActSeq.Add(1), 10)
|
|
}
|
|
if act.By == "" {
|
|
act.By = Caller()
|
|
}
|
|
if act.Cause == "" {
|
|
act.Cause = act.Verb
|
|
}
|
|
kv, err := handActs(ctx, conn)
|
|
if err != nil {
|
|
return act, err
|
|
}
|
|
body, err := json.Marshal(act)
|
|
if err != nil {
|
|
return act, err
|
|
}
|
|
_, err = kv.Put(ctx, act.ID, body)
|
|
return act, err
|
|
}
|
|
|
|
// HandActs is every entry since a moment, oldest first.
|
|
func HandActs(ctx context.Context, conn *nats.Conn, since time.Time) ([]HandAct, error) {
|
|
kv, err := handActs(ctx, conn)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
w, err := kv.WatchAll(ctx, jetstream.IgnoreDeletes())
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
defer func() { _ = w.Stop() }()
|
|
var out []HandAct
|
|
for {
|
|
select {
|
|
case <-ctx.Done():
|
|
return nil, fmt.Errorf("reading the hand-act log: %w", ctx.Err())
|
|
case entry := <-w.Updates():
|
|
if entry == nil {
|
|
sort.SliceStable(out, func(i, j int) bool { return out[i].At.Before(out[j].At) })
|
|
return out, nil
|
|
}
|
|
var a HandAct
|
|
if json.Unmarshal(entry.Value(), &a) == nil && !a.At.Before(since) {
|
|
out = append(out, a)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// RepeatedCauses are the causes recorded more than once within the fortnight before now, with how
|
|
// often: each is a repair done by hand again, which is what S15 will raise as a healer wanted.
|
|
func RepeatedCauses(acts []HandAct, now time.Time) map[string]int {
|
|
counts := map[string]int{}
|
|
for _, a := range acts {
|
|
if now.Sub(a.At) <= 14*24*time.Hour {
|
|
counts[a.Cause]++
|
|
}
|
|
}
|
|
for c, n := range counts {
|
|
if n < 2 {
|
|
delete(counts, c)
|
|
}
|
|
}
|
|
return counts
|
|
}
|
|
|
|
func handActs(ctx context.Context, conn *nats.Conn) (jetstream.KeyValue, error) {
|
|
api, err := jetstream.New(conn)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
kv, err := api.KeyValue(ctx, broker.HandActsBucket)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("the hand-act log %s is not on the bus — the controller asserts it at its "+
|
|
"start, so one older than this has not: %w", broker.HandActsBucket, err)
|
|
}
|
|
return kv, nil
|
|
}
|