novox/mesh-controller
7
0
Fork 0
Code Issues Pull Requests 6 Releases Wiki Activity

Keep calls and hand acts on the bus, answer status at once, record durations (hq to-be 45 Phase 0)

Browse Source 
A controller restart lost every call's outcome, `status` composed the mesh
while its caller waited (18.6s live on 2026-10-06, past the 10s window), a
repair by hand left no trace, and the core's bounds had nothing measured to
be set from.

- calls: kept in the controller's bucket mesh-controller_calls (last 1000 or
  14 days, answers bounded to 64 KiB), read by id across a restart; a
  controller starting marks a stopped one's running calls abandoned; each
  call names its caller from the inbox its answer goes to.
- status: the serving controller composes it at start, after news from a
  machine, a build or an acting verb, and every minute; the verb answers the
  last composition at once with when and how long it took. Composing resolves
  each machine once instead of twice.
- hand-act log in mesh-controller_hand-acts: push (required through the seat),
  plans stop/close, broker consumer-reset and the new hand-act record take
  --why/--cause/--condition; `hand-acts` lists them and repeated causes;
  status counts the week's.
- durations (migration 0066): apply (send to first report), heartbeat gap,
  plan tier and build, recorded as heard; `durations` summarises them.
- the controller's seat row takes this binary's definition of its own verbs,
  so the console no longer judges calls against an older build's schema.
- the controller is granted its two buckets' subjects.
This commit is contained in:
jochen
2026-10-06 02:59:36 +02:00
parent 146c48fd96
commit e74c32ed50
35 changed files with 2313 additions and 75 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cmd/mesh-controller/push.go
+60 -4
View File
@@ -105,7 +105,10 @@ func serve(ctx context.Context) error {
work := link.Enrolment{Inventory: inv, Identity: ident, Broker: known,
OnNATS: true}
server, err := connectLink(ctx, inv, work, work)
// `status` from a summary kept current here (novox/hq to-be 45 Phase 0): a machine saying
// something new is one thing that moves it, so the listener nudges it.
statusFrom = newStatusSummary(composeStatus(open))
server, err := connectLink(ctx, inv, work, nudgingListener{work, statusFrom})
if err != nil {
return err
}
@@ -121,6 +124,8 @@ func serve(ctx context.Context) error {
// Open plans move on a timer as well as on outcomes (novox/hq ADR 0162): a tier waiting for
// machines to report moves when they have, and a plan left by a replaced controller resumes.
go planTicker(ctx, open)
// The durations the core's bounds are set from are kept a month (novox/hq to-be 45 Phase 0).
go forgettingOldDurations(ctx, inv)
// And what the catalogue decided a build meant. The builder's own result is already handled
// above; this is the other half — the control plane is the only one of the three that knows
// which machines run the thing, so it is the one that acts (novox/hq ADR 0072).
@@ -158,8 +163,21 @@ func serve(ctx context.Context) error {
if !isNATS {
return errors.New("the mesh's verbs are served over the bus, and this control plane is not on it")
}
// The hand-act log is counted for `status` on this connection rather than a new one a minute.
handActConn = bus.Conn
// Composed now and kept current, before the verb that answers from it is served.
go statusFrom.keep(ctx)
// A call that outlasts its caller's patience is followed by `calls` (novox/hq issue 265).
link.Calls.Follow = catalogue.ControllerSeatName + ".calls"
// And every call is kept on the bus, so a restart of this process keeps what came of each
// (novox/hq to-be 45 §6). A bus without the bucket is said and served from memory, as before:
// answering no calls at all would be worse than answering them without the record.
said := log.New(os.Stdout, "", log.LstdFlags)
if keeper, err := link.CallsOnTheBus(ctx, bus.Conn); err != nil {
fmt.Printf("calls are kept in memory only, and lost when this controller stops: %v\n", err)
} else if err := link.Calls.Durably(ctx, keeper, controllerProcess(), said); err != nil {
fmt.Printf("calls are kept on the bus from now on; the ones kept before could not be read: %v\n", err)
}
stopServing, err := bus.ServeSeatTools(catalogue.ControllerSeatName, handlers, log.New(os.Stdout, "", log.LstdFlags))
if err != nil {
return err
@@ -260,6 +278,9 @@ func pushCommand(ctx context.Context, args []string) error {
// (novox/hq ADR 0010). 0 waits for nothing, which is the old fire-and-forget.
wait := set.Duration("wait", 0,
"for a named node, how long to wait for it to report applying what it was sent (0: do not wait)")
// A push by hand is a repair, and says why (novox/hq to-be 45 §7): required through the seat,
// recorded when given at a shell — see handacts.go for why a shell is not refused.
why := addHandActFlags(set)
positionals, err := parseAround(set, args)
if err != nil {
return err
@@ -274,6 +295,11 @@ func pushCommand(ctx context.Context, args []string) error {
return errors.New("push <node> or push --behind, not both: one names a machine and the " +
"other asks which machines need one")
}
recorded := append([]string(nil), args...)
if *behind {
recorded = append(recorded, "--behind")
}
why.record(ctx, "push", recorded)
open, err := openStores(ctx)
if err != nil {
return err
@@ -1039,6 +1065,20 @@ func digestOf(body []byte) string {
// be worked out" is a different problem with a different remedy, and `plan` is where it is said.
func wouldSend(ctx context.Context, open *stores,
nodes []inventory.Node) (map[string]string, error) {
return wouldSendFrom(ctx, open, nodes, nil)
}
// planned is one machine's plan as planFor answered it, for a caller that already asked.
type planned struct {
plan catalogue.Resolution
settings catalogue.SettingsBy
}
// wouldSendFrom is wouldSend reusing the plans a caller worked out a moment before: resolving a
// machine is most of what `status` costs, and it used to resolve every machine twice (novox/hq
// to-be 45 Phase 0). A machine absent from plans is worked out here.
func wouldSendFrom(ctx context.Context, open *stores,
nodes []inventory.Node, plans map[string]planned) (map[string]string, error) {
gens, err := generators(ctx, open)
if err != nil {
@@ -1046,9 +1086,12 @@ func wouldSend(ctx context.Context, open *stores,
}
out := map[string]string{}
for _, n := range nodes {
plan, settings, err := planFor(ctx, open, n.Name)
if err != nil {
continue
known, have := plans[n.Name]
plan, settings := known.plan, known.settings
if !have {
if plan, settings, err = planFor(ctx, open, n.Name); err != nil {
continue
}
}
declared, err := declarationWith(ctx, open, n.Name, plan, settings, gens, Reading)
if err != nil {
@@ -1136,6 +1179,11 @@ func raiseTheBus(ctx context.Context, inv *inventory.Inventory, address string)
if err := broker.RaiseCancelledSets(js, inventory.MeshSeats()); err != nil {
return err
}
// And the controller's own buckets (novox/hq to-be 45 §1): the calls it serves and the acts done
// by hand, kept where a restart of this process does not take them.
if err := js.EnsureControllerBuckets(); err != nil {
return err
}
// Every module's state (novox/hq ADR 0201), from the catalogue: a bucket exists from
// registration, so a module reading one may watch it before its owner runs anywhere. One that
// nothing declares any more is said and kept — what it holds is data.
@@ -1271,3 +1319,11 @@ func reportUnheldPushed(w io.Writer, named bool, asked []string, unheld map[stri
fmt.Fprintf(w, "%s: %d unmet seat dependenc(ies) — see `status`\n", node, len(lines))
}
}
// controllerProcess names this serving process among controllers: the machine, the process and when
// it started — what a call kept on the bus carries, so the next controller can tell a call this one
// left running from one it is running itself (novox/hq to-be 45 §6).
func controllerProcess() string {
host, _ := os.Hostname()
return fmt.Sprintf("controller@%s pid %d since %s", host, os.Getpid(), time.Now().UTC().Format(time.RFC3339))
}