Keep calls and hand acts on the bus, answer status at once, record durations (hq to-be 45 Phase 0)

A controller restart lost every call's outcome, `status` composed the mesh
while its caller waited (18.6s live on 2026-10-06, past the 10s window), a
repair by hand left no trace, and the core's bounds had nothing measured to
be set from.

- calls: kept in the controller's bucket mesh-controller_calls (last 1000 or
  14 days, answers bounded to 64 KiB), read by id across a restart; a
  controller starting marks a stopped one's running calls abandoned; each
  call names its caller from the inbox its answer goes to.
- status: the serving controller composes it at start, after news from a
  machine, a build or an acting verb, and every minute; the verb answers the
  last composition at once with when and how long it took. Composing resolves
  each machine once instead of twice.
- hand-act log in mesh-controller_hand-acts: push (required through the seat),
  plans stop/close, broker consumer-reset and the new hand-act record take
  --why/--cause/--condition; `hand-acts` lists them and repeated causes;
  status counts the week's.
- durations (migration 0066): apply (send to first report), heartbeat gap,
  plan tier and build, recorded as heard; `durations` summarises them.
- the controller's seat row takes this binary's definition of its own verbs,
  so the console no longer judges calls against an older build's schema.
- the controller is granted its two buckets' subjects.

internal/inventory/durations.go

@@ -0,0 +1,146 @@
+package inventory
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"time"
+
+	"github.com/jackc/pgx/v5"
+)
+
+// The durations the core's bounds are set from (novox/hq to-be 45 Phase 0): see migration 0066.
+
+// The kinds of duration recorded.
+const (
+	DurationApply        = "apply"
+	DurationHeartbeatGap = "heartbeat-gap"
+	DurationPlanTier     = "plan-tier"
+	DurationBuild        = "build"
+)
+
+// DurationKinds are every kind, in the order `durations` shows them.
+var DurationKinds = []string{DurationApply, DurationHeartbeatGap, DurationPlanTier, DurationBuild}
+
+// DurationsKeptFor is how long a duration is kept: long enough to set a bound from, and to correct it
+// in Phase 1's first live week.
+const DurationsKeptFor = 30 * 24 * time.Hour
+
+// Duration is one measurement.
+type Duration struct {
+	Kind    string        `json:"kind"`
+	Subject string        `json:"subject"`
+	Node    string        `json:"node,omitempty"`
+	Ref     string        `json:"ref"`
+	Started time.Time     `json:"started"`
+	Took    time.Duration `json:"took"`
+	Detail  string        `json:"detail,omitempty"`
+}
+
+// RecordDuration keeps one measurement, once: the same thing measured again is not a second row.
+func (i *Inventory) RecordDuration(ctx context.Context, d Duration) error {
+	if d.Took < 0 {
+		return nil // a clock that went back measures nothing
+	}
+	_, err := i.store.Pool().Exec(ctx,
+		`insert into duration (kind, subject, node, ref, started, took_ms, detail)
+		 values ($1, $2, $3, $4, $5, $6, $7) on conflict do nothing`,
+		d.Kind, d.Subject, d.Node, d.Ref, d.Started, d.Took.Milliseconds(), d.Detail)
+	return err
+}
+
+// RecordApplyDuration measures a machine's report of the declaration it was last sent: from the send
+// to the first report of it. A report of anything else, or of a send already measured, measures
+// nothing — a machine reconciling reports the same declaration every few minutes.
+func (i *Inventory) RecordApplyDuration(ctx context.Context, node, declared, outcome string) error {
+	if declared == "" {
+		return nil
+	}
+	_, err := i.store.Pool().Exec(ctx,
+		`insert into duration (kind, subject, node, ref, started, took_ms, detail)
+		 select $1, n.name, n.name, n.sent || '@' || to_char(n.sent_at at time zone 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS.US'),
+		        n.sent_at, (extract(epoch from (now() - n.sent_at)) * 1000)::bigint, $4
+		   from node n
+		  where n.name = $2 and n.sent = $3 and n.sent_at is not null
+		 on conflict do nothing`,
+		DurationApply, node, declared, outcome)
+	return err
+}
+
+// RecordHeartbeatGap measures the silence before a machine's word: from the last word heard before
+// it, which the caller read before recording this one.
+func (i *Inventory) RecordHeartbeatGap(ctx context.Context, node string, before time.Time) error {
+	if before.IsZero() {
+		return nil
+	}
+	now := time.Now()
+	return i.RecordDuration(ctx, Duration{Kind: DurationHeartbeatGap, Subject: node, Node: node,
+		Ref: before.UTC().Format(time.RFC3339Nano), Started: before, Took: now.Sub(before)})
+}
+
+// Durations is every measurement of a kind since a moment, oldest first; every kind when kind is empty.
+func (i *Inventory) Durations(ctx context.Context, kind string, since time.Time) ([]Duration, error) {
+	rows, err := i.store.Pool().Query(ctx,
+		`select kind, subject, node, ref, started, took_ms, detail from duration
+		  where ($1 = '' or kind = $1) and recorded >= $2 order by recorded`, kind, since)
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+	var out []Duration
+	for rows.Next() {
+		var d Duration
+		var ms int64
+		if err := rows.Scan(&d.Kind, &d.Subject, &d.Node, &d.Ref, &d.Started, &ms, &d.Detail); err != nil {
+			return nil, err
+		}
+		d.Took = time.Duration(ms) * time.Millisecond
+		out = append(out, d)
+	}
+	return out, rows.Err()
+}
+
+// ForgetOldDurations removes what is older than DurationsKeptFor, and says how many.
+func (i *Inventory) ForgetOldDurations(ctx context.Context) (int64, error) {
+	tag, err := i.store.Pool().Exec(ctx, `delete from duration where recorded < $1`,
+		time.Now().Add(-DurationsKeptFor))
+	if err != nil {
+		return 0, err
+	}
+	return tag.RowsAffected(), nil
+}
+
+// planTierLeft is the measurement a plan's save makes when it leaves a tier: the tier moved on, or
+// the plan ended. Read in the save's own transaction, so two saves cannot both measure one tier.
+func planTierLeft(ctx context.Context, tx pgx.Tx, p Plan, now time.Time) (entered time.Time, err error) {
+	var oldTier int
+	var oldState string
+	var since time.Time
+	err = tx.QueryRow(ctx,
+		`select tier, state, coalesce(tier_entered, created) from release_plan where id = $1 for update`,
+		p.ID).Scan(&oldTier, &oldState, &since)
+	if errors.Is(err, pgx.ErrNoRows) {
+		return now, nil // a new plan enters its first tier now
+	}
+	if err != nil {
+		return time.Time{}, err
+	}
+	wasOpen := oldState == PlanBuilding || oldState == PlanRolling
+	if !wasOpen || (oldTier == p.Tier && p.Open()) {
+		return since, nil // still in the tier, or already ended
+	}
+	var modules []string
+	if oldTier >= 0 && oldTier < len(p.Tiers) {
+		modules = p.Tiers[oldTier]
+	}
+	detail := fmt.Sprintf("plan %s, tier %d of %d (%v), left %s", p.ID, oldTier, len(p.Tiers), modules, p.State)
+	if p.Open() {
+		detail = fmt.Sprintf("plan %s, tier %d of %d (%v), moved on to tier %d", p.ID, oldTier, len(p.Tiers), modules, p.Tier)
+	}
+	_, err = tx.Exec(ctx,
+		`insert into duration (kind, subject, node, ref, started, took_ms, detail)
+		 values ($1, $2, '', $3, $4, $5, $6) on conflict do nothing`,
+		DurationPlanTier, p.Repository, fmt.Sprintf("%s/tier-%d", p.ID, oldTier), since,
+		now.Sub(since).Milliseconds(), detail)
+	return now, err
+}

internal/inventory/durations_test.go

@@ -0,0 +1,97 @@
+package inventory
+
+import (
+	"testing"
+	"time"
+)
+
+// **The durations the bounds are set from are recorded once each** (novox/hq to-be 45 Phase 0): a
+// send measured at its first report and not again at every reconcile that repeats it; a send of
+// something else measures nothing; a new send is a new measurement.
+func TestAnApplyIsMeasuredOncePerSend(t *testing.T) {
+	inv := ForTest(t)
+	ctx := t.Context()
+	node, err := inv.AddNode(ctx, "anchor")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := inv.RecordSent(ctx, node.ID, "d1", nil); err != nil {
+		t.Fatal(err)
+	}
+	for range 3 { // the report, then two reconciles saying the same
+		if err := inv.RecordApplyDuration(ctx, "anchor", "d1", OutcomeApplied); err != nil {
+			t.Fatal(err)
+		}
+	}
+	if err := inv.RecordApplyDuration(ctx, "anchor", "d0", OutcomeApplied); err != nil {
+		t.Fatal(err)
+	}
+	ds, err := inv.Durations(ctx, DurationApply, time.Now().Add(-time.Hour))
+	if err != nil || len(ds) != 1 || ds[0].Subject != "anchor" || ds[0].Detail != OutcomeApplied || ds[0].Took < 0 {
+		t.Fatalf("%v %+v", err, ds)
+	}
+	time.Sleep(10 * time.Millisecond)
+	if err := inv.RecordSent(ctx, node.ID, "d2", nil); err != nil {
+		t.Fatal(err)
+	}
+	if err := inv.RecordApplyDuration(ctx, "anchor", "d2", OutcomeFailed); err != nil {
+		t.Fatal(err)
+	}
+	if ds, _ = inv.Durations(ctx, "", time.Now().Add(-time.Hour)); len(ds) != 2 {
+		t.Fatalf("a second send was not measured: %+v", ds)
+	}
+}
+
+// A machine's silence is the time since its last word, once per word.
+func TestASilenceIsMeasuredFromTheLastWord(t *testing.T) {
+	inv := ForTest(t)
+	ctx := t.Context()
+	before := time.Now().Add(-90 * time.Second)
+	for range 2 {
+		if err := inv.RecordHeartbeatGap(ctx, "anchor", before); err != nil {
+			t.Fatal(err)
+		}
+	}
+	if err := inv.RecordHeartbeatGap(ctx, "anchor", time.Time{}); err != nil {
+		t.Fatal(err)
+	}
+	ds, err := inv.Durations(ctx, DurationHeartbeatGap, time.Now().Add(-time.Hour))
+	if err != nil || len(ds) != 1 || ds[0].Took < 90*time.Second || ds[0].Took > 2*time.Minute {
+		t.Fatalf("%v %+v", err, ds)
+	}
+}
+
+// A plan's tier is measured when the plan leaves it — moving on, or ending — and only then.
+func TestAPlansTierIsMeasuredWhenItIsLeft(t *testing.T) {
+	inv := ForTest(t)
+	ctx := t.Context()
+	p := Plan{ID: "plan-1", Repository: "novox/mesh-tools", Commit: "abc", Created: time.Now().UTC(),
+		State: PlanBuilding, Tiers: [][]string{{"mesh-tools"}, {"builder"}},
+		Modules: map[string]*PlanModule{"mesh-tools": {}, "builder": {}}}
+	if err := inv.SavePlan(ctx, p); err != nil {
+		t.Fatal(err)
+	}
+	p.State = PlanRolling // the same tier, saved again
+	if err := inv.SavePlan(ctx, p); err != nil {
+		t.Fatal(err)
+	}
+	if ds, _ := inv.Durations(ctx, DurationPlanTier, time.Now().Add(-time.Hour)); len(ds) != 0 {
+		t.Fatalf("a tier not left was measured: %+v", ds)
+	}
+	p.Tier = 1
+	if err := inv.SavePlan(ctx, p); err != nil {
+		t.Fatal(err)
+	}
+	p.State = PlanDone
+	if err := inv.SavePlan(ctx, p); err != nil {
+		t.Fatal(err)
+	}
+	if err := inv.SavePlan(ctx, p); err != nil { // saved again once ended: nothing more to measure
+		t.Fatal(err)
+	}
+	ds, err := inv.Durations(ctx, DurationPlanTier, time.Now().Add(-time.Hour))
+	if err != nil || len(ds) != 2 || ds[0].Subject != "novox/mesh-tools" || ds[0].Ref != "plan-1/tier-0" ||
+		ds[1].Ref != "plan-1/tier-1" {
+		t.Fatalf("%v %+v", err, ds)
+	}
+}

internal/inventory/migrations/0066-the-durations-the-bounds-are-set-from.sql

@@ -0,0 +1,31 @@
+-- The durations the core's bounds are set from (novox/hq to-be 45 Phase 0).
+--
+-- Every watchdog of the core's signals table has a bound — how long a machine may take to report
+-- after a send, how long it may be silent, how long a plan's tier or a build may take — and a bound
+-- guessed is a condition that cries wolf or one that never fires. So the controller records each
+-- duration as it is observed, and Phase 1 sets the bounds from what was recorded:
+--
+--   apply          a declaration sent → the machine's first report of that declaration, per machine
+--   heartbeat-gap  one word from a machine → the next, per machine
+--   plan-tier      a plan entering a tier → leaving it, per repository
+--   build          a build asked → its outcome heard, per module
+--
+-- One row per thing measured: `ref` names it (the send, the earlier word, the plan's tier, the
+-- build), so a report repeated by a reconcile is not a second measurement. Kept a month; read
+-- through `durations`.
+create table duration (
+    kind     text        not null,
+    subject  text        not null,
+    node     text        not null default '',
+    ref      text        not null,
+    started  timestamptz not null,
+    took_ms  bigint      not null,
+    detail   text        not null default '',
+    recorded timestamptz not null default now(),
+    primary key (kind, subject, ref)
+);
+
+create index duration_by_kind on duration (kind, recorded);
+
+-- When a plan entered the tier it is at, so leaving it measures the tier.
+alter table release_plan add column tier_entered timestamptz;

internal/inventory/plans.go

@@ -80,13 +80,29 @@ func (i *Inventory) SavePlan(ctx context.Context, p Plan) error {
 	if err != nil {
 		return err
 	}
-	_, err = i.store.Pool().Exec(ctx,
-		`insert into release_plan (id, repository, commit_hash, created, updated, state, tier, tiers, modules, note, branch)
-		 values ($1, $2, $3, $4, now(), $5, $6, $7, $8, $9, $10)
+	// **And how long the tier it left took** (novox/hq to-be 45 Phase 0): measured here, where the
+	// plan moves, in the same transaction as the move, so no save can move a tier unmeasured or
+	// measure one twice.
+	tx, err := i.store.Pool().Begin(ctx)
+	if err != nil {
+		return err
+	}
+	defer func() { _ = tx.Rollback(ctx) }()
+	entered, err := planTierLeft(ctx, tx, p, time.Now())
+	if err != nil {
+		return err
+	}
+	_, err = tx.Exec(ctx,
+		`insert into release_plan (id, repository, commit_hash, created, updated, state, tier, tiers, modules, note, branch, tier_entered)
+		 values ($1, $2, $3, $4, now(), $5, $6, $7, $8, $9, $10, $11)
 		 on conflict (id) do update set updated = now(), state = excluded.state, tier = excluded.tier,
-		        tiers = excluded.tiers, modules = excluded.modules, note = excluded.note, branch = excluded.branch`,
-		p.ID, p.Repository, p.Commit, p.Created, p.State, p.Tier, tiers, modules, p.Note, p.Branch)
-	return err
+		        tiers = excluded.tiers, modules = excluded.modules, note = excluded.note, branch = excluded.branch,
+		        tier_entered = excluded.tier_entered`,
+		p.ID, p.Repository, p.Commit, p.Created, p.State, p.Tier, tiers, modules, p.Note, p.Branch, entered)
+	if err != nil {
+		return err
+	}
+	return tx.Commit(ctx)
 }
 
 // OpenPlans is every plan still being worked, oldest first.

internal/inventory/seats.go

@@ -105,14 +105,27 @@ func (i *Inventory) widenProtocol(ctx context.Context, s catalogue.Seat) error {
 	changed := false
 	row.Accepts, changed = union(row.Accepts, s.Accepts, changed)
 	row.Emits, changed = union(row.Emits, s.Emits, changed)
-	have := map[string]bool{}
-	for _, v := range row.Serves {
-		have[v.Name] = true
+	have := map[string]int{}
+	for n, v := range row.Serves {
+		have[v.Name] = n
 	}
 	for _, v := range s.Serves {
-		if !have[v.Name] {
+		at, kept := have[v.Name]
+		if !kept {
 			row.Serves = append(row.Serves, v)
 			changed = true
+			continue
+		}
+		// **The controller's own verbs are described by the binary that runs them** (novox/hq
+		// issue 244, to-be 45 §7). Its seat's verbs are not an operator's to reshape: each is a
+		// command line this binary composes from the arguments its own table declares, and the
+		// console judges a call against the row. A row kept from an older build described `push`
+		// without the `behind` and `why` the binary takes, so the console refused an argument the verb
+		// needs. So a verb this binary defines takes this binary's definition; a verb only the row has
+		// — a newer build's, during a roll-out (ADR 0185) — is left as it is.
+		if s.Name == catalogue.ControllerSeatName && !sameVerb(row.Serves[at], v) {
+			row.Serves[at] = v
+			changed = true
 		}
 	}
 	if !changed {
@@ -282,3 +295,18 @@ func (i *Inventory) Holdings(ctx context.Context) ([]catalogue.Held, error) {
 	}
 	return out, rows.Err()
 }
+
+// sameVerb is whether two definitions of a verb say the same, read as the row stores them.
+func sameVerb(a, b catalogue.Verb) bool {
+	ja, errA := json.Marshal(a)
+	jb, errB := json.Marshal(b)
+	if errA != nil || errB != nil {
+		return false
+	}
+	var ra, rb any
+	_ = json.Unmarshal(ja, &ra)
+	_ = json.Unmarshal(jb, &rb)
+	ca, _ := json.Marshal(ra)
+	cb, _ := json.Marshal(rb)
+	return string(ca) == string(cb)
+}

internal/inventory/seats_test.go

@@ -113,3 +113,46 @@ func TestRenameSeatKeepsTheFormerNameAsAnAlias(t *testing.T) {
 		t.Fatal("renaming a seat to its own name was accepted")
 	}
 }
+
+// **The controller's verbs in the row are the binary's** (novox/hq issue 244, to-be 45 §7): a row
+// seeded by an older build describes `push` without the arguments this one takes, and the console
+// judges a call against the row — so re-seeding brings the controller's verbs to this binary's
+// definition, and keeps a verb only the row has, which a newer build added (ADR 0185).
+func TestTheControllersVerbsInTheRowAreTheBinarys(t *testing.T) {
+	inv := ForTest(t)
+	ctx := t.Context()
+	if _, err := inv.SeedSeats(ctx, catalogue.DefaultSeats()); err != nil {
+		t.Fatal(err)
+	}
+	old := `[{"name":"push","description":"an older push","input":{"type":"object","properties":{"node":{"type":"string"}}}},
+	         {"name":"newer","description":"a verb of a newer build"}]`
+	if _, err := inv.store.Pool().Exec(ctx, `update seat set serves = $1 where name = $2`,
+		[]byte(old), catalogue.ControllerSeatName); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := inv.SeedSeats(ctx, catalogue.DefaultSeats()); err != nil {
+		t.Fatal(err)
+	}
+	seats, err := inv.Seats(ctx)
+	if err != nil {
+		t.Fatal(err)
+	}
+	verbs := map[string]catalogue.Verb{}
+	for _, s := range seats {
+		if s.Name == catalogue.ControllerSeatName {
+			for _, v := range s.Serves {
+				verbs[v.Name] = v
+			}
+		}
+	}
+	props, _ := verbs["push"].Input["properties"].(map[string]any)
+	if _, takesWhy := props["why"]; !takesWhy || verbs["push"].Description == "an older push" {
+		t.Fatalf("push in the row is still the older build's: %+v", verbs["push"])
+	}
+	if _, kept := verbs["newer"]; !kept {
+		t.Fatal("a verb only the row has was dropped")
+	}
+	if _, added := verbs["durations"]; !added {
+		t.Fatal("a verb this binary adds was not added")
+	}
+}