Act under a lease, keep accounts by order, one writer at composition (hq to-be 45 Phase 2)

Two controllers could both act (issue 204), a reconcile's report could
overtake the apply after it and the digest decided (issue 267), and a grant
could make a second writer of a machine's report.

- The lease (internal/lease, ADR 0229): mesh-controller_lease key `holder`,
  15 s age, renewed every 5 s by compare-and-set; the epoch is the revision
  it was taken at. The gate is the clock (stops 3 s before expiry); a refused
  renewal is a loss and the process exits; a holder that stops gives it back.
  serve takes it before asserting the bus. Epochs kept in the store
  (migration 0068 controller_epoch) as a floor: a bucket raised from nothing
  is compacted past it. Unleased (no epoch, S12 urgent) only when nobody
  holds it and the bus will not let it be written. A shell command acts
  under the holder's epoch, or its own lease when none.
- Declarations carry `epoch` inside the signed envelope, only to a machine
  whose latest account carried a report_sequence (mesh-host #35); would-send
  is composed with the epoch last sent. Allot and the send both pass the gate.
- Reports: contract in internal/link/order.go (epoch, sequence,
  report_sequence, older_than, refused_older). Accounts kept by epoch, then
  sequence, then report sequence; older refused, counted; unordered reports
  keep the digest rule. Plans by compare-and-set on a revision, with epoch.
  Conditions and calls carry the epoch and are not written off the lease.
- S12 and S13 (naming the writer by epoch) watched, D5 run; reset of the
  bucket said. Writers table compiled in and enforced in PermissionsFor; the
  controller no longer publishes mesh.control.>. A contract per consumed
  kind, and the empty-on-error lint over the repository.
- mesh-host pinned to its main with the epoch in the validator (D1 validates
  the envelope as sent).

Needs mesh-host's genesis lock with the lease grant (mesh-host PR) for
TestTheInstallersFirstUserListIsWhatTheControllerWouldCompose.

cmd/mesh-controller/probes.go

@@ -14,6 +14,7 @@ import (
 	"time"
 
 	"github.com/nats-io/nats.go"
+	"github.com/nats-io/nats.go/jetstream"
 	"github.com/nats-io/nats.go/micro"
 	"github.com/novox/mesh-host/validate"
 	"golang.org/x/net/dns/dnsmessage"
@@ -22,6 +23,7 @@ import (
 	"github.com/novox/mesh-controller/internal/broker"
 	"github.com/novox/mesh-controller/internal/catalogue"
 	"github.com/novox/mesh-controller/internal/conditions"
+	"github.com/novox/mesh-controller/internal/lease"
 	"github.com/novox/mesh-controller/internal/link"
 	"github.com/novox/mesh-controller/internal/overlay"
 )
@@ -61,6 +63,15 @@ func probeDeclarations(ctx context.Context, d *doctor) ([]conditions.Observation
 		if err == nil && gensErr == nil {
 			var declared sendable
 			if declared, err = declarationWith(ctx, open, n.Name, plan, settings, gens, Reading); err == nil {
+				// With the order it was last sent, so the validator reads the envelope a machine is sent
+				// — its epoch included (novox/hq to-be 45 §6).
+				var serr error
+				if declared.Sequence, serr = open.inventory.Sequence(ctx, n.ID); serr == nil {
+					declared.Epoch, serr = open.inventory.SentEpoch(ctx, n.ID)
+				}
+				if serr != nil {
+					return nil, serr // the store, not the machine: the probe could not run
+				}
 				var body []byte
 				if body, err = declared.Body(); err == nil {
 					problems = validate.Declaration(body)
@@ -818,3 +829,73 @@ func askSeatTool(ctx context.Context, conn *nats.Conn, seat, verb, node string)
 
 // oneLine is a message of several lines said on one, its runs of space made one.
 func oneLine(s string) string { return strings.Join(strings.Fields(s), " ") }
+
+// probeLease is D5 (novox/hq to-be 45 §4, §6): exactly one lease holder — the key on the bus names this
+// controller at the epoch it acts under, and the mesh's record has that epoch and no other open — and no
+// message from a stale epoch refused in the last interval.
+func probeLease(ctx context.Context, d *doctor) ([]conditions.Observation, error) {
+	if d.js == nil {
+		return nil, errors.New("this controller is not on the bus")
+	}
+	st := theLease.standing()
+	var out []conditions.Observation
+	split := func(token, summary, said string) {
+		out = append(out, conditions.Observation{Scope: conditions.ScopeCore, ID: "controller.lease", Token: token,
+			Machine: d.host, Severity: conditions.Urgent, Summary: summary, Said: said})
+	}
+	if st.Unleased != "" {
+		split("unheld", "this controller acts without the lease, so nothing keeps another from acting beside it: "+
+			st.Unleased, st.Unleased)
+		return out, nil
+	}
+	api, err := jetstream.New(d.js.Conn())
+	if err != nil {
+		return nil, err
+	}
+	kv, err := api.KeyValue(ctx, broker.LeaseBucket)
+	if err != nil {
+		return nil, fmt.Errorf("the lease bucket cannot be read: %w", err)
+	}
+	holder, found, err := lease.Current(ctx, kv)
+	if err != nil {
+		return nil, fmt.Errorf("the lease cannot be read: %w", err)
+	}
+	switch {
+	case !found:
+		split("split", fmt.Sprintf("nobody holds the lease on the bus, and this controller acts as epoch %d", st.Epoch),
+			"the lease's key is absent")
+	case holder.Instance != instance || holder.Epoch != st.Epoch:
+		split("split", fmt.Sprintf("the lease on the bus names %s at epoch %d, and this controller (%s) acts as "+
+			"epoch %d: two controllers believe they may act", holder.Instance, holder.Epoch, instance, st.Epoch),
+			fmt.Sprintf("held by %s, epoch %d", holder.Instance, holder.Epoch))
+	}
+	// The record: one epoch open, this one.
+	epochs, err := d.open.inventory.EpochsSince(ctx, time.Now().Add(-time.Hour))
+	if err != nil {
+		return nil, fmt.Errorf("the epochs the mesh issued cannot be read: %w", err)
+	}
+	var open []string
+	for _, e := range epochs {
+		if e.Ended == nil && e.Epoch != st.Epoch {
+			open = append(open, fmt.Sprintf("epoch %d (%s)", e.Epoch, e.Instance))
+		}
+	}
+	if len(open) > 0 {
+		split("open-epochs", fmt.Sprintf("the mesh's record holds %s open beside this controller's epoch %d: a "+
+			"holder that neither gave the lease back nor was found expired", strings.Join(open, ", "), st.Epoch),
+			strings.Join(open, ", "))
+	}
+	// And no message from a stale epoch in the last interval.
+	for _, w := range link.StaleRefusals.Within(time.Now().Add(-doctorEvery)) {
+		if w.Epoch <= 0 || uint64(w.Epoch) >= st.Epoch {
+			continue
+		}
+		out = append(out, conditions.Observation{Scope: conditions.ScopeCore, ID: fmt.Sprintf("controller.epoch-%d", w.Epoch),
+			Token: "stale-epoch", Machine: firstOf(w.Receivers), Severity: conditions.Urgent,
+			Summary: fmt.Sprintf("%d declaration(s) from epoch %d — older than this controller's %d — reached %s in the "+
+				"last %s and were refused: a controller that lost the lease is still sending", w.Count, w.Epoch, st.Epoch,
+				strings.Join(w.Receivers, ", "), doctorEvery),
+			Said: fmt.Sprintf("%d refused, the last at %s", w.Count, w.Last.UTC().Format(time.RFC3339))})
+	}
+	return sortedFound(out), nil
+}