Act under a lease, keep accounts by order, one writer at composition (hq to-be 45 Phase 2)

Two controllers could both act (issue 204), a reconcile's report could
overtake the apply after it and the digest decided (issue 267), and a grant
could make a second writer of a machine's report.

- The lease (internal/lease, ADR 0229): mesh-controller_lease key `holder`,
  15 s age, renewed every 5 s by compare-and-set; the epoch is the revision
  it was taken at. The gate is the clock (stops 3 s before expiry); a refused
  renewal is a loss and the process exits; a holder that stops gives it back.
  serve takes it before asserting the bus. Epochs kept in the store
  (migration 0068 controller_epoch) as a floor: a bucket raised from nothing
  is compacted past it. Unleased (no epoch, S12 urgent) only when nobody
  holds it and the bus will not let it be written. A shell command acts
  under the holder's epoch, or its own lease when none.
- Declarations carry `epoch` inside the signed envelope, only to a machine
  whose latest account carried a report_sequence (mesh-host #35); would-send
  is composed with the epoch last sent. Allot and the send both pass the gate.
- Reports: contract in internal/link/order.go (epoch, sequence,
  report_sequence, older_than, refused_older). Accounts kept by epoch, then
  sequence, then report sequence; older refused, counted; unordered reports
  keep the digest rule. Plans by compare-and-set on a revision, with epoch.
  Conditions and calls carry the epoch and are not written off the lease.
- S12 and S13 (naming the writer by epoch) watched, D5 run; reset of the
  bucket said. Writers table compiled in and enforced in PermissionsFor; the
  controller no longer publishes mesh.control.>. A contract per consumed
  kind, and the empty-on-error lint over the repository.
- mesh-host pinned to its main with the epoch in the validator (D1 validates
  the envelope as sent).

Needs mesh-host's genesis lock with the lease grant (mesh-host PR) for
TestTheInstallersFirstUserListIsWhatTheControllerWouldCompose.

internal/broker/controller_buckets.go

@@ -31,8 +31,15 @@ var (
 	HandActsBucket         = BucketName(ControllerSeat, "hand-acts")
 	ConditionsBucket       = BucketName(ControllerSeat, "conditions")
 	ConditionHistoryBucket = BucketName(ControllerSeat, "condition-history")
+	// LeaseBucket holds the controller's lease (to-be 45 §6): one key, `holder`, which the instance
+	// allowed to act writes by compare-and-set and renews; its revision when taken is the epoch.
+	LeaseBucket = BucketName(ControllerSeat, "lease")
 )
 
+// LeaseTTL is how long the lease's key lives unrenewed (to-be 45 §6): fifteen seconds, renewed
+// every five. The bucket's age, so the bus forgets a holder that stopped renewing.
+const LeaseTTL = 15 * time.Second
+
 // The bounds to-be 45 §6 sets for calls: the last thousand, or fourteen days, whichever is fewer.
 // A call is two keys — its record, and its answer apart so a listing does not read every answer —
 // so the stream holds twice as many messages as it keeps calls.
@@ -52,12 +59,12 @@ const (
 // IsControllerBucket says a bucket is the controller's own, not a module's state nothing declares.
 func IsControllerBucket(bucket string) bool {
 	return bucket == CallsBucket || bucket == HandActsBucket || bucket == ConditionsBucket ||
-		bucket == ConditionHistoryBucket
+		bucket == ConditionHistoryBucket || bucket == LeaseBucket
 }
 
 // ControllerBuckets are the controller's own buckets, in the order they are asserted.
 func ControllerBuckets() []string {
-	return []string{CallsBucket, HandActsBucket, ConditionsBucket, ConditionHistoryBucket}
+	return []string{LeaseBucket, CallsBucket, HandActsBucket, ConditionsBucket, ConditionHistoryBucket}
 }
 
 // ControllerBucketsAsserter is what raising the controller's buckets needs of a connection.
@@ -74,6 +81,9 @@ func (j *JetStream) EnsureControllerBuckets() error {
 	}
 	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
 	defer cancel()
+	if err := EnsureLeaseBucket(ctx, js); err != nil {
+		return err
+	}
 	if _, err := js.CreateOrUpdateKeyValue(ctx, jetstream.KeyValueConfig{
 		Bucket: CallsBucket,
 		Description: "the calls of the mesh's own verbs and what came of each (novox/hq to-be 45 §6, issue " +
@@ -141,3 +151,24 @@ func (j *JetStream) EnsureControllerBuckets() error {
 	}
 	return nil
 }
+
+// EnsureLeaseBucket creates the lease's bucket if absent and brings its options to match (to-be 45
+// §6). **Before the lease is taken, by any candidate**: it is the lease's own precondition, and asserting
+// a bucket that exists changes nothing. File storage, so its revisions — the epochs — outlive a restart of
+// the bus; one raised again from nothing is moved past the highest epoch issued when the lease is taken.
+func EnsureLeaseBucket(ctx context.Context, js jetstream.JetStream) error {
+	if _, err := js.CreateOrUpdateKeyValue(ctx, jetstream.KeyValueConfig{
+		Bucket: LeaseBucket,
+		Description: "the controller's lease (novox/hq to-be 45 §6): the key `holder`, written by compare-and-set " +
+			"by the one controller instance that may act and renewed every five seconds; its revision when taken " +
+			"is the epoch every declaration and plan write carries",
+		History:      1,
+		TTL:          LeaseTTL,
+		MaxValueSize: 4 << 10,
+		MaxBytes:     1 << 20,
+		Storage:      jetstream.FileStorage,
+	}); err != nil {
+		return fmt.Errorf("asserting bucket %s: %w", LeaseBucket, err)
+	}
+	return nil
+}

internal/broker/nats.go

@@ -229,7 +229,12 @@ func PermissionsFor(p Principal) (Permissions, error) {
 		// stream definitions (design 25 §3), so it alone reaches the JetStream API — and it alone
 		// issues memberships (novox/hq ADR 0160), which it publishes into the assignments stream
 		// after each push; refused by the server on 2026-10-01 until this line named them.
-		pub = []string{"mesh.control.>", "mesh.node.>", "mesh.assignment.>", "$JS.API.>"}
+		//
+		// **Not what the machines say** (novox/hq to-be 45 §1): `mesh.control.>` is subscribed, never
+		// published — a machine's report has one writer, its node-engine, and the controller granted
+		// that subject would be a second (the writers table refuses it). It was granted from the first
+		// composition and nothing ever published under it.
+		pub = []string{"mesh.node.>", "mesh.assignment.>", "$JS.API.>"}
 		// **And where its consumers deliver.** A push consumer delivers on `_DELIVER.<its name>`,
 		// and a client bound to it subscribes exactly that; the server refused it for every
 		// principal the first time one bound a consumer (2026-09-28). Each kind below is granted
@@ -604,6 +609,11 @@ func PermissionsFor(p Principal) (Permissions, error) {
 
 	sort.Strings(pub)
 	sort.Strings(sub)
+	// One writer per piece of state (novox/hq to-be 45 §1): a grant that would make a second is
+	// refused here, at composition, naming the state and its writer.
+	if err := CheckWriters(p, pub); err != nil {
+		return Permissions{}, err
+	}
 	return Permissions{
 		Publish:   pub,
 		Subscribe: sub,

internal/broker/testdata/composed.conf

@@ -24,7 +24,7 @@ accounts {
     jetstream: enabled
     users = [
       { user: "controller", password: "$2a$11$cccccccccccccccccccccc", permissions: {
-        publish: { allow: ["$JS.ACK.CONTROL.controller.>", "$JS.ACK.EVENTS.controller.>", "$JS.API.>", "$KV.SEAT_MESH_BUILD_MACHINE_cancelled.>", "$KV.SEAT_NODE_BUILD_AGENT_cancelled.>", "$KV.mesh-controller_calls.>", "$KV.mesh-controller_condition-history.>", "$KV.mesh-controller_conditions.>", "$KV.mesh-controller_hand-acts.>", "$SRV.INFO", "_INBOX.enrol.>", "mesh.assignment.>", "mesh.control.>", "mesh.mod.*.tool.>", "mesh.node.>", "mesh.seat.mesh-build-machine.accept.>", "mesh.seat.mesh-build-machine.tool.>", "mesh.seat.mesh-controller.event.applied", "mesh.seat.mesh-controller.event.built-before", "mesh.seat.mesh-controller.event.condition-changed", "mesh.seat.mesh-controller.event.condition-cleared", "mesh.seat.mesh-controller.event.condition-raised", "mesh.seat.mesh-controller.event.doctor-heartbeat", "mesh.seat.mesh-controller.event.refused", "mesh.seat.mesh-controller.event.secret-replaced", "mesh.seat.node-build-agent.accept.>", "mesh.seat.node-build-agent.tool.>", "mesh.seat.node-intrusion-prevention.tool.banned.*"] }
+        publish: { allow: ["$JS.ACK.CONTROL.controller.>", "$JS.ACK.EVENTS.controller.>", "$JS.API.>", "$KV.SEAT_MESH_BUILD_MACHINE_cancelled.>", "$KV.SEAT_NODE_BUILD_AGENT_cancelled.>", "$KV.mesh-controller_calls.>", "$KV.mesh-controller_condition-history.>", "$KV.mesh-controller_conditions.>", "$KV.mesh-controller_hand-acts.>", "$KV.mesh-controller_lease.>", "$SRV.INFO", "_INBOX.enrol.>", "mesh.assignment.>", "mesh.mod.*.tool.>", "mesh.node.>", "mesh.seat.mesh-build-machine.accept.>", "mesh.seat.mesh-build-machine.tool.>", "mesh.seat.mesh-controller.event.applied", "mesh.seat.mesh-controller.event.built-before", "mesh.seat.mesh-controller.event.condition-changed", "mesh.seat.mesh-controller.event.condition-cleared", "mesh.seat.mesh-controller.event.condition-raised", "mesh.seat.mesh-controller.event.doctor-heartbeat", "mesh.seat.mesh-controller.event.refused", "mesh.seat.mesh-controller.event.secret-replaced", "mesh.seat.node-build-agent.accept.>", "mesh.seat.node-build-agent.tool.>", "mesh.seat.node-intrusion-prevention.tool.banned.*"] }
         subscribe: { allow: ["$JS.API.>", "$JS.EVENT.ADVISORY.CONSUMER.DELETED.>", "$JS.EVENT.ADVISORY.CONSUMER.MAX_DELIVERIES.>", "$SRV.INFO", "$SRV.INFO.mesh-controller", "$SRV.INFO.mesh-controller.>", "$SRV.PING", "$SRV.PING.mesh-controller", "$SRV.PING.mesh-controller.>", "$SRV.STATS", "$SRV.STATS.mesh-controller", "$SRV.STATS.mesh-controller.>", "_DELIVER.controller", "_DELIVER.controller.>", "_INBOX.controller.>", "mesh.control.>", "mesh.mod.*.event.provisioner.failing", "mesh.mod.*.event.provisioner.recovered", "mesh.mod.gitea.event.pull.merged", "mesh.mod.mesh-catalog.event.catching-up", "mesh.mod.mesh-catalog.event.upgraded", "mesh.seat.mesh-build-machine.event.built", "mesh.seat.mesh-controller.tool.>", "mesh.seat.node-build-agent.event.built"] }
         allow_responses: { max: 1, ttl: "1m" }
       } }

internal/broker/writers.go

@@ -0,0 +1,167 @@
+package broker
+
+import (
+	"fmt"
+	"slices"
+	"strings"
+)
+
+// The writers table (novox/hq to-be 45 §1, ADR 0227 rule 1), compiled in.
+//
+// **Every kind of state the core keeps has one writer; anyone else asks it.** The table is the design's,
+// row for row, with what each row writes on the bus where it writes there. Two things read it: the
+// composition of every principal's grants (PermissionsFor), which **refuses a grant that lets a
+// principal publish on a subject another writes** — so a second writer cannot be granted by accident,
+// and the composition says which state and whose — and the test beside it, which walks the rows
+// against the design's list and the composition of a whole mesh. Changing a writer is a change to
+// this table, through a decision.
+
+// WriterRow is one row of the writers table.
+type WriterRow struct {
+	State  string
+	Writer string
+	KeptIn string
+	Others string
+	// Subjects are the bus subjects a write of this state is a publish to; none for state kept off the
+	// bus (the controller's store, a module's own) or not built yet.
+	Subjects []string
+	// Writes says a principal granted a publish pattern overlapping Subjects is this state's writer —
+	// given the pattern, because a machine writes its own report and no other's.
+	Writes func(p Principal, pattern string) bool
+	// Shared says why more than one principal may publish here; empty for one writer.
+	Shared string
+}
+
+// isController, and the other writers' tests, are who a row's writer is as a principal.
+func isController(p Principal, _ string) bool { return p.Kind == KindController }
+
+// ownMachine is a node writing a subject whose third token is its own name, and no wildcard there.
+func ownMachine(p Principal, pattern string) bool {
+	tokens := strings.Split(pattern, ".")
+	return p.Kind == KindNode && len(tokens) > 2 && tokens[2] == p.Node
+}
+
+// holdsSeatOf is a principal holding the seat a `mesh.seat.<seat>.…` pattern names: its module's own
+// principal, or the node tools carrying a module that holds it (ADR 0175, the runtime is its modules).
+func holdsSeatOf(p Principal, pattern string) bool {
+	tokens := strings.Split(pattern, ".")
+	if len(tokens) < 3 {
+		return false
+	}
+	seat := tokens[2]
+	holds := func(seats []Seat) bool {
+		return slices.ContainsFunc(seats, func(s Seat) bool { return s.Name == seat })
+	}
+	switch p.Kind {
+	case KindModule:
+		return holds(p.Holds)
+	case KindNodeTools:
+		return slices.ContainsFunc(p.Carries, func(d Declared) bool { return holds(d.Holds) })
+	}
+	return false
+}
+
+// ownModule is a module publishing under its own name, or the node tools carrying it.
+func ownModule(p Principal, pattern string) bool {
+	tokens := strings.Split(pattern, ".")
+	if len(tokens) < 3 {
+		return false
+	}
+	module := tokens[2]
+	switch p.Kind {
+	case KindModule:
+		return p.Module == module
+	case KindNodeTools:
+		return slices.ContainsFunc(p.Carries, func(d Declared) bool { return d.Module == module })
+	}
+	return false
+}
+
+// kvOf is a bucket's write subjects.
+func kvOf(bucket string) []string { return []string{"$KV." + bucket + ".>"} }
+
+// WritersTable is to-be 45 §1, in its order.
+var WritersTable = []WriterRow{
+	{State: "a machine's declaration", Writer: "controller (lease holder)", KeptIn: "the bus, last per subject",
+		Others: "read", Subjects: []string{"mesh.node.*.declare"}, Writes: isController},
+	{State: "a machine's applied state and its report", Writer: "the node-engine's apply queue",
+		KeptIn: "the machine; the report on the bus", Others: "the reconcile and a delivery enqueue, never apply",
+		Subjects: []string{"mesh.control.*.report"}, Writes: ownMachine},
+	{State: "the controller lease", Writer: "the controller instance holding it", KeptIn: "key-value " + LeaseBucket,
+		Others: "a candidate waits", Subjects: kvOf(LeaseBucket), Writes: isController},
+	{State: "plans and their tiers", Writer: "controller (lease holder), compare-and-set on the plan's revision",
+		KeptIn: "the controller's store", Others: "read through plans"},
+	{State: "conditions", Writer: "controller", KeptIn: "key-value " + ConditionsBucket + " (and its history, " +
+		ConditionHistoryBucket + ")", Others: "raise or clear only through observations the controller reads",
+		Subjects: append(kvOf(ConditionsBucket), kvOf(ConditionHistoryBucket)...), Writes: isController},
+	{State: "calls and their outcomes", Writer: "controller", KeptIn: "key-value " + CallsBucket,
+		Others: "read by id", Subjects: kvOf(CallsBucket), Writes: isController},
+	{State: "the hand-act log", Writer: "controller, through the verbs that act", KeptIn: "key-value " + HandActsBucket,
+		Others: "—", Subjects: kvOf(HandActsBucket), Writes: isController},
+	{State: "stream definitions and bus permissions", Writer: "controller", KeptIn: "the bus", Others: "—",
+		// A stream's definition, and a durable consumer's by the API that names it so. Not every
+		// consumer create: a module watching its own bucket makes and deletes an ordered consumer on the
+		// bucket's stream (ADR 0201), which defines nothing the mesh keeps.
+		Subjects: []string{"$JS.API.STREAM.CREATE.>", "$JS.API.STREAM.UPDATE.>", "$JS.API.STREAM.DELETE.>",
+			"$JS.API.CONSUMER.DURABLE.CREATE.>"},
+		Writes: isController},
+	{State: "builds and their outcomes", Writer: "the build seat's holder", KeptIn: "its own state",
+		Others:   "the controller asks",
+		Subjects: []string{"mesh.seat.node-build-agent.event.built", "mesh.seat.mesh-build-machine.event.built"},
+		Writes:   holdsSeatOf,
+		Shared:   "every machine holding the build seat answers the asks it took; each outcome names its ask"},
+	{State: "a merge announced", Writer: "one announcer per forge (the hook, or the poll when the hook is absent — never both)",
+		KeptIn: "the bus", Others: "—", Subjects: []string{"mesh.mod.*.event.pull.merged"}, Writes: ownModule},
+	{State: "a provider's standing", Writer: "the provider", KeptIn: "the provider's events",
+		Others:   "the controller keeps the newest word as a condition",
+		Subjects: []string{"mesh.mod.*.event.provisioner.failing", "mesh.mod.*.event.provisioner.recovered"},
+		Writes:   ownModule},
+	{State: "the operator-channel's open messages", Writer: "the seat's holder", KeptIn: "its own key-value state",
+		Others: "—"},
+	{State: "the facts snapshot", Writer: "controller", KeptIn: "the artifact store, facts/latest",
+		Others: "the build seat reads"},
+}
+
+// CheckWriters refuses a grant that lets a principal publish on a subject the writers table gives
+// another writer (to-be 45 §1): which state, whose, and the pattern that would make a second writer.
+func CheckWriters(p Principal, publish []string) error {
+	var problems []string
+	for _, pattern := range publish {
+		for _, row := range WritersTable {
+			if row.Writes == nil {
+				continue
+			}
+			for _, subject := range row.Subjects {
+				if !SubjectsOverlap(pattern, subject) || row.Writes(p, pattern) {
+					continue
+				}
+				problems = append(problems, fmt.Sprintf("%s may publish %s, which writes %s (%s), whose writer is %s",
+					p.Username(), pattern, row.State, subject, row.Writer))
+			}
+		}
+	}
+	if len(problems) == 0 {
+		return nil
+	}
+	return fmt.Errorf("a second writer would be granted (novox/hq to-be 45 §1, one writer per piece of state):\n  %s",
+		strings.Join(problems, "\n  "))
+}
+
+// SubjectsOverlap says some subject matches both patterns: `*` is one token, `>` one or more to the end.
+func SubjectsOverlap(a, b string) bool {
+	x, y := strings.Split(a, "."), strings.Split(b, ".")
+	for i := 0; ; i++ {
+		switch {
+		case i == len(x) && i == len(y):
+			return true
+		case i == len(x) || i == len(y):
+			return false
+		case x[i] == ">" || y[i] == ">":
+			return true
+		case x[i] == "*" || y[i] == "*" || x[i] == y[i]:
+			continue
+		default:
+			return false
+		}
+	}
+}

internal/broker/writers_test.go

@@ -0,0 +1,143 @@
+package broker
+
+import (
+	"slices"
+	"strings"
+	"testing"
+)
+
+// The writers table, checked (novox/hq to-be 45 §1, ADR 0227 rule 1, "how it is checked"): it is the
+// design's table row for row; every row that writes on the bus names its writer; a whole mesh composes
+// with one writer per piece of state; and a grant that would make a second writer is refused, naming
+// the state and whose it is.
+
+// designRows are the states of to-be 45 §1, in its order. A row added to the design is added here and
+// to the table; one dropped from either fails.
+var designRows = []string{
+	"a machine's declaration",
+	"a machine's applied state and its report",
+	"the controller lease",
+	"plans and their tiers",
+	"conditions",
+	"calls and their outcomes",
+	"the hand-act log",
+	"stream definitions and bus permissions",
+	"builds and their outcomes",
+	"a merge announced",
+	"a provider's standing",
+	"the operator-channel's open messages",
+	"the facts snapshot",
+}
+
+func TestTheWritersTableIsTheDesigns(t *testing.T) {
+	var states []string
+	for _, row := range WritersTable {
+		states = append(states, row.State)
+		if row.Writer == "" || row.KeptIn == "" || row.Others == "" {
+			t.Errorf("%q does not say who writes it, where it is kept and what others do", row.State)
+		}
+		if len(row.Subjects) > 0 && row.Writes == nil {
+			t.Errorf("%q is written on the bus and names no writer among the principals", row.State)
+		}
+	}
+	if !slices.Equal(states, designRows) {
+		t.Fatalf("the writers table is not to-be 45 §1's:\n have %q\n want %q", states, designRows)
+	}
+}
+
+// A mesh of every kind of principal composes: nobody is granted a second writer's subject.
+func TestAWholeMeshComposesWithOneWriterPerState(t *testing.T) {
+	builder := Seat{Name: "node-build-agent", Scope: "node", Accepts: []string{"build"}, Emits: []string{"built", "started"}}
+	principals := []Principal{
+		{Kind: KindController, PasswordHash: "x"},
+		{Kind: KindNode, Node: "one", PasswordHash: "x"},
+		{Kind: KindEnrolment, Node: "two", PasswordHash: "x"},
+		{Kind: KindPerson, Module: "jochen", Invokes: []string{"*"}, PasswordHash: "x"},
+		{Kind: KindModule, Node: "one", Module: "gitea", Emits: []string{"pull.merged"}, PasswordHash: "x"},
+		{Kind: KindModule, Node: "one", Module: "postgres", Emits: []string{"provisioner.failing", "provisioner.recovered"},
+			PasswordHash: "x"},
+		{Kind: KindModule, Node: "one", Module: "build-agent", Holds: []Seat{builder}, PasswordHash: "x"},
+		{Kind: KindNodeTools, Node: "one", Module: RuntimeModule, PasswordHash: "x", Carries: []Declared{
+			{Module: "gitea", Emits: []string{"pull.merged"}},
+			{Module: "build-agent", Holds: []Seat{builder}}}},
+	}
+	for _, p := range principals {
+		if _, err := PermissionsFor(p); err != nil {
+			t.Errorf("%s does not compose: %v", p.Username(), err)
+		}
+	}
+	if _, err := ComposeAccounts(principals); err != nil {
+		t.Fatalf("the mesh does not compose: %v", err)
+	}
+}
+
+func TestASecondWriterIsRefusedAtComposition(t *testing.T) {
+	for _, c := range []struct {
+		name    string
+		p       Principal
+		publish []string
+		state   string
+	}{
+		{"the controller publishing what machines say", Principal{Kind: KindController},
+			[]string{"mesh.control.>"}, "a machine's applied state and its report"},
+		{"a machine publishing another's report", Principal{Kind: KindNode, Node: "one"},
+			[]string{"mesh.control.two.report"}, "a machine's applied state and its report"},
+		{"a machine publishing every machine's", Principal{Kind: KindNode, Node: "one"},
+			[]string{"mesh.control.*.>"}, "a machine's applied state and its report"},
+		{"a module writing the lease", Principal{Kind: KindModule, Module: "shop"},
+			[]string{"$KV.mesh-controller_lease.>"}, "the controller lease"},
+		{"a module sending a declaration", Principal{Kind: KindModule, Module: "shop"},
+			[]string{"mesh.node.one.declare"}, "a machine's declaration"},
+		{"a module defining a stream", Principal{Kind: KindModule, Module: "shop"},
+			[]string{"$JS.API.>"}, "stream definitions and bus permissions"},
+		{"a module announcing another forge's merge", Principal{Kind: KindModule, Module: "shop"},
+			[]string{"mesh.mod.gitea.event.pull.merged"}, "a merge announced"},
+		{"a module saying a build it did not do", Principal{Kind: KindModule, Module: "shop"},
+			[]string{"mesh.seat.node-build-agent.event.built"}, "builds and their outcomes"},
+	} {
+		t.Run(c.name, func(t *testing.T) {
+			err := CheckWriters(c.p, c.publish)
+			if err == nil {
+				t.Fatalf("%v granted to %s was not refused", c.publish, c.p.Username())
+			}
+			if !strings.Contains(err.Error(), c.state) {
+				t.Fatalf("the refusal does not name %q: %v", c.state, err)
+			}
+		})
+	}
+	// And the writers themselves are not refused.
+	for _, ok := range []struct {
+		p       Principal
+		publish []string
+	}{
+		{Principal{Kind: KindNode, Node: "one"}, []string{"mesh.control.one.>"}},
+		{Principal{Kind: KindController}, []string{"mesh.node.>", "$JS.API.>", "$KV.mesh-controller_lease.>"}},
+		{Principal{Kind: KindModule, Module: "gitea"}, []string{"mesh.mod.gitea.event.pull.merged"}},
+		{Principal{Kind: KindModule, Module: "shop"}, []string{"$JS.API.CONSUMER.CREATE.KV_shop_carts.>"}},
+	} {
+		if err := CheckWriters(ok.p, ok.publish); err != nil {
+			t.Errorf("a writer was refused its own: %v", err)
+		}
+	}
+}
+
+func TestSubjectsOverlap(t *testing.T) {
+	for _, c := range []struct {
+		a, b string
+		want bool
+	}{
+		{"mesh.control.>", "mesh.control.*.report", true},
+		{"mesh.control.one.>", "mesh.control.*.report", true},
+		{"mesh.control.one.alive", "mesh.control.*.report", false},
+		{"mesh.control.*", "mesh.control.*.report", false},
+		{"$JS.API.>", "$JS.API.STREAM.CREATE.>", true},
+		{"$JS.API.CONSUMER.CREATE.KV_x.>", "$JS.API.STREAM.CREATE.>", false},
+		{"a.b", "a.b", true},
+		{"a.b", "a.b.c", false},
+		{"a.>", "a", false},
+	} {
+		if got := SubjectsOverlap(c.a, c.b); got != c.want || SubjectsOverlap(c.b, c.a) != c.want {
+			t.Errorf("%s ~ %s: %v, want %v", c.a, c.b, got, c.want)
+		}
+	}
+}

internal/conditions/condition.go

@@ -129,9 +129,8 @@ type Condition struct {
 	Resolver string    `json:"resolver"`
 	// Silenced is null when no silence is in force: said, not left out, so a reader need not guess.
 	Silenced *Silence `json:"silenced"`
-	// Epoch is the controller lease epoch that last wrote it. Zero until the lease exists (to-be 45
-	// Phase 2): no controller holds an epoch yet, and a number invented here would be one nobody
-	// could compare.
+	// Epoch is the controller lease epoch that last wrote it (to-be 45 §6). Zero where it was written
+	// by a controller serving without the lease, or before the lease existed.
 	Epoch uint64 `json:"epoch"`
 }
 

internal/conditions/store.go

@@ -56,6 +56,7 @@ type Keeper struct {
 	now     func() time.Time
 	say     func(format string, args ...any)
 	changed func()
+	epoch   func() (uint64, error)
 
 	mu sync.Mutex
 	// cleared is when each recently cleared condition cleared and how often it had been raised, so
@@ -88,6 +89,9 @@ type Options struct {
 	Say func(format string, args ...any)
 	// Changed is told of every transition, at once — for `status`, which leads with what is open.
 	Changed func()
+	// Epoch is the controller lease this keeper writes under (to-be 45 §6): every write carries its
+	// epoch, and none is made while it is not held. Nil writes with no epoch (a test, the memory store).
+	Epoch func() (uint64, error)
 }
 
 // TellFor is how long one transition is offered to the bus before it is said lost.
@@ -97,6 +101,7 @@ var TellFor = 10 * time.Minute
 // that cleared just before this controller started and is raised again now is a reopening.
 func NewKeeper(ctx context.Context, o Options) *Keeper {
 	k := &Keeper{store: o.Store, history: o.History, teller: o.Teller, now: o.Now, say: o.Say, changed: o.Changed,
+		epoch:   o.Epoch,
 		cleared: map[string]clearing{}, out: make(chan Event, 1024), drained: make(chan struct{})}
 	if k.now == nil {
 		k.now = time.Now
@@ -137,6 +142,20 @@ func (k *Keeper) Unsaid() int {
 	return k.unsaid
 }
 
+// stamp is the gate every write passes: the epoch it carries, set on the condition, or why this
+// keeper may not write — a controller that lost the lease raises, observes and clears nothing.
+func (k *Keeper) stamp(c *Condition) error {
+	if k.epoch == nil {
+		return nil
+	}
+	epoch, err := k.epoch()
+	if err != nil {
+		return fmt.Errorf("the condition %s is not written: %w", c.Key, err)
+	}
+	c.Epoch = epoch
+	return nil
+}
+
 // tries bounds one compare-and-set: two writers rarely race more than once.
 const tries = 8
 
@@ -174,6 +193,9 @@ func (k *Keeper) Observe(ctx context.Context, o Observation) (Condition, error)
 				}
 			}
 			k.mu.Unlock()
+			if err := k.stamp(&c); err != nil {
+				return Condition{}, err
+			}
 			body, err := json.Marshal(c)
 			if err != nil {
 				return Condition{}, err
@@ -214,6 +236,9 @@ func (k *Keeper) Observe(ctx context.Context, o Observation) (Condition, error)
 		if len(c.Evidence) > KeptEvidence {
 			c.Evidence = c.Evidence[:KeptEvidence]
 		}
+		if err := k.stamp(&c); err != nil {
+			return Condition{}, err
+		}
 		body, err := json.Marshal(c)
 		if err != nil {
 			return Condition{}, err
@@ -247,6 +272,9 @@ func (k *Keeper) Clear(ctx context.Context, key, why string) (bool, error) {
 			// Unreadable is not resolved: kept, and said, rather than removed unread.
 			return false, fmt.Errorf("the condition %s on the bus cannot be read, so it is not cleared: %w", key, err)
 		}
+		if err := k.stamp(&c); err != nil {
+			return false, err
+		}
 		if err := k.store.Delete(ctx, key, entry.Revision); errors.Is(err, ErrMoved) {
 			continue
 		} else if err != nil {
@@ -318,6 +346,9 @@ func (k *Keeper) Silence(ctx context.Context, key string, d time.Duration, by, w
 		}
 		now := k.now().UTC()
 		c.Silenced = &Silence{Until: now.Add(d), By: by, Why: strings.TrimSpace(why), Since: now}
+		if err := k.stamp(&c); err != nil {
+			return Condition{}, err
+		}
 		body, err := json.Marshal(c)
 		if err != nil {
 			return Condition{}, err
@@ -362,6 +393,9 @@ func (k *Keeper) EndSilences(ctx context.Context) error {
 			}
 			was := held.Silenced.Why
 			held.Silenced = nil
+			if err := k.stamp(&held); err != nil {
+				return err
+			}
 			body, err := json.Marshal(held)
 			if err != nil {
 				return err

internal/inventory/durations_test.go

@@ -68,25 +68,25 @@ func TestAPlansTierIsMeasuredWhenItIsLeft(t *testing.T) {
 	p := Plan{ID: "plan-1", Repository: "novox/mesh-tools", Commit: "abc", Created: time.Now().UTC(),
 		State: PlanBuilding, Tiers: [][]string{{"mesh-tools"}, {"builder"}},
 		Modules: map[string]*PlanModule{"mesh-tools": {}, "builder": {}}}
-	if err := inv.SavePlan(ctx, p); err != nil {
+	if err := inv.SavePlan(ctx, &p); err != nil {
 		t.Fatal(err)
 	}
 	p.State = PlanRolling // the same tier, saved again
-	if err := inv.SavePlan(ctx, p); err != nil {
+	if err := inv.SavePlan(ctx, &p); err != nil {
 		t.Fatal(err)
 	}
 	if ds, _ := inv.Durations(ctx, DurationPlanTier, time.Now().Add(-time.Hour)); len(ds) != 0 {
 		t.Fatalf("a tier not left was measured: %+v", ds)
 	}
 	p.Tier = 1
-	if err := inv.SavePlan(ctx, p); err != nil {
+	if err := inv.SavePlan(ctx, &p); err != nil {
 		t.Fatal(err)
 	}
 	p.State = PlanDone
-	if err := inv.SavePlan(ctx, p); err != nil {
+	if err := inv.SavePlan(ctx, &p); err != nil {
 		t.Fatal(err)
 	}
-	if err := inv.SavePlan(ctx, p); err != nil { // saved again once ended: nothing more to measure
+	if err := inv.SavePlan(ctx, &p); err != nil { // saved again once ended: nothing more to measure
 		t.Fatal(err)
 	}
 	ds, err := inv.Durations(ctx, DurationPlanTier, time.Now().Add(-time.Hour))

internal/inventory/migrations/0068-order-and-one-writer.sql

@@ -0,0 +1,40 @@
+-- Order and one writer (novox/hq to-be 45 §6, Phase 2).
+--
+-- The controller acts only while it holds its lease, and every act carries the lease's epoch: a
+-- declaration, a plan write. A report carries the order of the declaration it accounts for, and the
+-- controller keeps the highest per machine, refusing an older account rather than letting the last
+-- one written win (issue 267). These are the records each of those needs.
+
+-- Every epoch the mesh issued: which controller instance held it, from when, and how it ended —
+-- given back, or lost (its key expired, or a renewal was refused). The highest one is the floor no
+-- new epoch may be at or under, even when the lease's bucket was raised again from nothing; and a
+-- stale refusal names its writer from here.
+create table controller_epoch (
+    epoch    bigint      primary key,
+    instance text        not null,
+    host     text        not null default '',
+    build    text        not null default '',
+    taken    timestamptz not null default now(),
+    ended    timestamptz,
+    -- how: 'released' (given back), 'lost' (its own renewal was refused or failed), 'expired'
+    -- (found expired by the next holder: it stopped renewing without saying so).
+    how      text        not null default ''
+);
+
+-- The epoch a machine was last sent, so what the mesh WOULD send is composed with it and reads as
+-- byte for byte what it DID send when nothing else changed — a new holder's epoch is not a change
+-- of the machine. Null for a declaration sent without one.
+alter table node add column sent_epoch bigint;
+-- Whether the machine's node-engine said it reads an epoch in a declaration (its report's `reads`):
+-- until it has, it is sent none, because an older node-engine refuses a key it does not know, whole.
+alter table node add column reads_epoch boolean not null default false;
+
+-- The order of the account kept for each machine: the declaration's sequence and epoch it is about
+-- and the node-engine's own report sequence. Null where the kept account carried none.
+alter table node_report add column reported_sequence bigint;
+alter table node_report add column reported_epoch bigint;
+alter table node_report add column report_sequence bigint;
+
+-- A plan is written by compare-and-set on its revision, and says the epoch that wrote it last.
+alter table release_plan add column revision bigint not null default 0;
+alter table release_plan add column epoch bigint;

internal/inventory/nodes.go

@@ -14,12 +14,17 @@ import (
 	"time"
 
 	"github.com/jackc/pgx/v5"
+	"github.com/jackc/pgx/v5/pgconn"
 	"github.com/novox/mesh-controller/internal/broker"
 	"github.com/novox/mesh-controller/internal/store"
 )
 
 // Inventory is this context, holding the store it exclusively owns.
-type Inventory struct{ store *store.Store }
+type Inventory struct {
+	store *store.Store
+	// acting is the gate a write that acts passes (ActsUnder, novox/hq to-be 45 §6); nil passes all.
+	acting func(ctx context.Context) (uint64, error)
+}
 
 // Open connects to the inventory store.
 func Open(ctx context.Context) (*Inventory, error) {
@@ -763,14 +768,35 @@ func sameFailure(a, b Doing) bool {
 // ADR 0134). A machine reconciles continuously and reports each time; the same outcome about the same
 // declaration is the same state said again, and a fact per report would be a fact per minute per
 // machine that tells nobody anything. Read here because the previous row is read here anyway.
+//
+// **An account that claims no order clears the order kept** (novox/hq to-be 45 §6): the account kept is
+// then one the next ordered report has nothing to compare against, and it is taken as it was before
+// reports carried an order. RecordOrderedDoing keeps an ordered one.
 func (i *Inventory) RecordDoing(ctx context.Context, node string, d Doing) (news bool, err error) {
+	news, err = recordDoing(ctx, i.store.Pool(), node, d)
+	if err != nil {
+		return false, err
+	}
+	_, err = i.store.Pool().Exec(ctx,
+		`update node_report set reported_sequence = null, reported_epoch = null, report_sequence = null
+		  where node = $1`, node)
+	return news, err
+}
+
+// queries is what recordDoing writes through: the pool, or a transaction an ordered account holds.
+type queries interface {
+	QueryRow(ctx context.Context, sql string, args ...any) pgx.Row
+	Exec(ctx context.Context, sql string, args ...any) (pgconn.CommandTag, error)
+}
+
+func recordDoing(ctx context.Context, q queries, node string, d Doing) (news bool, err error) {
 	failed, err := json.Marshal(d.Failed)
 	if err != nil {
 		return false, err
 	}
 	var before Doing
 	var beforeFailed []byte
-	found := i.store.Pool().QueryRow(ctx,
+	found := q.QueryRow(ctx,
 		`select outcome, refused, failed, failing_since, failures, coalesce(declared,'')
 		   from node_report where node = $1`,
 		node).Scan(&before.Outcome, &before.Refused, &beforeFailed, &before.Since, &before.Times,
@@ -795,7 +821,7 @@ func (i *Inventory) RecordDoing(ctx context.Context, node string, d Doing) (news
 			since, times = before.Since, before.Times+1
 		}
 	}
-	_, err = i.store.Pool().Exec(ctx,
+	_, err = q.Exec(ctx,
 		`insert into node_report (node, outcome, refused, failed, applied, at, declared,
 		                          failing_since, failures)
 		 values ($1, $2, $3, $4, $5, now(), $6, $7, $8)
@@ -923,6 +949,18 @@ func (i *Inventory) LastReports(ctx context.Context) ([]Reported, error) {
 // is a push's to send to a machine it did not name. Nil records that it is not known, as for a
 // declaration sent by hand.
 func (i *Inventory) RecordSent(ctx context.Context, node, digest string, builds map[string]string) error {
+	return i.RecordSentUnder(ctx, node, digest, builds, 0)
+}
+
+// RecordSentUnder is RecordSent for a declaration that carried an epoch (novox/hq to-be 45 §6): kept
+// beside the digest, so what the mesh would send is composed with it. Zero is one that carried none.
+func (i *Inventory) RecordSentUnder(ctx context.Context, node, digest string, builds map[string]string,
+	epoch uint64) error {
+	var sentEpoch *int64
+	if epoch > 0 {
+		e := int64(epoch)
+		sentEpoch = &e
+	}
 	var carried *string
 	if builds != nil {
 		raw, err := json.Marshal(builds)
@@ -933,7 +971,8 @@ func (i *Inventory) RecordSent(ctx context.Context, node, digest string, builds
 		carried = &text
 	}
 	_, err := i.store.Pool().Exec(ctx,
-		`update node set sent = $2, sent_at = now(), sent_builds = $3::jsonb where id = $1`, node, digest, carried)
+		`update node set sent = $2, sent_at = now(), sent_builds = $3::jsonb, sent_epoch = $4 where id = $1`,
+		node, digest, carried, sentEpoch)
 	return err
 }
 

internal/inventory/order.go

@@ -0,0 +1,250 @@
+package inventory
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"time"
+
+	"github.com/jackc/pgx/v5"
+)
+
+// Order and one writer, as this context keeps them (novox/hq to-be 45 §6, Phase 2): the epochs the
+// mesh issued, the epoch each machine was sent, whether its node-engine reads one, and the order of
+// the account kept for it.
+
+// ActsUnder gives this inventory the gate every write that acts passes (a plan's): the epoch of the
+// controller lease this process acts under, or why it may not act. Nil — a test, a command reading —
+// writes with no epoch and refuses nothing.
+func (i *Inventory) ActsUnder(epoch func(ctx context.Context) (uint64, error)) { i.acting = epoch }
+
+// actingEpoch is the epoch a write carries, nil when there is no gate or it claims none.
+func (i *Inventory) actingEpoch(ctx context.Context) (*int64, error) {
+	if i.acting == nil {
+		return nil, nil
+	}
+	epoch, err := i.acting(ctx)
+	if err != nil {
+		return nil, err
+	}
+	if epoch == 0 {
+		return nil, nil
+	}
+	e := int64(epoch)
+	return &e, nil
+}
+
+// Epoch is one epoch the mesh issued.
+type Epoch struct {
+	Epoch    uint64
+	Instance string
+	Host     string
+	Build    string
+	Taken    time.Time
+	// Ended is when it ended, nil while it is held; How is how: EpochReleased, EpochLost, EpochExpired.
+	Ended *time.Time
+	How   string
+}
+
+// How an epoch ends.
+const (
+	// EpochReleased is a holder that gave the lease back.
+	EpochReleased = "released"
+	// EpochLost is a holder whose own renewal was refused or failed, and which said so.
+	EpochLost = "lost"
+	// EpochExpired is a holder the next one found gone: it stopped renewing without saying anything.
+	EpochExpired = "expired"
+)
+
+// HighestEpoch is the highest epoch the mesh has issued, zero before the first: the floor no new one
+// may be at or under.
+func (i *Inventory) HighestEpoch(ctx context.Context) (uint64, error) {
+	var highest int64
+	if err := i.store.Pool().QueryRow(ctx, `select coalesce(max(epoch), 0) from controller_epoch`).Scan(&highest); err != nil {
+		return 0, fmt.Errorf("reading the highest epoch issued: %w", err)
+	}
+	return uint64(highest), nil
+}
+
+// TookEpoch records an epoch taken, and ends every earlier one still open as found expired: the lease
+// was free to take, so whoever held it last neither holds it nor said it let go. Answers the epochs it
+// ended that way, newest first.
+func (i *Inventory) TookEpoch(ctx context.Context, e Epoch) ([]Epoch, error) {
+	tx, err := i.store.Pool().Begin(ctx)
+	if err != nil {
+		return nil, err
+	}
+	defer func() { _ = tx.Rollback(ctx) }()
+	rows, err := tx.Query(ctx,
+		`update controller_epoch set ended = now(), how = $2
+		  where ended is null and epoch < $1
+		  returning epoch, instance, host, build, taken, ended, how`, int64(e.Epoch), EpochExpired)
+	if err != nil {
+		return nil, err
+	}
+	expired, err := scanEpochs(rows)
+	if err != nil {
+		return nil, err
+	}
+	if _, err := tx.Exec(ctx,
+		`insert into controller_epoch (epoch, instance, host, build, taken) values ($1, $2, $3, $4, $5)
+		 on conflict (epoch) do nothing`,
+		int64(e.Epoch), e.Instance, e.Host, e.Build, e.Taken); err != nil {
+		return nil, err
+	}
+	return expired, tx.Commit(ctx)
+}
+
+// EndEpoch records how an epoch ended. One already ended keeps its first word.
+func (i *Inventory) EndEpoch(ctx context.Context, epoch uint64, how string) error {
+	_, err := i.store.Pool().Exec(ctx,
+		`update controller_epoch set ended = now(), how = $2 where epoch = $1 and ended is null`, int64(epoch), how)
+	return err
+}
+
+// EpochOf is the epoch issued at a number; false when the mesh issued none there.
+func (i *Inventory) EpochOf(ctx context.Context, epoch uint64) (Epoch, bool, error) {
+	rows, err := i.store.Pool().Query(ctx,
+		`select epoch, instance, host, build, taken, ended, how from controller_epoch where epoch = $1`, int64(epoch))
+	if err != nil {
+		return Epoch{}, false, err
+	}
+	found, err := scanEpochs(rows)
+	if err != nil || len(found) == 0 {
+		return Epoch{}, false, err
+	}
+	return found[0], true, nil
+}
+
+// EpochsSince is every epoch taken or ended since a moment, newest first.
+func (i *Inventory) EpochsSince(ctx context.Context, since time.Time) ([]Epoch, error) {
+	rows, err := i.store.Pool().Query(ctx,
+		`select epoch, instance, host, build, taken, ended, how from controller_epoch
+		  where taken >= $1 or ended >= $1 or ended is null order by epoch desc`, since)
+	if err != nil {
+		return nil, err
+	}
+	return scanEpochs(rows)
+}
+
+func scanEpochs(rows pgx.Rows) ([]Epoch, error) {
+	defer rows.Close()
+	var out []Epoch
+	for rows.Next() {
+		var e Epoch
+		var epoch int64
+		if err := rows.Scan(&epoch, &e.Instance, &e.Host, &e.Build, &e.Taken, &e.Ended, &e.How); err != nil {
+			return nil, err
+		}
+		e.Epoch = uint64(epoch)
+		out = append(out, e)
+	}
+	return out, rows.Err()
+}
+
+// SentEpoch is the epoch a machine was last sent, by its id; zero for one sent without.
+func (i *Inventory) SentEpoch(ctx context.Context, id string) (uint64, error) {
+	var epoch *int64
+	if err := i.store.Pool().QueryRow(ctx, `select sent_epoch from node where id = $1`, id).Scan(&epoch); err != nil {
+		return 0, fmt.Errorf("reading the epoch %s was last sent: %w", id, err)
+	}
+	if epoch == nil {
+		return 0, nil
+	}
+	return uint64(*epoch), nil
+}
+
+// ReadsEpoch says a machine's node-engine said it reads an epoch in a declaration, by its id.
+func (i *Inventory) ReadsEpoch(ctx context.Context, id string) (bool, error) {
+	var reads bool
+	if err := i.store.Pool().QueryRow(ctx, `select reads_epoch from node where id = $1`, id).Scan(&reads); err != nil {
+		return false, fmt.Errorf("reading whether %s reads an epoch: %w", id, err)
+	}
+	return reads, nil
+}
+
+// RecordReadsEpoch keeps what a machine's latest report said of reading an epoch. Every report of a
+// node-engine that orders its reports says it, so a node-engine rolled back says it no longer does.
+func (i *Inventory) RecordReadsEpoch(ctx context.Context, id string, reads bool) error {
+	_, err := i.store.Pool().Exec(ctx, `update node set reads_epoch = $2 where id = $1`, id, reads)
+	return err
+}
+
+// ReportOrder is the order of an account: the declaration's epoch and sequence it is about, and the
+// node-engine's own report sequence. Zero in any claims none.
+type ReportOrder struct {
+	Epoch          int64
+	Sequence       int64
+	ReportSequence int64
+}
+
+// ErrOlderAccount is an account refused because the one kept is newer.
+var ErrOlderAccount = errors.New("an older account than the one kept")
+
+// KeptOrder is the order of the account kept for a machine, by its id; zero when it carried none.
+func (i *Inventory) KeptOrder(ctx context.Context, id string) (ReportOrder, error) {
+	o, err := keptOrder(ctx, i.store.Pool(), id, "")
+	if errors.Is(err, pgx.ErrNoRows) {
+		return ReportOrder{}, nil
+	}
+	return o, err
+}
+
+func keptOrder(ctx context.Context, q queries, id, lock string) (ReportOrder, error) {
+	var epoch, seq, rseq *int64
+	err := q.QueryRow(ctx,
+		`select reported_epoch, reported_sequence, report_sequence from node_report where node = $1`+lock, id).
+		Scan(&epoch, &seq, &rseq)
+	if err != nil {
+		return ReportOrder{}, err
+	}
+	var o ReportOrder
+	for _, f := range []struct {
+		from *int64
+		to   *int64
+	}{{epoch, &o.Epoch}, {seq, &o.Sequence}, {rseq, &o.ReportSequence}} {
+		if f.from != nil {
+			*f.to = *f.from
+		}
+	}
+	return o, nil
+}
+
+// RecordOrderedDoing is RecordDoing for an account that carries its order (novox/hq to-be 45 §6):
+// written only when it is not older than the account kept, as olderThan judges — the rule is the
+// link's (link.Account), stated once — and in one transaction with the row locked, so two accounts
+// arriving together cannot both win. ErrOlderAccount when it is older; nothing is written then.
+func (i *Inventory) RecordOrderedDoing(ctx context.Context, id string, d Doing, o ReportOrder,
+	olderThan func(kept ReportOrder) bool) (news bool, err error) {
+	tx, err := i.store.Pool().Begin(ctx)
+	if err != nil {
+		return false, err
+	}
+	defer func() { _ = tx.Rollback(ctx) }()
+	kept, err := keptOrder(ctx, tx, id, " for update")
+	switch {
+	case errors.Is(err, pgx.ErrNoRows):
+	case err != nil:
+		return false, err
+	case olderThan(kept):
+		return false, ErrOlderAccount
+	}
+	news, err = recordDoing(ctx, tx, id, d)
+	if err != nil {
+		return false, err
+	}
+	if _, err := tx.Exec(ctx,
+		`update node_report set reported_epoch = $2, reported_sequence = $3, report_sequence = $4 where node = $1`,
+		id, nullIfZero(o.Epoch), nullIfZero(o.Sequence), nullIfZero(o.ReportSequence)); err != nil {
+		return false, err
+	}
+	return news, tx.Commit(ctx)
+}
+
+// nullIfZero is a claimed order or none.
+func nullIfZero(n int64) *int64 {
+	if n == 0 {
+		return nil
+	}
+	return &n
+}

internal/inventory/order_test.go

@@ -0,0 +1,126 @@
+package inventory
+
+import (
+	"context"
+	"errors"
+	"testing"
+	"time"
+)
+
+// Order and one writer, as the store keeps them (novox/hq to-be 45 §6).
+
+// **A plan is written by compare-and-set on its revision, carrying the epoch**: a write against a plan
+// another writer moved since is refused, and nothing is written by a process that may not act.
+func TestAPlanIsWrittenByCompareAndSetUnderTheLease(t *testing.T) {
+	inv := ForTest(t)
+	ctx := context.Background()
+	epoch := uint64(57)
+	inv.ActsUnder(func(context.Context) (uint64, error) { return epoch, nil })
+
+	p := Plan{ID: "plan-cas", Repository: "novox/app", Commit: "c0ffee00", Created: time.Now(), State: PlanBuilding,
+		Tiers: [][]string{{"app"}}, Modules: map[string]*PlanModule{"app": {}}}
+	if err := inv.SavePlan(ctx, &p); err != nil {
+		t.Fatal(err)
+	}
+	if p.Revision != 1 || p.Epoch != 57 {
+		t.Fatalf("a new plan was written at revision %d, epoch %d", p.Revision, p.Epoch)
+	}
+	// Two readers of revision 1: the first write wins, the second is refused and writes nothing.
+	first, err := inv.PlanByID(ctx, "plan-cas")
+	if err != nil {
+		t.Fatal(err)
+	}
+	second := first
+	first.Note = "the newer word"
+	if err := inv.SavePlan(ctx, &first); err != nil {
+		t.Fatal(err)
+	}
+	second.State = PlanFailed
+	if err := inv.SavePlan(ctx, &second); !errors.Is(err, ErrPlanMoved) {
+		t.Fatalf("a write against a plan moved since it was read was not refused: %v", err)
+	}
+	kept, _ := inv.PlanByID(ctx, "plan-cas")
+	if kept.State != PlanBuilding || kept.Note != "the newer word" || kept.Revision != 2 {
+		t.Fatalf("the refused write reached the plan: %+v", kept)
+	}
+	// The writer saves its own plan again without reading it: its revision moved with its write.
+	first.Tier = 0
+	if err := inv.SavePlan(ctx, &first); err != nil {
+		t.Fatalf("a writer could not save its own plan twice: %v", err)
+	}
+	// A new plan under an id already written is refused, not laid over it.
+	again := Plan{ID: "plan-cas", Repository: "novox/app", Commit: "deadbeef", Created: time.Now(), State: PlanBuilding}
+	if err := inv.SavePlan(ctx, &again); !errors.Is(err, ErrPlanMoved) {
+		t.Fatalf("a second plan under one id was written: %v", err)
+	}
+	// And a process that does not hold the lease writes nothing.
+	inv.ActsUnder(func(context.Context) (uint64, error) { return 0, errors.New("this controller lost the lease") })
+	first.Note = "from a controller that lost the lease"
+	if err := inv.SavePlan(ctx, &first); err == nil {
+		t.Fatal("a plan was written by a controller that does not hold the lease")
+	}
+	if kept, _ := inv.PlanByID(ctx, "plan-cas"); kept.Note != "the newer word" || kept.Epoch != 57 {
+		t.Fatalf("a controller without the lease wrote the plan: %+v", kept)
+	}
+}
+
+// The epochs the mesh issued: the highest is the floor; taking one ends every earlier one nobody gave
+// back as found expired, and says which; one given back says so and is not said again.
+func TestTheEpochsIssuedAreKept(t *testing.T) {
+	inv := ForTest(t)
+	ctx := context.Background()
+	if highest, err := inv.HighestEpoch(ctx); err != nil || highest != 0 {
+		t.Fatalf("a mesh that issued none has %d (%v)", highest, err)
+	}
+	if _, err := inv.TookEpoch(ctx, Epoch{Epoch: 41, Instance: "a", Taken: time.Now()}); err != nil {
+		t.Fatal(err)
+	}
+	// a stopped renewing and said nothing; b takes the lease.
+	expired, err := inv.TookEpoch(ctx, Epoch{Epoch: 57, Instance: "b", Taken: time.Now()})
+	if err != nil || len(expired) != 1 || expired[0].Epoch != 41 || expired[0].How != EpochExpired {
+		t.Fatalf("taking 57 ended %+v (%v), want 41 found expired", expired, err)
+	}
+	if err := inv.EndEpoch(ctx, 57, EpochReleased); err != nil {
+		t.Fatal(err)
+	}
+	expired, err = inv.TookEpoch(ctx, Epoch{Epoch: 60, Instance: "c", Taken: time.Now()})
+	if err != nil || len(expired) != 0 {
+		t.Fatalf("a lease given back was found expired: %+v (%v)", expired, err)
+	}
+	if highest, _ := inv.HighestEpoch(ctx); highest != 60 {
+		t.Fatalf("the highest epoch issued is %d, want 60", highest)
+	}
+	e, found, err := inv.EpochOf(ctx, 57)
+	if err != nil || !found || e.Instance != "b" || e.How != EpochReleased || e.Ended == nil {
+		t.Fatalf("epoch 57 reads %+v (%v, %v)", e, found, err)
+	}
+	recent, err := inv.EpochsSince(ctx, time.Now().Add(-time.Hour))
+	if err != nil || len(recent) != 3 || recent[0].Epoch != 60 || recent[0].Ended != nil {
+		t.Fatalf("the epochs of the last hour read %+v (%v)", recent, err)
+	}
+}
+
+// What a machine was sent under, and whether it reads an epoch.
+func TestTheEpochAMachineWasSentIsKept(t *testing.T) {
+	inv := ForTest(t)
+	ctx := context.Background()
+	node, err := inv.AddNode(ctx, "anchor")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := inv.RecordSentUnder(ctx, node.ID, "d1", nil, 57); err != nil {
+		t.Fatal(err)
+	}
+	if e, err := inv.SentEpoch(ctx, node.ID); err != nil || e != 57 {
+		t.Fatalf("sent under %d (%v)", e, err)
+	}
+	if err := inv.RecordSent(ctx, node.ID, "d2", nil); err != nil {
+		t.Fatal(err)
+	}
+	if e, _ := inv.SentEpoch(ctx, node.ID); e != 0 {
+		t.Fatalf("a declaration sent without an epoch left %d kept", e)
+	}
+	if reads, _ := inv.ReadsEpoch(ctx, node.ID); reads {
+		t.Fatal("a machine that never said so reads an epoch")
+	}
+}

internal/inventory/plans.go

@@ -32,8 +32,17 @@ type Plan struct {
 	// never written from here: a save measures the tier it leaves and stamps the next. What the
 	// watchdog of a plan's progress (S3) reads.
 	TierEntered time.Time `json:"tier_entered,omitempty"`
+	// Revision is the plan's as it was read, and the one a save must find (novox/hq to-be 45 §6): a
+	// plan is written by compare-and-set, so a write against a plan another writer moved since is
+	// refused rather than laid over it. Zero is a plan never saved. SavePlan moves it.
+	Revision int64 `json:"revision"`
+	// Epoch is the controller lease epoch that wrote it last; zero for a write that claimed none.
+	Epoch uint64 `json:"epoch,omitempty"`
 }
 
+// ErrPlanMoved is a save against a plan written by somebody else since it was read.
+var ErrPlanMoved = errors.New("the plan was written by somebody else since it was read")
+
 // PlanModule is one module's state within a plan.
 type PlanModule struct {
 	// State: asked, built, failed; empty for a module whose tier has not been asked yet.
@@ -75,7 +84,16 @@ const (
 func (p Plan) Open() bool { return p.State == PlanBuilding || p.State == PlanRolling }
 
 // SavePlan writes a plan, new or changed, whole: the plan is small and read as one thing.
-func (i *Inventory) SavePlan(ctx context.Context, p Plan) error {
+//
+// **By compare-and-set on its revision, carrying the epoch** (novox/hq to-be 45 §6): written only if
+// the plan is still at the revision it was read at — a new one only if it does not exist — and refused
+// with ErrPlanMoved otherwise; and only by a process that may act (ActsUnder), whose epoch it records.
+// On success p's revision and epoch are the ones written, so the caller may save it again.
+func (i *Inventory) SavePlan(ctx context.Context, p *Plan) error {
+	epoch, err := i.actingEpoch(ctx)
+	if err != nil {
+		return fmt.Errorf("the plan for %s %s is not written: %w", p.Repository, p.Commit, err)
+	}
 	tiers, err := json.Marshal(p.Tiers)
 	if err != nil {
 		return err
@@ -92,21 +110,49 @@ func (i *Inventory) SavePlan(ctx context.Context, p Plan) error {
 		return err
 	}
 	defer func() { _ = tx.Rollback(ctx) }()
-	entered, err := planTierLeft(ctx, tx, p, time.Now())
+	entered, err := planTierLeft(ctx, tx, *p, time.Now())
 	if err != nil {
 		return err
 	}
-	_, err = tx.Exec(ctx,
-		`insert into release_plan (id, repository, commit_hash, created, updated, state, tier, tiers, modules, note, branch, tier_entered)
-		 values ($1, $2, $3, $4, now(), $5, $6, $7, $8, $9, $10, $11)
+	var revision int64
+	err = tx.QueryRow(ctx,
+		`insert into release_plan (id, repository, commit_hash, created, updated, state, tier, tiers, modules, note,
+		                           branch, tier_entered, revision, epoch)
+		 values ($1, $2, $3, $4, now(), $5, $6, $7, $8, $9, $10, $11, 1, $13)
 		 on conflict (id) do update set updated = now(), state = excluded.state, tier = excluded.tier,
 		        tiers = excluded.tiers, modules = excluded.modules, note = excluded.note, branch = excluded.branch,
-		        tier_entered = excluded.tier_entered`,
-		p.ID, p.Repository, p.Commit, p.Created, p.State, p.Tier, tiers, modules, p.Note, p.Branch, entered)
+		        tier_entered = excluded.tier_entered, revision = release_plan.revision + 1, epoch = excluded.epoch
+		  where release_plan.revision = $12
+		 returning revision`,
+		p.ID, p.Repository, p.Commit, p.Created, p.State, p.Tier, tiers, modules, p.Note, p.Branch, entered,
+		p.Revision, epoch).Scan(&revision)
+	if errors.Is(err, pgx.ErrNoRows) {
+		// The row is there and at another revision — moved since this was read, or there already
+		// when this one is new: either way not this writer's to overwrite. (A plan saved before plans
+		// had revisions is at zero, and its first save here is from a read at zero.)
+		return fmt.Errorf("the plan for %s %s (%s) is not written: %w", p.Repository, short(p.Commit), p.ID, ErrPlanMoved)
+	}
 	if err != nil {
 		return err
 	}
-	return tx.Commit(ctx)
+	if err := tx.Commit(ctx); err != nil {
+		return err
+	}
+	p.Revision, p.TierEntered = revision, entered
+	if epoch != nil {
+		p.Epoch = uint64(*epoch)
+	} else {
+		p.Epoch = 0
+	}
+	return nil
+}
+
+// short is a commit as a person reads it.
+func short(commit string) string {
+	if len(commit) > 8 {
+		return commit[:8]
+	}
+	return commit
 }
 
 // OpenPlans is every plan still being worked, oldest first.
@@ -134,7 +180,7 @@ func (i *Inventory) PlanByID(ctx context.Context, id string) (Plan, error) {
 func (i *Inventory) plans(ctx context.Context, tail string) ([]Plan, error) {
 	rows, err := i.store.Pool().Query(ctx,
 		`select id, repository, commit_hash, created, updated, state, tier, tiers, modules, note, branch,
-		        coalesce(tier_entered, created)
+		        coalesce(tier_entered, created), revision, coalesce(epoch, 0)
 		   from release_plan `+tail)
 	if err != nil {
 		return nil, err
@@ -144,10 +190,12 @@ func (i *Inventory) plans(ctx context.Context, tail string) ([]Plan, error) {
 	for rows.Next() {
 		var p Plan
 		var tiers, modules []byte
+		var epoch int64
 		if err := rows.Scan(&p.ID, &p.Repository, &p.Commit, &p.Created, &p.Updated, &p.State,
-			&p.Tier, &tiers, &modules, &p.Note, &p.Branch, &p.TierEntered); err != nil {
+			&p.Tier, &tiers, &modules, &p.Note, &p.Branch, &p.TierEntered, &p.Revision, &epoch); err != nil {
 			return nil, err
 		}
+		p.Epoch = uint64(epoch)
 		if err := json.Unmarshal(tiers, &p.Tiers); err != nil {
 			return nil, err
 		}

internal/inventory/plans_test.go

@@ -13,7 +13,7 @@ func TestAPlanIsKeptAdvancedAndResumedFromTheStore(t *testing.T) {
 	p := Plan{ID: "plan-1", Repository: "novox/mesh-tools", Commit: "abc", Created: time.Now().UTC(),
 		State: PlanBuilding, Tiers: [][]string{{"mesh-tools"}, {"builder"}, {"shop"}},
 		Modules: map[string]*PlanModule{"mesh-tools": {}, "builder": {}, "shop": {}}}
-	if err := inv.SavePlan(ctx, p); err != nil {
+	if err := inv.SavePlan(ctx, &p); err != nil {
 		t.Fatal(err)
 	}
 	open, err := inv.OpenPlans(ctx)
@@ -28,7 +28,7 @@ func TestAPlanIsKeptAdvancedAndResumedFromTheStore(t *testing.T) {
 	resumed.Modules["mesh-tools"].BuiltAt = &now
 	resumed.State = PlanRolling
 	resumed.Note = "tier 0 built; waiting for builder on anchor to be applied"
-	if err := inv.SavePlan(ctx, resumed); err != nil {
+	if err := inv.SavePlan(ctx, &resumed); err != nil {
 		t.Fatal(err)
 	}
 	again, err := inv.PlanByID(ctx, "plan-1")
@@ -36,7 +36,7 @@ func TestAPlanIsKeptAdvancedAndResumedFromTheStore(t *testing.T) {
 		t.Fatalf("the advanced plan did not come back as left: %v %+v", err, again)
 	}
 	again.State = PlanDone
-	if err := inv.SavePlan(ctx, again); err != nil {
+	if err := inv.SavePlan(ctx, &again); err != nil {
 		t.Fatal(err)
 	}
 	if open, _ = inv.OpenPlans(ctx); len(open) != 0 {
@@ -54,7 +54,7 @@ func TestASupersededPlanIsNotOpen(t *testing.T) {
 	p := Plan{ID: "plan-1", Repository: "novox/mesh-catalog", Branch: "main", Commit: "abc",
 		Created: time.Now().UTC(), State: PlanBuilding, Tiers: [][]string{{"gitea"}},
 		Modules: map[string]*PlanModule{"gitea": {}}}
-	if err := inv.SavePlan(ctx, p); err != nil {
+	if err := inv.SavePlan(ctx, &p); err != nil {
 		t.Fatal(err)
 	}
 	kept, err := inv.PlanByID(ctx, "plan-1")
@@ -63,7 +63,7 @@ func TestASupersededPlanIsNotOpen(t *testing.T) {
 	}
 	kept.State = PlanSuperseded
 	kept.Note = "superseded at tier 0 by plan-2"
-	if err := inv.SavePlan(ctx, kept); err != nil {
+	if err := inv.SavePlan(ctx, &kept); err != nil {
 		t.Fatal(err)
 	}
 	if open, err := inv.OpenPlans(ctx); err != nil || len(open) != 0 {

internal/lease/lease.go

@@ -0,0 +1,471 @@
+// Package lease is the controller's lease (novox/hq to-be 45 §6, ADR 0227 rule 1): the one key that
+// says which controller instance may act, and the epoch every act of it carries.
+//
+// **A controller instance acts only while it holds the key `holder`** in its lease bucket — sends a
+// declaration, writes a plan, a condition or a call. The key is written by compare-and-set, lives a
+// bucket's age (fifteen seconds) unless renewed, and is renewed every five. The **epoch** is the
+// bucket's revision at which the instance took it: a later holder's is higher, so a machine that has
+// heard from epoch 57 can refuse anything still arriving from 41 (issue 204).
+//
+//	controller A (epoch 41)  ──renew──renew──╳ (renewal refused)──► stops acting, exits
+//	controller B             ──wait──────────────take (epoch 57)──► acts
+//
+// **Three ways an instance stops acting, all at once.** A renewal refused — somebody else wrote the key
+// — is a loss, said and final: Lost closes and the process exits, so its service manager restarts it as
+// a candidate. A renewal that cannot be made in time is the same, because past its time the key may be
+// somebody else's. And Epoch, the gate every act passes, answers an error from the moment the last
+// renewal is older than the key's age less a margin — **before** another instance could have taken
+// it, whatever the renewing goroutine is doing — so a send in flight at the moment of loss is refused
+// by the clock, not by a goroutine that may be stuck.
+package lease
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"sync"
+	"time"
+
+	"github.com/nats-io/nats.go/jetstream"
+)
+
+// Key is the one key of the lease bucket.
+const Key = "holder"
+
+// Holder is who holds the lease, as the key says it.
+type Holder struct {
+	// Instance names one controller process: its machine, its process and when it started. Two
+	// instances on one machine — the handover of issue 213 — are two names.
+	Instance string `json:"instance"`
+	Host     string `json:"host,omitempty"`
+	Build    string `json:"build,omitempty"`
+	// Epoch is the revision the key was taken at. Zero in the value written to take it, because the
+	// revision is known only once written; every renewal says it, and Current reads the revision of a
+	// key never renewed.
+	Epoch   uint64    `json:"epoch,omitempty"`
+	Taken   time.Time `json:"taken"`
+	Renewed time.Time `json:"renewed"`
+}
+
+// Defaults, as to-be 45 §6 sets them. The age is the bucket's, read from it (Open), so the bucket
+// and the lease cannot disagree about it.
+const (
+	RenewEvery = 5 * time.Second
+	// Margin is how long before the key could expire this instance stops acting on it: the gap
+	// between its own clock and the bus's, and a send already on its way.
+	Margin = 3 * time.Second
+	// Poll is how often a candidate looks again at a key somebody else holds.
+	Poll = time.Second
+)
+
+// ErrNotHeld is an act asked of an instance that does not hold the lease.
+var ErrNotHeld = errors.New("this controller does not hold the lease")
+
+// ErrUnwritable is a lease nobody holds that the bus would not let this instance write: the one case a
+// caller may serve without it, since nothing else holds it either. Every other failure to take it —
+// the holder unreadable, the floor unreadable — says nothing about whether another instance acts.
+var ErrUnwritable = errors.New("nobody holds the lease, and the bus would not let this controller write it")
+
+// Options are what a Lease is made with.
+type Options struct {
+	Holder Holder
+	// RenewEvery, Margin and Poll default to the package's.
+	RenewEvery, Margin, Poll time.Duration
+	// Floor is the highest epoch this mesh has issued, from the controller's own store. **An epoch is
+	// never issued twice**: a lease bucket raised again from nothing — a bus whose data was replaced —
+	// starts its revisions over, and every machine that heard epoch 57 would refuse the next controller
+	// for ever. Below the floor, the bucket's revisions are moved past it before the key is taken.
+	// Nil is no floor.
+	Floor func(ctx context.Context) (uint64, error)
+	// Say is where what the lease does is said: waiting, taking, losing.
+	Say func(format string, args ...any)
+	// Moved is told when the bucket's revisions were moved past the floor: the bucket was raised again
+	// from nothing, which the caller says as a condition. Nil tells nobody.
+	Moved func(was, floor uint64)
+	// Now is the clock; nil is time.Now.
+	Now func() time.Time
+}
+
+// Lease is one instance's hold, or its wait for one.
+type Lease struct {
+	kv     jetstream.KeyValue
+	stream jetstream.Stream
+	ttl    time.Duration
+	o      Options
+
+	mu sync.Mutex
+	// held is whether this instance holds the key; epoch the revision it took it at; revision the
+	// revision of its last write, which the next renewal must find.
+	held     bool
+	epoch    uint64
+	revision uint64
+	// validUntil is when this instance stops acting unless it renews: the moment before its last
+	// renewal was sent, plus the key's age, less the margin.
+	validUntil time.Time
+	// renewed is when this instance last took or renewed the key.
+	renewed time.Time
+	lostWhy error
+	lost    chan struct{}
+	taken   time.Time
+}
+
+// Open is a lease over the bucket, read for its age. The bucket is the caller's to assert.
+func Open(ctx context.Context, js jetstream.JetStream, bucket string, o Options) (*Lease, error) {
+	kv, err := js.KeyValue(ctx, bucket)
+	if err != nil {
+		return nil, fmt.Errorf("the lease bucket %s is not on the bus: %w", bucket, err)
+	}
+	status, err := kv.Status(ctx)
+	if err != nil {
+		return nil, fmt.Errorf("the lease bucket %s cannot be read: %w", bucket, err)
+	}
+	if status.TTL() <= 0 {
+		// A key that never expires is a lease a dead controller holds for ever.
+		return nil, fmt.Errorf("the lease bucket %s keeps its keys for ever, so a controller that died "+
+			"holding it would hold it for ever: it must have an age", bucket)
+	}
+	stream, err := js.Stream(ctx, "KV_"+bucket)
+	if err != nil {
+		return nil, fmt.Errorf("the stream under the lease bucket %s cannot be read: %w", bucket, err)
+	}
+	if o.RenewEvery <= 0 {
+		o.RenewEvery = RenewEvery
+	}
+	if o.Margin <= 0 {
+		o.Margin = Margin
+	}
+	if o.Poll <= 0 {
+		o.Poll = Poll
+	}
+	if o.Say == nil {
+		o.Say = func(string, ...any) {}
+	}
+	if o.Now == nil {
+		o.Now = time.Now
+	}
+	if o.RenewEvery+o.Margin >= status.TTL() {
+		return nil, fmt.Errorf("a lease renewed every %s with a margin of %s does not fit a key that lives %s",
+			o.RenewEvery, o.Margin, status.TTL())
+	}
+	return &Lease{kv: kv, stream: stream, ttl: status.TTL(), o: o, lost: make(chan struct{})}, nil
+}
+
+// TTL is how long the key lives unrenewed.
+func (l *Lease) TTL() time.Duration { return l.ttl }
+
+// Current is who holds the lease now, and false when nobody does. An error is an error: never "nobody".
+func Current(ctx context.Context, kv jetstream.KeyValue) (Holder, bool, error) {
+	entry, err := kv.Get(ctx, Key)
+	if errors.Is(err, jetstream.ErrKeyNotFound) {
+		return Holder{}, false, nil
+	}
+	if err != nil {
+		return Holder{}, false, err
+	}
+	var h Holder
+	if err := json.Unmarshal(entry.Value(), &h); err != nil {
+		return Holder{}, false, fmt.Errorf("the lease's key holds something that is not a holder: %w", err)
+	}
+	if h.Epoch == 0 {
+		// Taken and not yet renewed: the revision it was taken at is the one it has.
+		h.Epoch = entry.Revision()
+	}
+	return h, true, nil
+}
+
+// Current is who holds this lease now.
+func (l *Lease) Current(ctx context.Context) (Holder, bool, error) { return Current(ctx, l.kv) }
+
+// ErrTaken is a take that found the key held by somebody else; Take waits on it, TryTake answers it.
+var ErrTaken = errors.New("another controller holds the lease")
+
+// Take waits until the key is absent or expired and takes it, answering the epoch. A key somebody
+// else holds is waited on and said once; an error reading or writing the bucket is answered, because
+// a candidate that cannot see the key cannot tell whether it may act.
+func (l *Lease) Take(ctx context.Context) (uint64, error) {
+	said := ""
+	for {
+		epoch, err := l.TryTake(ctx)
+		if err == nil {
+			return epoch, nil
+		}
+		var held *heldBy
+		if !errors.As(err, &held) {
+			return 0, err
+		}
+		if held.h.Instance != said {
+			l.o.Say("another controller holds the lease (%s, epoch %d, taken %s); waiting until it lets go "+
+				"or stops renewing", held.h.Instance, held.h.Epoch, held.h.Taken.UTC().Format(time.RFC3339))
+			said = held.h.Instance
+		}
+		select {
+		case <-ctx.Done():
+			return 0, ctx.Err()
+		case <-time.After(l.o.Poll):
+		}
+	}
+}
+
+// heldBy is ErrTaken naming the holder.
+type heldBy struct{ h Holder }
+
+func (e *heldBy) Error() string {
+	return fmt.Sprintf("%v: %s, epoch %d", ErrTaken, e.h.Instance, e.h.Epoch)
+}
+func (e *heldBy) Unwrap() error { return ErrTaken }
+
+// TryTake takes the key if nobody holds it, once, without waiting: the epoch, or ErrTaken naming the
+// holder, or what the bus said.
+func (l *Lease) TryTake(ctx context.Context) (uint64, error) {
+	l.mu.Lock()
+	switch {
+	case l.lostWhy != nil:
+		// A lease lost is not taken again by the instance that lost it: that instance exits, and its
+		// successor is a new candidate with a new name.
+		defer l.mu.Unlock()
+		return 0, fmt.Errorf("%w: it lost it — %v", ErrNotHeld, l.lostWhy)
+	case l.held:
+		defer l.mu.Unlock()
+		return l.epoch, nil
+	}
+	l.mu.Unlock()
+	if h, found, err := l.Current(ctx); err != nil {
+		return 0, fmt.Errorf("who holds the lease cannot be read: %w", err)
+	} else if found {
+		return 0, &heldBy{h}
+	}
+	if err := l.aboveTheFloor(ctx); err != nil {
+		return 0, err
+	}
+	now := l.o.Now()
+	h := l.o.Holder
+	h.Taken, h.Renewed, h.Epoch = now, now, 0
+	value, err := json.Marshal(h)
+	if err != nil {
+		return 0, err
+	}
+	anchor := l.o.Now()
+	revision, err := l.kv.Create(ctx, Key, value)
+	if errors.Is(err, jetstream.ErrKeyExists) {
+		// Somebody took it between the read and the write: theirs.
+		if h, found, rerr := l.Current(ctx); rerr == nil && found {
+			return 0, &heldBy{h}
+		}
+		return 0, ErrTaken
+	}
+	if err != nil {
+		return 0, fmt.Errorf("%w: %w", ErrUnwritable, err)
+	}
+	if floor, err := l.floor(ctx); err != nil {
+		_ = l.kv.Delete(ctx, Key, jetstream.LastRevision(revision))
+		return 0, err
+	} else if revision <= floor {
+		// The bucket moved under the floor between the check and the write (another candidate raising
+		// it again): never an epoch already issued. Given back, and taken again on the next try.
+		_ = l.kv.Delete(ctx, Key, jetstream.LastRevision(revision))
+		return 0, fmt.Errorf("the lease was taken at revision %d, not above the highest epoch issued (%d); given "+
+			"back to be taken again", revision, floor)
+	}
+	l.mu.Lock()
+	l.held, l.epoch, l.revision, l.taken, l.renewed = true, revision, revision, now, anchor
+	l.validUntil = anchor.Add(l.ttl - l.o.Margin)
+	l.mu.Unlock()
+	l.o.Say("took the controller lease at epoch %d (%s)", revision, h.Instance)
+	return revision, nil
+}
+
+// floor is the highest epoch issued, zero without a floor.
+func (l *Lease) floor(ctx context.Context) (uint64, error) {
+	if l.o.Floor == nil {
+		return 0, nil
+	}
+	floor, err := l.o.Floor(ctx)
+	if err != nil {
+		return 0, fmt.Errorf("the highest epoch this mesh has issued cannot be read, so no epoch can be "+
+			"issued that is surely higher: %w", err)
+	}
+	return floor, nil
+}
+
+// aboveTheFloor moves the bucket's revisions past the highest epoch issued, when a bucket raised again
+// from nothing is behind it. Only while nobody holds the key: the stream is compacted to the floor, which
+// sets the next revision above it.
+func (l *Lease) aboveTheFloor(ctx context.Context) error {
+	floor, err := l.floor(ctx)
+	if err != nil || floor == 0 {
+		return err
+	}
+	info, err := l.stream.Info(ctx)
+	if err != nil {
+		return fmt.Errorf("the lease bucket's revisions cannot be read: %w", err)
+	}
+	if info.State.LastSeq > floor {
+		return nil
+	}
+	l.o.Say("the lease bucket is at revision %d and this mesh has issued epoch %d: the bucket was raised "+
+		"again from nothing, so its revisions are moved past %d before the lease is taken", info.State.LastSeq,
+		floor, floor)
+	if err := l.stream.Purge(ctx, jetstream.WithPurgeSequence(floor+1)); err != nil {
+		return fmt.Errorf("the lease bucket's revisions could not be moved past epoch %d: %w", floor, err)
+	}
+	if l.o.Moved != nil {
+		l.o.Moved(info.State.LastSeq, floor)
+	}
+	return nil
+}
+
+// Epoch is the gate every act passes: this instance's epoch while it holds the lease and its last
+// renewal is in time, and an error otherwise — not held, lost, or a renewal overdue.
+func (l *Lease) Epoch() (uint64, error) {
+	l.mu.Lock()
+	defer l.mu.Unlock()
+	switch {
+	case l.lostWhy != nil:
+		return 0, fmt.Errorf("%w: it lost it — %v", ErrNotHeld, l.lostWhy)
+	case !l.held:
+		return 0, ErrNotHeld
+	case l.o.Now().After(l.validUntil):
+		return 0, fmt.Errorf("%w in time: its last renewal was due by %s", ErrNotHeld,
+			l.validUntil.UTC().Format(time.RFC3339))
+	}
+	return l.epoch, nil
+}
+
+// Renewed is when this instance last took or renewed the key; zero before it took it.
+func (l *Lease) Renewed() time.Time {
+	l.mu.Lock()
+	defer l.mu.Unlock()
+	return l.renewed
+}
+
+// Held says this instance holds the lease now, as Epoch's gate does.
+func (l *Lease) Held() bool {
+	_, err := l.Epoch()
+	return err == nil
+}
+
+// Lost is closed when this instance loses the lease it held. A caller exits on it.
+func (l *Lease) Lost() <-chan struct{} { return l.lost }
+
+// LostWhy is why it was lost, nil while it was not.
+func (l *Lease) LostWhy() error {
+	l.mu.Lock()
+	defer l.mu.Unlock()
+	return l.lostWhy
+}
+
+// Keep renews the lease until ctx ends or a renewal fails; on a failure it is lost.
+func (l *Lease) Keep(ctx context.Context) {
+	tick := time.NewTicker(l.o.RenewEvery)
+	defer tick.Stop()
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case <-tick.C:
+		}
+		if err := l.Renew(ctx); err != nil {
+			return // lost, said by lose; or stopping
+		}
+	}
+}
+
+// Renew writes the key once more at the revision this instance last wrote. A refusal or a failure is
+// the lease lost: past this renewal's time the key may be somebody else's, and an instance that is not
+// sure it holds the lease does not act on it.
+func (l *Lease) Renew(ctx context.Context) error {
+	l.mu.Lock()
+	if !l.held || l.lostWhy != nil {
+		defer l.mu.Unlock()
+		if l.lostWhy != nil {
+			return l.lostWhy
+		}
+		return ErrNotHeld
+	}
+	revision, epoch, taken := l.revision, l.epoch, l.taken
+	l.mu.Unlock()
+
+	h := l.o.Holder
+	h.Epoch, h.Taken = epoch, taken
+	anchor := l.o.Now()
+	h.Renewed = anchor
+	value, err := json.Marshal(h)
+	if err != nil {
+		return l.lose(err)
+	}
+	renewing, cancel := context.WithTimeout(ctx, l.o.RenewEvery)
+	defer cancel()
+	next, err := l.kv.Update(renewing, Key, value, revision)
+	if err != nil {
+		if ctx.Err() != nil {
+			// Stopping, not losing: Release gives it back.
+			return ctx.Err()
+		}
+		why := fmt.Errorf("the renewal at revision %d was refused: %w", revision, err)
+		if !errors.Is(err, jetstream.ErrKeyExists) && !isWrongLast(err) {
+			why = fmt.Errorf("the renewal could not be made: %w", err)
+		}
+		return l.lose(why)
+	}
+	l.mu.Lock()
+	l.revision, l.renewed = next, anchor
+	l.validUntil = anchor.Add(l.ttl - l.o.Margin)
+	l.mu.Unlock()
+	return nil
+}
+
+// isWrongLast is the bus refusing a compare-and-set because the key moved.
+func isWrongLast(err error) bool {
+	var apiErr *jetstream.APIError
+	return errors.As(err, &apiErr) && apiErr.ErrorCode == jetstream.JSErrCodeStreamWrongLastSequence
+}
+
+// lose marks the lease lost, once, and says why.
+func (l *Lease) lose(why error) error {
+	l.mu.Lock()
+	defer l.mu.Unlock()
+	if l.lostWhy == nil {
+		l.lostWhy = why
+		l.held = false
+		close(l.lost)
+		l.o.Say("LOST the controller lease (epoch %d): %v — this controller stops acting at once", l.epoch, why)
+	}
+	return l.lostWhy
+}
+
+// Release gives the lease back, so the next candidate takes it at once rather than after its age.
+// Only the key this instance last wrote is deleted: one that moved is somebody else's.
+func (l *Lease) Release(ctx context.Context) {
+	l.mu.Lock()
+	held, revision, epoch := l.held && l.lostWhy == nil, l.revision, l.epoch
+	l.held = false
+	if l.lostWhy == nil {
+		l.lostWhy = errors.New("given back")
+		close(l.lost)
+	}
+	l.mu.Unlock()
+	if !held {
+		return
+	}
+	releasing, cancel := context.WithTimeout(context.WithoutCancel(ctx), 5*time.Second)
+	defer cancel()
+	err := l.kv.Delete(releasing, Key, jetstream.LastRevision(revision))
+	if err != nil && isWrongLast(err) {
+		// A renewal was in flight as this began and landed first: the key moved, and is still this
+		// instance's if it still names it at this epoch. Deleted at the revision it has now.
+		if entry, gerr := l.kv.Get(releasing, Key); gerr == nil {
+			var h Holder
+			if json.Unmarshal(entry.Value(), &h) == nil && h.Instance == l.o.Holder.Instance &&
+				(h.Epoch == epoch || h.Epoch == 0 && entry.Revision() == epoch) {
+				err = l.kv.Delete(releasing, Key, jetstream.LastRevision(entry.Revision()))
+			}
+		}
+	}
+	if err != nil {
+		l.o.Say("the controller lease (epoch %d) could not be given back, so the next controller waits for "+
+			"it to expire: %v", epoch, err)
+		return
+	}
+	l.o.Say("gave the controller lease back (epoch %d)", epoch)
+}

internal/lease/lease_test.go

@@ -0,0 +1,296 @@
+package lease
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"os"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/nats-io/nats.go"
+	"github.com/nats-io/nats.go/jetstream"
+)
+
+// Two controllers at once, on a real bus (novox/hq to-be 45 §6, replay R1's half that lives here):
+// one takes the lease and the other waits; a handover gives the second a higher epoch; a holder that
+// stops renewing is taken over once its key expires, and has stopped acting before then; a renewal
+// refused is a loss at once; and a bucket raised again from nothing never issues an epoch twice.
+//
+//	MESH_TEST_NATS=nats://127.0.0.1:14222 go test ./internal/lease/
+
+// The bounds in these tests are the design's divided by five, so the cases run in seconds.
+const (
+	testTTL   = 3 * time.Second
+	testRenew = time.Second
+	testPoll  = 100 * time.Millisecond
+)
+
+// aBucket is a fresh lease bucket on the test bus, with the test's age.
+func aBucket(t *testing.T) (jetstream.JetStream, string) {
+	t.Helper()
+	url := os.Getenv("MESH_TEST_NATS")
+	if url == "" {
+		t.Skip("MESH_TEST_NATS unset")
+	}
+	conn, err := nats.Connect(url)
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Cleanup(conn.Close)
+	js, err := jetstream.New(conn)
+	if err != nil {
+		t.Fatal(err)
+	}
+	bucket := fmt.Sprintf("lease-test-%d", time.Now().UnixNano())
+	if _, err := js.CreateKeyValue(t.Context(), jetstream.KeyValueConfig{Bucket: bucket, History: 1,
+		TTL: testTTL, Storage: jetstream.FileStorage}); err != nil {
+		t.Fatal(err)
+	}
+	t.Cleanup(func() { _ = js.DeleteKeyValue(context.Background(), bucket) })
+	return js, bucket
+}
+
+// aController is one controller instance's lease over the bucket, on a connection of its own.
+func aController(t *testing.T, bucket, name string, floor func(context.Context) (uint64, error)) *Lease {
+	t.Helper()
+	conn, err := nats.Connect(os.Getenv("MESH_TEST_NATS"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Cleanup(conn.Close)
+	js, err := jetstream.New(conn)
+	if err != nil {
+		t.Fatal(err)
+	}
+	var said sync.Mutex
+	l, err := Open(t.Context(), js, bucket, Options{Holder: Holder{Instance: name, Host: name},
+		RenewEvery: testRenew, Margin: testRenew / 2, Poll: testPoll, Floor: floor,
+		Say: func(format string, args ...any) {
+			said.Lock()
+			defer said.Unlock()
+			t.Logf(name+": "+format, args...)
+		}})
+	if err != nil {
+		t.Fatal(err)
+	}
+	return l
+}
+
+// takeWithin takes the lease in a goroutine and answers when it did, or fails past the bound.
+func takeWithin(t *testing.T, l *Lease) <-chan uint64 {
+	t.Helper()
+	took := make(chan uint64, 1)
+	go func() {
+		epoch, err := l.Take(t.Context())
+		if err != nil {
+			t.Errorf("taking: %v", err)
+			close(took)
+			return
+		}
+		took <- epoch
+	}()
+	return took
+}
+
+func TestTheSecondControllerWaitsAndTakesAHigherEpochOnHandover(t *testing.T) {
+	_, bucket := aBucket(t)
+	a, b := aController(t, bucket, "a", nil), aController(t, bucket, "b", nil)
+
+	epochA, err := a.Take(t.Context())
+	if err != nil {
+		t.Fatal(err)
+	}
+	keeping, stop := context.WithCancel(t.Context())
+	defer stop()
+	go a.Keep(keeping)
+
+	// b waits while a renews — across more than one age of the key, so it is the renewals holding it.
+	took := takeWithin(t, b)
+	select {
+	case epoch := <-took:
+		t.Fatalf("b took the lease (epoch %d) while a held it and renewed", epoch)
+	case <-time.After(testTTL + testRenew):
+	}
+	if _, err := b.Epoch(); !errors.Is(err, ErrNotHeld) {
+		t.Fatalf("b, waiting, may act: %v", err)
+	}
+	if e, err := a.Epoch(); err != nil || e != epochA {
+		t.Fatalf("a, holding, answers %d, %v", e, err)
+	}
+	holder, found, err := b.Current(t.Context())
+	if err != nil || !found || holder.Instance != "a" || holder.Epoch != epochA {
+		t.Fatalf("b reads the holder as %+v (%v, %v), want a at epoch %d", holder, found, err, epochA)
+	}
+
+	// a gives it back: b takes it at once, not after the key's age, at a higher epoch.
+	stop()
+	handedOver := time.Now()
+	a.Release(t.Context())
+	select {
+	case epochB := <-took:
+		if epochB <= epochA {
+			t.Fatalf("b took epoch %d after a's %d: an epoch must only grow", epochB, epochA)
+		}
+		if waited := time.Since(handedOver); waited > testTTL {
+			t.Fatalf("b took the lease %s after a gave it back: it waited out the key's age", waited)
+		}
+	case <-time.After(2 * testTTL):
+		t.Fatal("b never took the lease a gave back")
+	}
+	if _, err := a.Epoch(); !errors.Is(err, ErrNotHeld) {
+		t.Fatalf("a, having given it back, may still act: %v", err)
+	}
+	if _, err := a.TryTake(t.Context()); err == nil {
+		t.Fatal("a took the lease again after giving it back: its successor is a new candidate")
+	}
+}
+
+func TestAHolderThatStopsRenewingStopsActingBeforeItIsTakenOver(t *testing.T) {
+	_, bucket := aBucket(t)
+	a, b := aController(t, bucket, "a", nil), aController(t, bucket, "b", nil)
+	epochA, err := a.Take(t.Context())
+	if err != nil {
+		t.Fatal(err)
+	}
+	// a is stuck: it never renews (a process stopped, a goroutine wedged). It was not told anything.
+	took := takeWithin(t, b)
+
+	// Its gate closes by its own clock, before the key can expire.
+	closed := time.Now()
+	for a.Held() {
+		if time.Since(closed) > testTTL {
+			t.Fatal("a still may act past its key's age without renewing")
+		}
+		time.Sleep(20 * time.Millisecond)
+	}
+	select {
+	case epoch := <-took:
+		t.Fatalf("b took the lease (epoch %d) before a stopped acting", epoch)
+	default:
+	}
+	select {
+	case epochB := <-took:
+		if epochB <= epochA {
+			t.Fatalf("b took epoch %d after a's %d", epochB, epochA)
+		}
+		// And a, the moment it tries to renew, knows it lost: the key moved.
+		if err := a.Renew(t.Context()); err == nil {
+			t.Fatal("a renewed a lease b holds")
+		}
+		select {
+		case <-a.Lost():
+		default:
+			t.Fatal("a's renewal was refused and a was not told it lost the lease")
+		}
+		if _, err := a.Epoch(); !errors.Is(err, ErrNotHeld) {
+			t.Fatalf("a, having lost the lease, may act: %v", err)
+		}
+	case <-time.After(3 * testTTL):
+		t.Fatal("b never took over a lease nobody renewed")
+	}
+}
+
+func TestARenewalRefusedIsALossAtOnce(t *testing.T) {
+	js, bucket := aBucket(t)
+	a := aController(t, bucket, "a", nil)
+	if _, err := a.Take(t.Context()); err != nil {
+		t.Fatal(err)
+	}
+	// Somebody else writes the key — a second controller that read it as expired on a skewed clock,
+	// or a person — so a's next renewal finds a revision it did not write.
+	kv, err := js.KeyValue(t.Context(), bucket)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if _, err := kv.Put(t.Context(), Key, []byte(`{"instance":"intruder"}`)); err != nil {
+		t.Fatal(err)
+	}
+	if err := a.Renew(t.Context()); err == nil {
+		t.Fatal("a renewed over a key somebody else wrote")
+	}
+	select {
+	case <-a.Lost():
+	case <-time.After(time.Second):
+		t.Fatal("a was not told it lost the lease")
+	}
+	if _, err := a.Epoch(); !errors.Is(err, ErrNotHeld) {
+		t.Fatalf("a acts after its renewal was refused: %v", err)
+	}
+	// And giving back a lease it lost deletes nothing of the one who holds it now.
+	a.Release(t.Context())
+	if h, found, err := Current(t.Context(), kv); err != nil || !found || h.Instance != "intruder" {
+		t.Fatalf("after a gave back what it lost, the key holds %+v (%v, %v)", h, found, err)
+	}
+}
+
+func TestKeepingRenewsAcrossManyAges(t *testing.T) {
+	_, bucket := aBucket(t)
+	a := aController(t, bucket, "a", nil)
+	epoch, err := a.Take(t.Context())
+	if err != nil {
+		t.Fatal(err)
+	}
+	keeping, stop := context.WithCancel(t.Context())
+	defer stop()
+	go a.Keep(keeping)
+	until := time.Now().Add(3 * testTTL)
+	for time.Now().Before(until) {
+		if e, err := a.Epoch(); err != nil || e != epoch {
+			t.Fatalf("a, renewing, answers %d, %v", e, err)
+		}
+		time.Sleep(200 * time.Millisecond)
+	}
+	h, found, err := a.Current(t.Context())
+	if err != nil || !found || h.Epoch != epoch || h.Instance != "a" {
+		t.Fatalf("the key says %+v (%v, %v)", h, found, err)
+	}
+}
+
+func TestAnEpochIsNeverIssuedTwiceWhenTheBucketStartsOver(t *testing.T) {
+	_, bucket := aBucket(t)
+	// The mesh has issued epoch 500 before: its store says so. The bucket is new — a bus whose data
+	// was replaced — and its revisions start at one.
+	floor := func(context.Context) (uint64, error) { return 500, nil }
+	a := aController(t, bucket, "a", floor)
+	moved := false
+	a.o.Moved = func(was, floor uint64) { moved = was < floor && floor == 500 }
+	epoch, err := a.Take(t.Context())
+	if err != nil {
+		t.Fatal(err)
+	}
+	if epoch <= 500 {
+		t.Fatalf("a took epoch %d with 500 already issued: every machine that heard 500 would refuse it", epoch)
+	}
+	if !moved {
+		t.Fatal("the bucket's revisions were moved past the floor and nobody was told")
+	}
+	// A floor that cannot be read takes nothing: an epoch that may not be higher is not issued.
+	_, other := aBucket(t)
+	b := aController(t, other, "b", func(context.Context) (uint64, error) { return 0, errors.New("store away") })
+	if _, err := b.TryTake(t.Context()); err == nil {
+		t.Fatal("b took a lease without knowing the highest epoch issued")
+	}
+}
+
+func TestALeaseBucketWithoutAnAgeIsRefused(t *testing.T) {
+	url := os.Getenv("MESH_TEST_NATS")
+	if url == "" {
+		t.Skip("MESH_TEST_NATS unset")
+	}
+	conn, err := nats.Connect(url)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer conn.Close()
+	js, _ := jetstream.New(conn)
+	bucket := fmt.Sprintf("lease-ageless-%d", time.Now().UnixNano())
+	if _, err := js.CreateKeyValue(t.Context(), jetstream.KeyValueConfig{Bucket: bucket, History: 1}); err != nil {
+		t.Fatal(err)
+	}
+	defer func() { _ = js.DeleteKeyValue(context.Background(), bucket) }()
+	if _, err := Open(t.Context(), js, bucket, Options{Holder: Holder{Instance: "a"}}); err == nil {
+		t.Fatal("a lease over keys that never expire was opened: a controller that died holding it would hold it for ever")
+	}
+}

internal/link/calls.go

@@ -78,6 +78,8 @@ type Call struct {
 	// Holder is the controller process that served it, so one starting can tell its own running
 	// calls from those a stopped one left.
 	Holder string `json:"holder,omitempty"`
+	// Epoch is the controller lease epoch it was served under (novox/hq to-be 45 §6); zero for none.
+	Epoch uint64 `json:"epoch,omitempty"`
 
 	reply string // the subject the answer went to, which the bus names when it refuses it
 }
@@ -111,6 +113,18 @@ type CallLog struct {
 	logger  *log.Logger
 	lost    int // writes the keeper could not take, said once each
 	keepErr error
+	// epoch is the lease this process serves under (UnderLease): a call carries its epoch, and its
+	// record is written only while the lease is held. Nil writes every record with no epoch.
+	epoch func() (uint64, error)
+}
+
+// UnderLease makes every call carry the controller lease's epoch, and every write of a call's record
+// pass the lease (novox/hq to-be 45 §6): a controller that lost the lease writes no finish over a call
+// the next holder has marked abandoned — that mark is the record's last word.
+func (l *CallLog) UnderLease(epoch func() (uint64, error)) {
+	l.mu.Lock()
+	defer l.mu.Unlock()
+	l.epoch = epoch
 }
 
 // Calls is this process's log: one holder process serves its seats on one connection.
@@ -150,9 +164,9 @@ func (l *CallLog) Durably(ctx context.Context, keeper CallKeeper, holder string,
 			return fmt.Errorf("marking %s abandoned: %w", c.ID, err)
 		}
 		if logger != nil {
-			logger.Printf("%s (%s.%s, asked %s by %s) was running under %s, which stopped: marked abandoned — "+
-				"it may have done part of what it was asked, and nothing will finish it", c.ID, c.Seat, c.Verb,
-				c.Started.Format(time.RFC3339), orSomebody(c.Caller), orSomebody(c.Holder))
+			logger.Printf("%s (%s.%s, asked %s by %s) was running under %s (epoch %d), which stopped: marked "+
+				"abandoned — it may have done part of what it was asked, and nothing will finish it", c.ID, c.Seat,
+				c.Verb, c.Started.Format(time.RFC3339), orSomebody(c.Caller), orSomebody(c.Holder), c.Epoch)
 		}
 	}
 	return nil
@@ -184,6 +198,18 @@ func (l *CallLog) keep(c Call) {
 
 func (l *CallLog) keepWrites() {
 	for c := range l.writes {
+		l.mu.Lock()
+		gate := l.epoch
+		l.mu.Unlock()
+		if gate != nil {
+			if _, err := gate(); err != nil {
+				if l.logger != nil {
+					l.logger.Printf("%s (%s.%s, %s) is not kept on the bus: %v — the controller holding the lease "+
+						"marks it abandoned", c.ID, c.Seat, c.Verb, c.State, err)
+				}
+				continue
+			}
+		}
 		var err error
 		for try := 0; try < keepTries; try++ {
 			ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
@@ -237,6 +263,9 @@ func (l *CallLog) begin(seat, verb string, args json.RawMessage, reply string) *
 	c := &Call{ID: "call-" + strconv.FormatInt(l.now().UnixNano(), 10) + "-" + strconv.FormatUint(l.next, 10),
 		Seat: seat, Verb: verb, Args: args, Started: l.now(), State: CallRunning, reply: reply,
 		Caller: callerOf(reply), Holder: l.holder}
+	if l.epoch != nil {
+		c.Epoch, _ = l.epoch() // zero when not held: its record is then not written either
+	}
 	l.calls = append(l.calls, c)
 	if len(l.calls) > KeptCalls {
 		l.calls = l.calls[len(l.calls)-KeptCalls:]

internal/link/contracts.go

@@ -0,0 +1,47 @@
+package link
+
+// The contract of every message kind the controller consumes (novox/hq to-be 45 Phase 2, ADR 0227 rule
+// 2 "how it is checked": a contract test per consumed message kind in the receiver's repository, and a
+// check listing every consumed subject against the tests that name it).
+//
+// **Each kind says how an older one is told from a newer, and the tests that deliver n, then n−1.** A
+// kind that carries no order says why none is needed — a word whose newest is simply the latest heard,
+// a request answered once. The test beside it fails a kind the controller can be handed without an
+// entry here, and an entry naming a test that does not exist.
+
+// Contract is one consumed kind's order.
+type Contract struct {
+	// Ordered is how an older message of this kind is refused; empty for a kind that carries no order.
+	Ordered string
+	// Unordered is why a kind needs no order; empty for an ordered one.
+	Unordered string
+	// Tests are the tests that deliver the newer and then the older, and assert the older refused.
+	Tests []string
+}
+
+// Contracts are every kind the controller consumes, by kind.
+var Contracts = map[string]Contract{
+	KindReport: {Ordered: "by the epoch and sequence of the declaration it is about, then the node-engine's " +
+		"report sequence (order.go); an older account is refused and counted. A report from a node-engine " +
+		"that orders nothing is judged by the digest it names (issue 267)",
+		Tests: []string{"TestAnAccountOfAnOlderDeclarationIsRefusedByItsSequence",
+			"TestAnOlderReportOfTheSameDeclarationIsRefused", "TestAnAccountOfAnOlderDeclarationDoesNotReplaceTheNewer",
+			"TestAnAccountIsOlderByEpochThenSequenceThenReportSequence"}},
+	KindBuilt: {Ordered: "by the build's ask: an older ask finishing later is recorded and not registered (issue 219)",
+		Tests: []string{"TestAnOlderBuildHeardLaterDoesNotReplaceTheNewer"}},
+	KindSourceMoved: {Ordered: "by the merge's commit against what was built from it: a merge already acted on " +
+		"or older than the last look is history, not a second plan (issues 250, 266)",
+		Tests: []string{"TestAMergeOlderThanTheLastLookIsHistory", "TestAMergeMatchesTheSourcesBuiltFromIt"}},
+	KindHeartbeat: {Unordered: "a word that the machine is there: the newest heard is the newest said, and one " +
+		"lost is the next one"},
+	KindToolsHeartbeat: {Unordered: "a word that the node tools are there, as a machine's heartbeat"},
+	KindEnrolment: {Unordered: "a request answered once, under a token spent once: a second presentation is " +
+		"refused by the token, not by an order (issue 083)",
+		Tests: []string{"TestAnEnrolmentMetByAHeldTokenIsAskedToTryAgain"}},
+	KindModuleMoved: {Unordered: "the catalogue saying a module's current build moved: acted on by reading the " +
+		"catalogue's record, which is the order, so a late one reads the same record"},
+	KindCatchUp: {Unordered: "a catalogue asking what it missed: answered from the record, whenever asked"},
+	KindProvisioner: {Unordered: "a provider's newest word about a consumer, said again every fifteen minutes " +
+		"while it holds (ADR 0224): the condition keeps the last observed, and S8 says when the words stop",
+		Tests: []string{"TestAProviderFailingAConsumerBreaksAllWellUntilItRecovers"}},
+}

internal/link/contracts_test.go

@@ -0,0 +1,84 @@
+package link
+
+import (
+	"go/parser"
+	"go/token"
+	"io/fs"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/novox/mesh-controller/internal/broker"
+)
+
+// Every subject the controller consumes is a kind with a contract, and every test a contract names
+// exists (novox/hq to-be 45 Phase 2, ADR 0227 rule 2).
+func TestEveryConsumedKindHasAContract(t *testing.T) {
+	// What the controller can be handed, from the subjects it is granted and the ones it derives.
+	subjects := []string{EnrolSubject, BuiltSubject, ReportSubject("anchor"), AliveSubject("anchor"),
+		ToolsAliveSubject("anchor"), "mesh.mod.postgres.event.provisioner.failing"}
+	subjects = append(subjects, broker.ControllerFollows...)
+	kinds := map[string]bool{}
+	for _, s := range subjects {
+		kind, known := kindOfSubject(s)
+		if !known {
+			if strings.Contains(s, "*") {
+				continue // a pattern among the follows; its concrete subject is listed above
+			}
+			t.Errorf("the controller follows %s and has no kind for it", s)
+			continue
+		}
+		kinds[kind] = true
+	}
+	for kind := range kinds {
+		c, ok := Contracts[kind]
+		switch {
+		case !ok:
+			t.Errorf("the controller consumes %s and no contract says how an older one is told from a newer", kind)
+		case (c.Ordered == "") == (c.Unordered == ""):
+			t.Errorf("%s's contract says neither, or both, how it is ordered and why it need not be: %+v", kind, c)
+		case c.Ordered != "" && len(c.Tests) == 0:
+			t.Errorf("%s is ordered and no test delivers the newer and then the older", kind)
+		}
+	}
+	for kind := range Contracts {
+		if !kinds[kind] {
+			t.Errorf("a contract for %s, which the controller does not consume", kind)
+		}
+	}
+
+	// Every test named exists in this repository.
+	exists := map[string]bool{}
+	fset := token.NewFileSet()
+	err := filepath.WalkDir("../..", func(path string, d fs.DirEntry, err error) error {
+		if err != nil {
+			return err
+		}
+		if d.IsDir() && (d.Name() == "vendor" || d.Name() == ".git") {
+			return filepath.SkipDir
+		}
+		if d.IsDir() || !strings.HasSuffix(path, "_test.go") {
+			return nil
+		}
+		f, err := parser.ParseFile(fset, path, nil, 0)
+		if err != nil {
+			return err
+		}
+		for name, obj := range f.Scope.Objects {
+			if obj.Kind.String() == "func" && strings.HasPrefix(name, "Test") {
+				exists[name] = true
+			}
+		}
+		return nil
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	for kind, c := range Contracts {
+		for _, name := range c.Tests {
+			if !exists[name] {
+				t.Errorf("%s's contract names %s, which no test in this repository is", kind, name)
+			}
+		}
+	}
+}

internal/link/declare.go

@@ -12,6 +12,10 @@ type Signer interface {
 	Sign(ctx context.Context, message []byte) ([]byte, error)
 }
 
+// ActingGate is what every send passes before it is made (novox/hq to-be 45 §6): whether this process
+// may act under the controller's lease. Set by the controller; nil passes every send — a test, a tool.
+var ActingGate func(ctx context.Context) error
+
 // Declare sends a node what it should be, signed.
 //
 // The signature is over the declaration exactly as it is published — the same bytes the node
@@ -25,6 +29,12 @@ func Declare(ctx context.Context, bus Bus, signer Signer, node string,
 	if !json.Valid(declaration) {
 		return fmt.Errorf("refusing to send %s something that is not a declaration", node)
 	}
+	// **At the send, not only where it was composed**: a lease lost between the two stops this one.
+	if ActingGate != nil {
+		if err := ActingGate(ctx); err != nil {
+			return fmt.Errorf("%s was not sent its declaration: %w", node, err)
+		}
+	}
 
 	signature, err := signer.Sign(ctx, declaration)
 	if err != nil {

internal/link/enrolment.go

@@ -11,6 +11,7 @@ import (
 	"fmt"
 	"log"
 	"sort"
+	"time"
 
 	"github.com/novox/mesh-controller/internal/broker"
 	"github.com/novox/mesh-controller/internal/identity"
@@ -421,23 +422,54 @@ func (e Enrolment) Heard(ctx context.Context, report Report) (news bool, err err
 	// the newer one arrived reported *after* the newer apply's report, and stored last, it read as the
 	// machine never having applied what it was sent — the plan waited until somebody pushed by hand.
 	// One row per node means the last write wins, so the order of arrival must not decide it.
-	if report.Declared != "" {
-		sent, err := e.Inventory.Outstanding(ctx, report.Node)
+	//
+	// **By order, from a node-engine that orders its reports** (novox/hq to-be 45 §6, the contract in
+	// order.go): the account is kept only if it is not older than the one kept — by the epoch and
+	// sequence of the declaration it is about, then the node-engine's own report sequence — and refused
+	// otherwise, said and counted (rule 2). One rule, whatever the account is about and whenever it
+	// arrives; the digest the mesh last sent decides nothing for it. **By digest, from an older
+	// node-engine**: its report claims no order, and the rule of issue 267 stands for it.
+	//
+	// And whether the node-engine reads an epoch in a declaration, from every account it gives: the
+	// mesh sends one only to a machine that said so, and one rolled back says so no longer.
+	if err := e.Inventory.RecordReadsEpoch(ctx, node.ID, report.ReadsEpoch()); err != nil {
+		return false, err
+	}
+	if report.Ordered() {
+		account := AccountOf(report)
+		news, err = e.Inventory.RecordOrderedDoing(ctx, node.ID, doing,
+			inventory.ReportOrder{Epoch: report.Epoch, Sequence: report.Sequence, ReportSequence: report.ReportSequence},
+			func(kept inventory.ReportOrder) bool {
+				return account.OlderThan(Account{Order: Order{Epoch: kept.Epoch, Sequence: kept.Sequence},
+					ReportSequence: kept.ReportSequence})
+			})
+		if errors.Is(err, inventory.ErrOlderAccount) {
+			log.Printf("kept what %s says about the machine, and refused its account of declaration %s (%s, report %d): "+
+				"the account kept is newer", report.Node, short(report.Declared), report.Order.Words(), report.ReportSequence)
+			StaleRefusals.Refused(Refusal{Writer: WriterNodeEngine(report.Node), Receiver: "controller", At: time.Now()})
+			return false, e.Inventory.Seen(ctx, node.ID)
+		}
 		if err != nil {
 			return false, err
 		}
-		if Superseded(report.Declared, sent) {
-			log.Printf("kept what %s says about the machine, and not its account of declaration %s: "+
-				"the mesh has sent it %s since", report.Node, short(report.Declared), short(sent))
-			return false, e.Inventory.Seen(ctx, node.ID)
+	} else {
+		if report.Declared != "" {
+			sent, err := e.Inventory.Outstanding(ctx, report.Node)
+			if err != nil {
+				return false, err
+			}
+			if Superseded(report.Declared, sent) {
+				log.Printf("kept what %s says about the machine, and not its account of declaration %s: "+
+					"the mesh has sent it %s since", report.Node, short(report.Declared), short(sent))
+				return false, e.Inventory.Seen(ctx, node.ID)
+			}
+		}
+		// **Whether this is news** is the store's answer: it holds the previous report, and a machine
+		// that reconciles every minute says the same thing until something changes (novox/hq ADR 0134).
+		news, err = e.Inventory.RecordDoing(ctx, node.ID, doing)
+		if err != nil {
+			return false, err
 		}
-	}
-
-	// **Whether this is news** is the store's answer: it holds the previous report, and a machine
-	// that reconciles every minute says the same thing until something changes (novox/hq ADR 0134).
-	news, err = e.Inventory.RecordDoing(ctx, node.ID, doing)
-	if err != nil {
-		return false, err
 	}
 	// And how long the machine took, from the send to this first account of it (novox/hq to-be 45
 	// Phase 0): what the sent-not-reported bound will be set from. Said if lost, never a failure.

internal/link/heard_order_test.go

@@ -0,0 +1,163 @@
+package link_test
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	"github.com/novox/mesh-controller/internal/inventory"
+	"github.com/novox/mesh-controller/internal/link"
+)
+
+// A report kept by its order (novox/hq to-be 45 §6, ADR 0227 rule 2): the contract test of the report,
+// the message kind the controller consumes from every machine. Deliver n, then n−1: refused and counted.
+// Of the same declaration, report r then r−1: refused. The digest the mesh last sent decides nothing
+// for an ordered report; an unordered one, from an older node-engine, is judged by it as before.
+
+func anOrderedMachine(t *testing.T) (*inventory.Inventory, link.Enrolment, string) {
+	t.Helper()
+	inv := inventory.ForTest(t)
+	node, err := inv.AddNode(context.Background(), "home-server")
+	if err != nil {
+		t.Fatal(err)
+	}
+	return inv, link.Enrolment{Inventory: inv}, node.ID
+}
+
+func ordered(declared string, epoch, sequence, report int64, applied ...string) link.Report {
+	return link.Report{Node: "home-server", Declared: declared, Applied: applied,
+		Order: link.Order{Epoch: epoch, Sequence: sequence}, ReportSequence: report}
+}
+
+// Issue 267, ordered: a reconcile's account of declaration 11 reaches the mesh after the apply's of 12.
+func TestAnAccountOfAnOlderDeclarationIsRefusedByItsSequence(t *testing.T) {
+	inv, heard, id := anOrderedMachine(t)
+	ctx := context.Background()
+	before := countFor(link.WriterNodeEngine("home-server"))
+	if _, err := heard.Heard(ctx, ordered("d12", 57, 12, 41, "a", "b")); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := heard.Heard(ctx, ordered("d11", 57, 11, 40, "a")); err != nil {
+		t.Fatal(err)
+	}
+	doing, _, err := inv.DoingOf(ctx, "home-server")
+	if err != nil || doing.Declared != "d12" || doing.Applied != 2 {
+		t.Fatalf("the older account replaced the newer: %+v (%v)", doing, err)
+	}
+	if got := countFor(link.WriterNodeEngine("home-server")) - before; got != 1 {
+		t.Fatalf("the refusal was counted %d times, want once", got)
+	}
+	kept, err := inv.KeptOrder(ctx, id)
+	if err != nil || kept != (inventory.ReportOrder{Epoch: 57, Sequence: 12, ReportSequence: 41}) {
+		t.Fatalf("the order kept is %+v (%v)", kept, err)
+	}
+}
+
+func TestAnOlderReportOfTheSameDeclarationIsRefused(t *testing.T) {
+	inv, heard, _ := anOrderedMachine(t)
+	ctx := context.Background()
+	if _, err := heard.Heard(ctx, ordered("d12", 57, 12, 41, "a", "b")); err != nil {
+		t.Fatal(err)
+	}
+	failed := ordered("d12", 57, 12, 40)
+	failed.Failed = map[string]string{"b": "it failed a moment before it applied"}
+	if _, err := heard.Heard(ctx, failed); err != nil {
+		t.Fatal(err)
+	}
+	doing, _, _ := inv.DoingOf(ctx, "home-server")
+	if doing.Outcome != inventory.OutcomeApplied {
+		t.Fatalf("an older report of the same declaration replaced the newer: %+v", doing)
+	}
+	// A newer report of it — the next reconcile — is kept.
+	again := ordered("d12", 57, 12, 42)
+	again.Failed = map[string]string{"b": "it failed since"}
+	if _, err := heard.Heard(ctx, again); err != nil {
+		t.Fatal(err)
+	}
+	if doing, _, _ := inv.DoingOf(ctx, "home-server"); doing.Outcome != inventory.OutcomeFailed {
+		t.Fatalf("a newer report of the same declaration was not kept: %+v", doing)
+	}
+}
+
+// An ordered account is judged by its order, not by the digest the mesh last sent: the account of 12
+// is kept although the mesh has sent 13 since — it is still the newest account of the machine.
+func TestAnOrderedAccountIsNotJudgedByTheDigestSent(t *testing.T) {
+	inv, heard, id := anOrderedMachine(t)
+	ctx := context.Background()
+	if err := inv.RecordSent(ctx, id, "d13", nil); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := heard.Heard(ctx, ordered("d12", 57, 12, 41, "a")); err != nil {
+		t.Fatal(err)
+	}
+	if doing, said, _ := inv.DoingOf(ctx, "home-server"); !said || doing.Declared != "d12" {
+		t.Fatalf("the newest account the machine gave was set aside by the digest: %+v", doing)
+	}
+	// And the machine reads an epoch: it orders its reports.
+	if reads, err := inv.ReadsEpoch(ctx, id); err != nil || !reads {
+		t.Fatalf("a machine whose reports carry a report sequence is not recorded as reading an epoch: %v", err)
+	}
+}
+
+// A node-engine rolled back to one that orders nothing: its account is taken by the digest rule, the
+// order kept is cleared, and it is no longer sent an epoch.
+func TestAnUnorderedAccountClearsTheOrderAndTheEpoch(t *testing.T) {
+	inv, heard, id := anOrderedMachine(t)
+	ctx := context.Background()
+	if _, err := heard.Heard(ctx, ordered("d12", 57, 12, 41, "a")); err != nil {
+		t.Fatal(err)
+	}
+	if err := inv.RecordSent(ctx, id, "d13", nil); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := heard.Heard(ctx, link.Report{Node: "home-server", Declared: "d13", Applied: []string{"a", "b"}}); err != nil {
+		t.Fatal(err)
+	}
+	if doing, _, _ := inv.DoingOf(ctx, "home-server"); doing.Declared != "d13" {
+		t.Fatalf("an older node-engine's account of what was sent was not kept: %+v", doing)
+	}
+	if kept, _ := inv.KeptOrder(ctx, id); kept != (inventory.ReportOrder{}) {
+		t.Fatalf("an unordered account left the order kept: %+v", kept)
+	}
+	if reads, _ := inv.ReadsEpoch(ctx, id); reads {
+		t.Fatal("a node-engine that orders nothing is still sent an epoch")
+	}
+	// The next ordered account, whatever its numbers, has nothing to be older than.
+	if _, err := heard.Heard(ctx, ordered("d14", 58, 14, 1, "a")); err != nil {
+		t.Fatal(err)
+	}
+	if doing, _, _ := inv.DoingOf(ctx, "home-server"); doing.Declared != "d14" {
+		t.Fatalf("the first ordered account after an unordered one was refused: %+v", doing)
+	}
+}
+
+// A stale refusal names its writer by the epoch the refused declaration claimed, and a refusal whose
+// own report was lost is still counted from the machine's own tally.
+func TestAStaleRefusalIsCountedByItsWriter(t *testing.T) {
+	count := link.NewRefusalCount()
+	now := time.Now()
+	refusal := link.Report{Node: "anchor", Refused: "older", Order: link.Order{Epoch: 41, Sequence: 11},
+		OlderThan: &link.Order{Epoch: 57, Sequence: 12}, ReportSequence: 9, RefusedOlder: 3}
+	if !refusal.StaleRefusalOf() {
+		t.Fatal("a refusal naming the order it holds is not stale")
+	}
+	count.Lifetime("anchor", 2, now) // the machine's tally, as the controller first heard it
+	count.Refused(link.Refusal{Writer: link.WriterEpoch(refusal.Epoch), Epoch: refusal.Epoch, Receiver: "anchor", At: now})
+	count.Lifetime("anchor", refusal.RefusedOlder, now)
+	count.Lifetime("anchor", 5, now) // two more refused, their reports lost
+	got := count.Within(now.Add(-time.Minute))
+	if len(got) != 2 || got[0].Count != 2 || got[0].Epoch != 0 || got[1].Writer != "the controller of epoch 41" ||
+		got[1].Count != 1 || got[1].Receivers[0] != "anchor" {
+		t.Fatalf("the refusals by writer are %+v", got)
+	}
+}
+
+// countFor is how many refusals of a writer this process has heard in the last minute.
+func countFor(writer string) int {
+	for _, w := range link.StaleRefusals.Within(time.Now().Add(-time.Minute)) {
+		if w.Writer == writer {
+			return w.Count
+		}
+	}
+	return 0
+}

internal/link/order.go

@@ -0,0 +1,118 @@
+package link
+
+import "strconv"
+
+// The order a declaration carries, and the order a report about one carries back (novox/hq to-be 45
+// §6, ADR 0227 rule 2).
+//
+// **This file is the controller's side of the contract with the node-engine for Phase 2**, and the one
+// place it is written here. It mirrors mesh-host internal/link/messages.go (`Order`, and the report's
+// `report_sequence`, `older_than`, `refused_older`) field for field and rule for rule, as Report has
+// always mirrored the host's report: a key added on one side and not the other is a key the other side
+// does not know exists.
+//
+// **The wire.**
+//
+//   - A declaration's order is two top-level keys of its signed envelope, beside "declaration" and
+//     "resources": `sequence` (since issue 107) and `epoch` (new). Inside the signed bytes, so a message
+//     cannot be given a newer order than the controller gave it.
+//   - A report carries the order of the declaration it is about as the same two top-level keys, beside
+//     "declared"; `report_sequence`, the node-engine's own number for the report; `older_than`, on a
+//     refusal of a declaration older than one it applied, the order of the one it holds; and
+//     `refused_older`, how many it has refused so, ever, on every report.
+//
+// Every key is optional and absent when zero, and zero is "no order claimed", never "first". So each
+// side reads the other's older shape:
+//
+//   - **An older node-engine with this controller.** It decodes a declaration strictly and refuses a key
+//     it does not know, whole — so `epoch` is sent only to a machine whose latest report carried a
+//     `report_sequence`, which a node-engine that reads the epoch always does. Until then it is sent the
+//     sequence alone, as today; its reports carry no report sequence and are judged by the digest they
+//     name, as today (issue 267).
+//   - **A newer node-engine with an older controller.** It is sent no epoch, which claims none: it
+//     refuses nothing by epoch. The keys it adds to a report are fields the older controller ignores.
+//   - **A controller rolled back to a build without the lease** sends no epoch again and is not refused
+//     for it: a node-engine refuses by epoch only when both declarations claim one. That gives up the
+//     epoch's protection while such a build runs — the price of a rollback that cannot strand every
+//     machine.
+//
+// **Epoch** is the controller lease's epoch (to-be 45 §6): the lease bucket's revision at which the
+// instance that composed the declaration took the lease. It only grows: a later holder's is higher, and
+// the controller never issues one at or under the highest it has issued (internal/lease, the floor).
+// **Sequence** is the declaration's number for that machine, one higher every send, taken from the
+// controller's store before the declaration is composed (issues 107, 204).
+
+// Order is where a declaration stands among everything the mesh has sent a machine: the lease epoch it
+// was sent under and its sequence. Zero in either claims none.
+type Order struct {
+	Epoch    int64 `json:"epoch,omitempty"`
+	Sequence int64 `json:"sequence,omitempty"`
+}
+
+// Older is the node-engine's rule, as mesh-host states it: a declaration of this order is older than
+// one of order `than` the machine applied **only when both claim an epoch** — and then when its epoch
+// is lower, or its epoch is the same and its sequence lower (both claimed). A higher epoch is a new lease
+// holder and is never older, whatever its sequence. Here so the controller's tests state what the
+// machines will do with what it sends.
+func (o Order) Older(than Order) bool {
+	if o.Epoch <= 0 || than.Epoch <= 0 {
+		return false
+	}
+	if o.Epoch != than.Epoch {
+		return o.Epoch < than.Epoch
+	}
+	return o.Sequence > 0 && than.Sequence > 0 && o.Sequence < than.Sequence
+}
+
+// Words is an order as a person reads it in a log line. Not String: Report embeds Order, and a Stringer
+// promoted onto every report would print each one as its order alone.
+func (o Order) Words() string {
+	switch {
+	case o.Epoch > 0:
+		return "epoch " + strconv.FormatInt(o.Epoch, 10) + ", sequence " + strconv.FormatInt(o.Sequence, 10)
+	case o.Sequence > 0:
+		return "sequence " + strconv.FormatInt(o.Sequence, 10) + ", no epoch"
+	}
+	return "no order"
+}
+
+// Account is the order of one account of a machine — a report about a declaration: that declaration's
+// order and the node-engine's own number for the report.
+type Account struct {
+	Order
+	ReportSequence int64
+}
+
+// AccountOf is a report's account order.
+func AccountOf(r Report) Account { return Account{Order: r.Order, ReportSequence: r.ReportSequence} }
+
+// Ordered says a report comes from a node-engine that orders its reports: it carries a report sequence.
+// Such a report is judged by its order (OlderThan); one without is judged by the digest it names, as
+// before (issue 267).
+func (r Report) Ordered() bool { return r.ReportSequence > 0 }
+
+// ReadsEpoch says the node-engine that made the report takes an epoch in a declaration: one that orders
+// its reports does (mesh-host: "its presence also says this host reads a declaration's epoch").
+func (r Report) ReadsEpoch() bool { return r.ReportSequence > 0 }
+
+// StaleRefusalOf says a report refuses a declaration older than one the machine applied: the node-engine
+// names the order it holds (`older_than`), or, from a node-engine older than that, says so in the words
+// of its sequence refusal.
+func (r Report) StaleRefusalOf() bool {
+	return r.OlderThan != nil || (r.Refused != "" && IsStaleRefusal(r.Refused))
+}
+
+// OlderThan is the controller's rule (to-be 45 §6): whether this account is older than the one kept for
+// the machine — **by epoch, then sequence, then report sequence**, each compared only where both claim
+// one. An account of a declaration of an older epoch is refused; of an older sequence, refused; an older
+// report of the same declaration (a reconcile's, overtaken by the apply's — issue 267), refused. Equal is
+// the same report again, redelivered, and is not refused.
+func (a Account) OlderThan(kept Account) bool {
+	if a.Epoch > 0 && kept.Epoch > 0 && a.Epoch != kept.Epoch {
+		return a.Epoch < kept.Epoch
+	}
+	if a.Sequence > 0 && kept.Sequence > 0 && a.Sequence != kept.Sequence {
+		return a.Sequence < kept.Sequence
+	}
+	return a.ReportSequence > 0 && kept.ReportSequence > 0 && a.ReportSequence < kept.ReportSequence
+}

internal/link/order_test.go

@@ -0,0 +1,113 @@
+package link
+
+import (
+	"encoding/json"
+	"testing"
+)
+
+// The contract of order.go, case by case: what a node-engine refuses of a declaration (the rule
+// mesh-host states, restated so the controller's tests say what the machines do with what it sends),
+// what the controller refuses of a report, and the bytes on the wire both ways.
+
+func TestADeclarationIsOlderOnlyWhenBothClaimAnEpoch(t *testing.T) {
+	for _, c := range []struct {
+		name     string
+		arriving Order
+		held     Order
+		older    bool
+	}{
+		{"a newer sequence of the same epoch", Order{57, 12}, Order{57, 11}, false},
+		{"the same declaration again (a reconcile)", Order{57, 12}, Order{57, 12}, false},
+		{"a lower sequence of the same epoch", Order{57, 11}, Order{57, 12}, true},
+		// Issue 204: a controller that lost its lease goes on sending.
+		{"an older epoch, whatever its sequence", Order{41, 99}, Order{57, 12}, true},
+		{"a newer epoch, whatever its sequence", Order{57, 3}, Order{41, 99}, false},
+		{"no epoch arriving: a rolled-back controller is not stranded", Order{0, 11}, Order{57, 12}, false},
+		{"no epoch held: what the machine held before any lease", Order{57, 11}, Order{0, 12}, false},
+		{"no order at all", Order{}, Order{57, 12}, false},
+	} {
+		t.Run(c.name, func(t *testing.T) {
+			if got := c.arriving.Older(c.held); got != c.older {
+				t.Fatalf("%s against %s: older %v, want %v", c.arriving.Words(), c.held.Words(), got, c.older)
+			}
+		})
+	}
+}
+
+func TestAnAccountIsOlderByEpochThenSequenceThenReportSequence(t *testing.T) {
+	for _, c := range []struct {
+		name     string
+		arriving Account
+		kept     Account
+		older    bool
+	}{
+		// Issue 267: a reconcile's account of declaration 11 arrives after the apply's of 12.
+		{"an account of an older declaration", Account{Order{57, 11}, 40}, Account{Order{57, 12}, 41}, true},
+		{"an older report of the same declaration", Account{Order{57, 12}, 40}, Account{Order{57, 12}, 41}, true},
+		{"a newer report of the same declaration (a reconcile after the apply)",
+			Account{Order{57, 12}, 42}, Account{Order{57, 12}, 41}, false},
+		{"the same report again (redelivered)", Account{Order{57, 12}, 41}, Account{Order{57, 12}, 41}, false},
+		{"an account of a newer declaration, whatever its report sequence",
+			Account{Order{57, 13}, 1}, Account{Order{57, 12}, 41}, false},
+		{"an account of an older epoch", Account{Order{41, 99}, 50}, Account{Order{57, 12}, 41}, true},
+		{"an account of a newer epoch", Account{Order{58, 1}, 2}, Account{Order{57, 12}, 41}, false},
+		{"no epoch either side: the sequences", Account{Order{0, 11}, 50}, Account{Order{0, 12}, 41}, true},
+		{"no report sequence: the declaration's alone", Account{Order{57, 12}, 0}, Account{Order{57, 12}, 41}, false},
+		{"an unordered declaration: the report sequences", Account{Order{}, 40}, Account{Order{}, 41}, true},
+		{"nothing kept yet", Account{Order{57, 3}, 1}, Account{}, false},
+	} {
+		t.Run(c.name, func(t *testing.T) {
+			if got := c.arriving.OlderThan(c.kept); got != c.older {
+				t.Fatalf("%+v against %+v: older %v, want %v", c.arriving, c.kept, got, c.older)
+			}
+		})
+	}
+}
+
+// The bytes, as mesh-host's report writes them: every key optional, absent when zero.
+func TestTheOrderOnTheWire(t *testing.T) {
+	// A stale refusal, as the node-engine says it: the refused declaration's order at the top, the one
+	// it holds in older_than, and how many it has refused ever.
+	raw := []byte(`{"node":"anchor","refused":"older","declared":"d1","epoch":41,"sequence":11,` +
+		`"report_sequence":7,"older_than":{"epoch":57,"sequence":12},"refused_older":3}`)
+	var r Report
+	if err := json.Unmarshal(raw, &r); err != nil {
+		t.Fatal(err)
+	}
+	if r.Order != (Order{41, 11}) || r.ReportSequence != 7 || r.OlderThan == nil || *r.OlderThan != (Order{57, 12}) ||
+		r.RefusedOlder != 3 {
+		t.Fatalf("a node-engine's stale refusal reads as %+v", r)
+	}
+	if !r.Ordered() || !r.ReadsEpoch() || !r.StaleRefusalOf() {
+		t.Fatalf("a node-engine that orders its reports reads as one that does not: %+v", r)
+	}
+
+	// A report from a node-engine older than the contract says none of it.
+	var older Report
+	if err := json.Unmarshal([]byte(`{"node":"anchor","applied":["a"],"declared":"d1"}`), &older); err != nil {
+		t.Fatal(err)
+	}
+	if older.Ordered() || older.ReadsEpoch() || older.StaleRefusalOf() {
+		t.Fatalf("an older node-engine's report reads as ordered: %+v", older)
+	}
+	// Its stale refusal, in the words it has always used, is still one.
+	older.Refused = "this declaration " + StaleRefusal + ": it is sequence 3"
+	if !older.StaleRefusalOf() {
+		t.Fatal("an older node-engine's refusal of an older sequence does not read as stale")
+	}
+
+	// And a report with no order puts no order key on the wire.
+	raw, err := json.Marshal(Report{Node: "anchor", Declared: "d1"})
+	if err != nil {
+		t.Fatal(err)
+	}
+	var keys map[string]any
+	if err := json.Unmarshal(raw, &keys); err != nil {
+		t.Fatal(err)
+	}
+	for _, key := range []string{"epoch", "sequence", "report_sequence", "older_than", "refused_older"} {
+		if _, there := keys[key]; there {
+			t.Fatalf("an unordered report carries %s: %s", key, raw)
+		}
+	}
+}

internal/link/protocol.go

@@ -178,6 +178,23 @@ type Report struct {
 	// the same way, as the `sent` digest the mesh recorded. Which declaration, not when.
 	Declared string `json:"declared,omitempty"`
 
+	// Order is where the declaration this report is about stands — its `epoch` and `sequence` as the
+	// declaration carried them — so the mesh keeps accounts by what they are about rather than by when
+	// they arrived (novox/hq to-be 45 §6; the contract is order.go). Two top-level keys; absent for a
+	// declaration that claimed no order, and from a node-engine older than the contract.
+	Order
+	// ReportSequence is the node-engine's own number for this report: one higher for every report it
+	// makes, kept on disk across restarts and self-updates. Zero claims none — every report an older
+	// node-engine makes. Its presence also says the node-engine reads a declaration's epoch.
+	ReportSequence int64 `json:"report_sequence,omitempty"`
+	// OlderThan is set on a report refusing a declaration older than one the machine applied: the order
+	// of the one it holds. The refused declaration is the report's own Declared and Order — whose epoch
+	// names the controller that sent it (S13).
+	OlderThan *Order `json:"older_than,omitempty"`
+	// RefusedOlder is how many declarations the node-engine has refused as older, ever, on every report:
+	// a refusal whose own report was lost is still counted from the next.
+	RefusedOlder int64 `json:"refused_older,omitempty"`
+
 	// Held is what an adopted node found and is keeping as it was until its module is taken
 	// (novox/hq ADR 0100). Without it an adopted node reads as converged.
 	Held []Held `json:"held,omitempty"`

internal/link/serve.go

@@ -317,6 +317,11 @@ func (s *Server) reported(ctx context.Context, m Control) {
 		// the server's, it comes back after the newer was applied whatever the controller does.
 		outstanding := s.outstanding(ctx, report.Node)
 		declaredIn := staleAgainst(report)
+		if report.Ordered() {
+			// Kept by its order, not by the digest the mesh last sent (novox/hq to-be 45 §6): the
+			// listener refuses an older account under the one rule, whatever it is about.
+			declaredIn = ""
+		}
 		if Superseded(declaredIn, outstanding) {
 			s.log.Printf("set aside %s: the mesh has moved past that declaration", what)
 			_ = m.Took()

internal/link/watched.go

@@ -5,6 +5,8 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"slices"
+	"sort"
 	"strings"
 	"sync"
 	"sync/atomic"
@@ -176,48 +178,130 @@ const StaleRefusal = "is older than what the mesh last said to this node"
 // IsStaleRefusal says a report's refusal is the node-engine refusing a declaration older than it holds.
 func IsStaleRefusal(refused string) bool { return strings.Contains(refused, StaleRefusal) }
 
-// RefusalCount keeps when each machine refused a stale declaration, for S13 (novox/hq to-be 45 §3):
-// more than five from one writer in five minutes is a writer sending what it has moved past.
+// A Refusal is one receiver refusing something older than what it holds (novox/hq to-be 45 §6, ADR 0227
+// rule 2): a machine refusing a declaration, or this controller refusing an account.
+type Refusal struct {
+	// Writer is who sent what was refused, in the mesh's words: WriterEpoch for a declaration, a
+	// machine's node-engine for an account.
+	Writer string
+	// Epoch is the lease epoch the refused declaration claimed, for a controller to be named by; zero
+	// for a declaration that claimed none, and for an account.
+	Epoch int64
+	// Receiver is the machine, or "controller", that refused it.
+	Receiver string
+	At       time.Time
+}
+
+// WriterEpoch is how a refused declaration's writer is said before it is named: by the epoch it
+// claimed, or as a controller that claimed none.
+func WriterEpoch(epoch int64) string {
+	if epoch > 0 {
+		return fmt.Sprintf("the controller of epoch %d", epoch)
+	}
+	return "a controller that claimed no epoch"
+}
+
+// WriterNodeEngine is a machine's node-engine as the writer of an account the controller refused.
+func WriterNodeEngine(node string) string { return "the node-engine on " + node }
+
+// RefusalCount keeps every stale refusal heard, for S13 (novox/hq to-be 45 §3): more than five from
+// one writer in five minutes is a writer sending what it — or the mesh — has moved past.
 type RefusalCount struct {
-	mu  sync.Mutex
-	per map[string][]time.Time
+	mu   sync.Mutex
+	list []Refusal
+	// lifetime is each machine's own count of declarations it refused as older, as its last report
+	// said it (`refused_older`), and named how many of those this process heard as refusal reports
+	// since: what the count rose by beyond them is refusals whose reports never arrived.
+	lifetime map[string]int64
+	named    map[string]int64
 }
 
 // StaleRefusals is this process's count.
-var StaleRefusals = &RefusalCount{per: map[string][]time.Time{}}
+var StaleRefusals = NewRefusalCount()
 
-// Refused records one refusal by a machine.
-func (r *RefusalCount) Refused(node string, at time.Time) {
-	r.mu.Lock()
-	defer r.mu.Unlock()
-	r.per[node] = append(r.per[node], at)
+// NewRefusalCount is an empty count.
+func NewRefusalCount() *RefusalCount {
+	return &RefusalCount{lifetime: map[string]int64{}, named: map[string]int64{}}
 }
 
-// Within is how many refusals each machine made since a moment; older ones are forgotten.
-func (r *RefusalCount) Within(since time.Time) map[string]int {
+// Refused records one refusal.
+func (r *RefusalCount) Refused(f Refusal) {
 	r.mu.Lock()
 	defer r.mu.Unlock()
-	out := map[string]int{}
-	for node, times := range r.per {
-		var kept []time.Time
-		for _, t := range times {
-			if !t.Before(since) {
-				kept = append(kept, t)
-			}
-		}
-		if len(kept) == 0 {
-			delete(r.per, node)
+	r.list = append(r.list, f)
+	if f.Writer != WriterNodeEngine(f.Receiver) {
+		r.named[f.Receiver]++
+	}
+}
+
+// Lifetime reads a machine's own count of declarations it refused as older, from any report: what it
+// rose by since the last, beyond the refusals heard by name, is recorded as refused by a writer the
+// mesh never heard named — the refusal's report was lost, and the count is still a fact.
+func (r *RefusalCount) Lifetime(node string, total int64, at time.Time) {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	before, known := r.lifetime[node]
+	r.lifetime[node] = total
+	named := r.named[node]
+	r.named[node] = 0
+	if !known || total <= before {
+		return // the first word since this controller started, or a node-engine that started over
+	}
+	for missing := total - before - named; missing > 0; missing-- {
+		r.list = append(r.list, Refusal{Writer: "a writer whose refused declaration was never reported",
+			Receiver: node, At: at})
+	}
+}
+
+// WriterRefusals is one writer's refusals within a window.
+type WriterRefusals struct {
+	Writer    string
+	Epoch     int64
+	Count     int
+	Receivers []string
+	Last      time.Time
+}
+
+// Within is every writer's refusals since a moment, most first; older ones are forgotten.
+func (r *RefusalCount) Within(since time.Time) []WriterRefusals {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	kept := r.list[:0]
+	by := map[string]*WriterRefusals{}
+	var order []string
+	for _, f := range r.list {
+		if f.At.Before(since) {
 			continue
 		}
-		r.per[node] = kept
-		out[node] = len(kept)
+		kept = append(kept, f)
+		w, ok := by[f.Writer]
+		if !ok {
+			w = &WriterRefusals{Writer: f.Writer, Epoch: f.Epoch}
+			by[f.Writer] = w
+			order = append(order, f.Writer)
+		}
+		w.Count++
+		if !slices.Contains(w.Receivers, f.Receiver) {
+			w.Receivers = append(w.Receivers, f.Receiver)
+		}
+		if f.At.After(w.Last) {
+			w.Last = f.At
+		}
 	}
+	r.list = kept
+	out := make([]WriterRefusals, 0, len(order))
+	for _, writer := range order {
+		w := by[writer]
+		sort.Strings(w.Receivers)
+		out = append(out, *w)
+	}
+	sort.SliceStable(out, func(i, j int) bool { return out[i].Count > out[j].Count })
 	return out
 }
 
 // holding is whether this process holds the controller's consumer of what nodes say: the controller
-// acting, not one standing by for another (issue 213). Until the lease (to-be 45 §6) it is how a
-// watchdog knows it is the one that hears.
+// acting, not one standing by for another (issue 213). Beside the lease (to-be 45 §6), which decides
+// who may act, it is how a watchdog knows this process is the one that hears.
 var holding atomic.Bool
 
 // Holding says this process is the controller acting now.

internal/lint/emptyonerror.go

@@ -0,0 +1,237 @@
+// Package lint holds the checks this repository runs over its own source (novox/hq to-be 45 Phase 2).
+//
+// **Empty on error** (ADR 0227 rule 4, "nothing is dropped silently"): a reader that cannot read and
+// answers an empty collection is a reader whose caller cannot tell *nothing is there* from *I could not
+// tell* — issue 241's unreadable contributions file read as "nobody asks", and seven databases were
+// dropped. EmptyOnError finds every error branch that answers an empty collection with no error; the
+// test beside it fails the build on each one that does not say, where it does it, why the empty answer
+// is the truth (a `// empty-on-error: <why>` comment on the return or the line above).
+package lint
+
+import (
+	"fmt"
+	"go/ast"
+	"go/parser"
+	"go/token"
+	"io/fs"
+	"path/filepath"
+	"sort"
+	"strings"
+)
+
+// Allow is the comment that says, at the return, why an empty answer to an error is the truth.
+const Allow = "empty-on-error:"
+
+// Finding is one error branch answering an empty collection.
+type Finding struct {
+	File string
+	Line int
+	Func string
+	// Allowed is the reason given where it was allowed; empty for a finding that is a failure.
+	Allowed string
+}
+
+func (f Finding) String() string { return fmt.Sprintf("%s:%d (%s)", f.File, f.Line, f.Func) }
+
+// EmptyOnError walks every Go file under root but tests, vendor/ and testdata/, and answers every
+// return inside an `if <err> != nil` branch that answers an empty collection — nil, or a literal with no
+// elements, where the function answers a slice or a map — and no error: a nil error, or none declared.
+func EmptyOnError(root string) ([]Finding, error) {
+	var out []Finding
+	fset := token.NewFileSet()
+	err := filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error {
+		if err != nil {
+			return err
+		}
+		if d.IsDir() {
+			switch d.Name() {
+			case "vendor", "testdata", ".git", "node_modules":
+				return filepath.SkipDir
+			}
+			return nil
+		}
+		if !strings.HasSuffix(path, ".go") || strings.HasSuffix(path, "_test.go") {
+			return nil
+		}
+		file, err := parser.ParseFile(fset, path, nil, parser.ParseComments)
+		if err != nil {
+			return fmt.Errorf("%s cannot be read: %w", path, err)
+		}
+		rel, _ := filepath.Rel(root, path)
+		out = append(out, inFile(fset, file, rel)...)
+		return nil
+	})
+	sort.Slice(out, func(i, j int) bool {
+		if out[i].File != out[j].File {
+			return out[i].File < out[j].File
+		}
+		return out[i].Line < out[j].Line
+	})
+	return out, err
+}
+
+// inFile is every finding in one file.
+func inFile(fset *token.FileSet, file *ast.File, rel string) []Finding {
+	// Every comment's text by the line it ends on, so an allowance is read on the return's line or the
+	// line above it.
+	comments := map[int]string{}
+	for _, group := range file.Comments {
+		for _, c := range group.List {
+			comments[fset.Position(c.End()).Line] += c.Text
+		}
+	}
+	var out []Finding
+	var walk func(name string, results *ast.FieldList, body *ast.BlockStmt)
+	walk = func(name string, results *ast.FieldList, body *ast.BlockStmt) {
+		if body == nil {
+			return
+		}
+		ast.Inspect(body, func(n ast.Node) bool {
+			switch n := n.(type) {
+			case *ast.FuncLit:
+				// Its returns are its own, judged against its own results.
+				walk(name+" (a function inside it)", n.Type.Results, n.Body)
+				return false
+			case *ast.IfStmt:
+				if !errNotNil(n.Cond) {
+					return true
+				}
+				for _, ret := range returnsIn(n.Body) {
+					if !emptyOnError(results, ret) {
+						continue
+					}
+					line := fset.Position(ret.Pos()).Line
+					f := Finding{File: rel, Line: line, Func: name}
+					for _, l := range []int{line, line - 1} {
+						if text, ok := comments[l]; ok {
+							if _, why, found := strings.Cut(text, Allow); found && strings.TrimSpace(why) != "" {
+								f.Allowed = strings.TrimSpace(why)
+							}
+						}
+					}
+					out = append(out, f)
+				}
+			}
+			return true
+		})
+	}
+	for _, decl := range file.Decls {
+		fn, ok := decl.(*ast.FuncDecl)
+		if !ok {
+			continue
+		}
+		name := fn.Name.Name
+		if fn.Recv != nil && len(fn.Recv.List) > 0 {
+			name = "(" + exprString(fn.Recv.List[0].Type) + ")." + name
+		}
+		walk(name, fn.Type.Results, fn.Body)
+	}
+	return out
+}
+
+// errNotNil is a condition `<x> != nil` where x is an error by its name: err, or one ending in Err.
+func errNotNil(cond ast.Expr) bool {
+	found := false
+	ast.Inspect(cond, func(n ast.Node) bool {
+		b, ok := n.(*ast.BinaryExpr)
+		if !ok || b.Op != token.NEQ {
+			return true
+		}
+		x, xok := b.X.(*ast.Ident)
+		y, yok := b.Y.(*ast.Ident)
+		if xok && yok && y.Name == "nil" && (x.Name == "err" || strings.HasSuffix(x.Name, "Err")) {
+			found = true
+		}
+		return !found
+	})
+	return found
+}
+
+// returnsIn is the returns of a branch, not those of a function literal inside it.
+func returnsIn(body *ast.BlockStmt) []*ast.ReturnStmt {
+	var out []*ast.ReturnStmt
+	ast.Inspect(body, func(n ast.Node) bool {
+		switch n := n.(type) {
+		case *ast.FuncLit:
+			return false
+		case *ast.ReturnStmt:
+			out = append(out, n)
+		}
+		return true
+	})
+	return out
+}
+
+// emptyOnError says a return answers an empty collection and no error, for a function's results.
+func emptyOnError(results *ast.FieldList, ret *ast.ReturnStmt) bool {
+	if results == nil {
+		return false
+	}
+	var types []ast.Expr
+	for _, f := range results.List {
+		n := len(f.Names)
+		if n == 0 {
+			n = 1
+		}
+		for i := 0; i < n; i++ {
+			types = append(types, f.Type)
+		}
+	}
+	if len(ret.Results) != len(types) {
+		return false // a bare return of named results, or a call answering them: not judged here
+	}
+	empty := false
+	for i, t := range types {
+		value := ret.Results[i]
+		if isError(t) {
+			if !isNil(value) {
+				return false // an error is answered: said, not dropped
+			}
+			continue
+		}
+		if isCollection(t) && isEmpty(value) {
+			empty = true
+		}
+	}
+	return empty
+}
+
+func isError(t ast.Expr) bool {
+	id, ok := t.(*ast.Ident)
+	return ok && id.Name == "error"
+}
+
+func isCollection(t ast.Expr) bool {
+	switch t := t.(type) {
+	case *ast.ArrayType:
+		return t.Len == nil
+	case *ast.MapType:
+		return true
+	}
+	return false
+}
+
+func isNil(e ast.Expr) bool {
+	id, ok := e.(*ast.Ident)
+	return ok && id.Name == "nil"
+}
+
+func isEmpty(e ast.Expr) bool {
+	if isNil(e) {
+		return true
+	}
+	lit, ok := e.(*ast.CompositeLit)
+	return ok && len(lit.Elts) == 0
+}
+
+func exprString(e ast.Expr) string {
+	switch e := e.(type) {
+	case *ast.StarExpr:
+		return "*" + exprString(e.X)
+	case *ast.Ident:
+		return e.Name
+	case *ast.IndexExpr:
+		return exprString(e.X)
+	}
+	return "?"
+}

internal/lint/emptyonerror_test.go

@@ -0,0 +1,100 @@
+package lint
+
+import (
+	"go/parser"
+	"go/token"
+	"strings"
+	"testing"
+)
+
+// The lint, over this repository (novox/hq to-be 45 Phase 2, ADR 0227 rule 4 "how it is checked"): every
+// error branch that answers an empty collection says why that is the truth, or the build fails naming it.
+func TestNoReaderAnswersEmptyForAnError(t *testing.T) {
+	found, err := EmptyOnError("../..")
+	if err != nil {
+		t.Fatal(err)
+	}
+	allowed := 0
+	for _, f := range found {
+		if f.Allowed != "" {
+			allowed++
+			continue
+		}
+		t.Errorf("%s answers an empty collection and no error when it could not read: refuse by name "+
+			"(return the error), or say at the return why empty is the truth (// %s <why>)", f, Allow)
+	}
+	t.Logf("%d error branch(es) answer empty, each saying why", allowed)
+}
+
+// The lint itself: what it finds, and what it does not.
+func TestTheLintFindsEmptyOnError(t *testing.T) {
+	const src = `package x
+
+func readContributions() []string {
+	raw, err := read()
+	if err != nil {
+		return nil
+	}
+	return parse(raw)
+}
+
+func consumers() ([]string, error) {
+	if err := load(); err != nil {
+		return []string{}, nil
+	}
+	return nil, nil
+}
+
+func byName() (map[string]int, error) {
+	if err := load(); err != nil {
+		return nil, err
+	}
+	return map[string]int{}, nil
+}
+
+func allowed() ([]string, error) {
+	if err := load(); err != nil {
+		// empty-on-error: a store not yet seeded holds no seats, and the caller keeps its defaults
+		return nil, nil
+	}
+	return nil, nil
+}
+
+func notFound() ([]string, error) {
+	if errors.Is(err, ErrNoRows) {
+		return nil, nil
+	}
+	return nil, nil
+}
+
+func inside() {
+	f := func() []int {
+		if readErr != nil {
+			return nil
+		}
+		return []int{1}
+	}
+	_ = f
+}
+`
+	fset := token.NewFileSet()
+	file, err := parser.ParseFile(fset, "x.go", src, parser.ParseComments)
+	if err != nil {
+		t.Fatal(err)
+	}
+	var failing, allowedFns []string
+	for _, f := range inFile(fset, file, "x.go") {
+		if f.Allowed != "" {
+			allowedFns = append(allowedFns, f.Func)
+			continue
+		}
+		failing = append(failing, f.Func)
+	}
+	want := "readContributions consumers inside (a function inside it)"
+	if strings.Join(failing, " ") != want {
+		t.Fatalf("the lint found %q, want %q", failing, want)
+	}
+	if strings.Join(allowedFns, " ") != "allowed" {
+		t.Fatalf("the allowance was not read: %q", allowedFns)
+	}
+}