Act under a lease, keep accounts by order, one writer at composition (hq to-be 45 Phase 2)

Two controllers could both act (issue 204), a reconcile's report could
overtake the apply after it and the digest decided (issue 267), and a grant
could make a second writer of a machine's report.

- The lease (internal/lease, ADR 0229): mesh-controller_lease key `holder`,
  15 s age, renewed every 5 s by compare-and-set; the epoch is the revision
  it was taken at. The gate is the clock (stops 3 s before expiry); a refused
  renewal is a loss and the process exits; a holder that stops gives it back.
  serve takes it before asserting the bus. Epochs kept in the store
  (migration 0068 controller_epoch) as a floor: a bucket raised from nothing
  is compacted past it. Unleased (no epoch, S12 urgent) only when nobody
  holds it and the bus will not let it be written. A shell command acts
  under the holder's epoch, or its own lease when none.
- Declarations carry `epoch` inside the signed envelope, only to a machine
  whose latest account carried a report_sequence (mesh-host #35); would-send
  is composed with the epoch last sent. Allot and the send both pass the gate.
- Reports: contract in internal/link/order.go (epoch, sequence,
  report_sequence, older_than, refused_older). Accounts kept by epoch, then
  sequence, then report sequence; older refused, counted; unordered reports
  keep the digest rule. Plans by compare-and-set on a revision, with epoch.
  Conditions and calls carry the epoch and are not written off the lease.
- S12 and S13 (naming the writer by epoch) watched, D5 run; reset of the
  bucket said. Writers table compiled in and enforced in PermissionsFor; the
  controller no longer publishes mesh.control.>. A contract per consumed
  kind, and the empty-on-error lint over the repository.
- mesh-host pinned to its main with the epoch in the validator (D1 validates
  the envelope as sent).

Needs mesh-host's genesis lock with the lease grant (mesh-host PR) for
TestTheInstallersFirstUserListIsWhatTheControllerWouldCompose.

internal/link/calls.go

@@ -78,6 +78,8 @@ type Call struct {
 	// Holder is the controller process that served it, so one starting can tell its own running
 	// calls from those a stopped one left.
 	Holder string `json:"holder,omitempty"`
+	// Epoch is the controller lease epoch it was served under (novox/hq to-be 45 §6); zero for none.
+	Epoch uint64 `json:"epoch,omitempty"`
 
 	reply string // the subject the answer went to, which the bus names when it refuses it
 }
@@ -111,6 +113,18 @@ type CallLog struct {
 	logger  *log.Logger
 	lost    int // writes the keeper could not take, said once each
 	keepErr error
+	// epoch is the lease this process serves under (UnderLease): a call carries its epoch, and its
+	// record is written only while the lease is held. Nil writes every record with no epoch.
+	epoch func() (uint64, error)
+}
+
+// UnderLease makes every call carry the controller lease's epoch, and every write of a call's record
+// pass the lease (novox/hq to-be 45 §6): a controller that lost the lease writes no finish over a call
+// the next holder has marked abandoned — that mark is the record's last word.
+func (l *CallLog) UnderLease(epoch func() (uint64, error)) {
+	l.mu.Lock()
+	defer l.mu.Unlock()
+	l.epoch = epoch
 }
 
 // Calls is this process's log: one holder process serves its seats on one connection.
@@ -150,9 +164,9 @@ func (l *CallLog) Durably(ctx context.Context, keeper CallKeeper, holder string,
 			return fmt.Errorf("marking %s abandoned: %w", c.ID, err)
 		}
 		if logger != nil {
-			logger.Printf("%s (%s.%s, asked %s by %s) was running under %s, which stopped: marked abandoned — "+
-				"it may have done part of what it was asked, and nothing will finish it", c.ID, c.Seat, c.Verb,
-				c.Started.Format(time.RFC3339), orSomebody(c.Caller), orSomebody(c.Holder))
+			logger.Printf("%s (%s.%s, asked %s by %s) was running under %s (epoch %d), which stopped: marked "+
+				"abandoned — it may have done part of what it was asked, and nothing will finish it", c.ID, c.Seat,
+				c.Verb, c.Started.Format(time.RFC3339), orSomebody(c.Caller), orSomebody(c.Holder), c.Epoch)
 		}
 	}
 	return nil
@@ -184,6 +198,18 @@ func (l *CallLog) keep(c Call) {
 
 func (l *CallLog) keepWrites() {
 	for c := range l.writes {
+		l.mu.Lock()
+		gate := l.epoch
+		l.mu.Unlock()
+		if gate != nil {
+			if _, err := gate(); err != nil {
+				if l.logger != nil {
+					l.logger.Printf("%s (%s.%s, %s) is not kept on the bus: %v — the controller holding the lease "+
+						"marks it abandoned", c.ID, c.Seat, c.Verb, c.State, err)
+				}
+				continue
+			}
+		}
 		var err error
 		for try := 0; try < keepTries; try++ {
 			ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
@@ -237,6 +263,9 @@ func (l *CallLog) begin(seat, verb string, args json.RawMessage, reply string) *
 	c := &Call{ID: "call-" + strconv.FormatInt(l.now().UnixNano(), 10) + "-" + strconv.FormatUint(l.next, 10),
 		Seat: seat, Verb: verb, Args: args, Started: l.now(), State: CallRunning, reply: reply,
 		Caller: callerOf(reply), Holder: l.holder}
+	if l.epoch != nil {
+		c.Epoch, _ = l.epoch() // zero when not held: its record is then not written either
+	}
 	l.calls = append(l.calls, c)
 	if len(l.calls) > KeptCalls {
 		l.calls = l.calls[len(l.calls)-KeptCalls:]

internal/link/contracts.go

@@ -0,0 +1,47 @@
+package link
+
+// The contract of every message kind the controller consumes (novox/hq to-be 45 Phase 2, ADR 0227 rule
+// 2 "how it is checked": a contract test per consumed message kind in the receiver's repository, and a
+// check listing every consumed subject against the tests that name it).
+//
+// **Each kind says how an older one is told from a newer, and the tests that deliver n, then n−1.** A
+// kind that carries no order says why none is needed — a word whose newest is simply the latest heard,
+// a request answered once. The test beside it fails a kind the controller can be handed without an
+// entry here, and an entry naming a test that does not exist.
+
+// Contract is one consumed kind's order.
+type Contract struct {
+	// Ordered is how an older message of this kind is refused; empty for a kind that carries no order.
+	Ordered string
+	// Unordered is why a kind needs no order; empty for an ordered one.
+	Unordered string
+	// Tests are the tests that deliver the newer and then the older, and assert the older refused.
+	Tests []string
+}
+
+// Contracts are every kind the controller consumes, by kind.
+var Contracts = map[string]Contract{
+	KindReport: {Ordered: "by the epoch and sequence of the declaration it is about, then the node-engine's " +
+		"report sequence (order.go); an older account is refused and counted. A report from a node-engine " +
+		"that orders nothing is judged by the digest it names (issue 267)",
+		Tests: []string{"TestAnAccountOfAnOlderDeclarationIsRefusedByItsSequence",
+			"TestAnOlderReportOfTheSameDeclarationIsRefused", "TestAnAccountOfAnOlderDeclarationDoesNotReplaceTheNewer",
+			"TestAnAccountIsOlderByEpochThenSequenceThenReportSequence"}},
+	KindBuilt: {Ordered: "by the build's ask: an older ask finishing later is recorded and not registered (issue 219)",
+		Tests: []string{"TestAnOlderBuildHeardLaterDoesNotReplaceTheNewer"}},
+	KindSourceMoved: {Ordered: "by the merge's commit against what was built from it: a merge already acted on " +
+		"or older than the last look is history, not a second plan (issues 250, 266)",
+		Tests: []string{"TestAMergeOlderThanTheLastLookIsHistory", "TestAMergeMatchesTheSourcesBuiltFromIt"}},
+	KindHeartbeat: {Unordered: "a word that the machine is there: the newest heard is the newest said, and one " +
+		"lost is the next one"},
+	KindToolsHeartbeat: {Unordered: "a word that the node tools are there, as a machine's heartbeat"},
+	KindEnrolment: {Unordered: "a request answered once, under a token spent once: a second presentation is " +
+		"refused by the token, not by an order (issue 083)",
+		Tests: []string{"TestAnEnrolmentMetByAHeldTokenIsAskedToTryAgain"}},
+	KindModuleMoved: {Unordered: "the catalogue saying a module's current build moved: acted on by reading the " +
+		"catalogue's record, which is the order, so a late one reads the same record"},
+	KindCatchUp: {Unordered: "a catalogue asking what it missed: answered from the record, whenever asked"},
+	KindProvisioner: {Unordered: "a provider's newest word about a consumer, said again every fifteen minutes " +
+		"while it holds (ADR 0224): the condition keeps the last observed, and S8 says when the words stop",
+		Tests: []string{"TestAProviderFailingAConsumerBreaksAllWellUntilItRecovers"}},
+}

internal/link/contracts_test.go

@@ -0,0 +1,84 @@
+package link
+
+import (
+	"go/parser"
+	"go/token"
+	"io/fs"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/novox/mesh-controller/internal/broker"
+)
+
+// Every subject the controller consumes is a kind with a contract, and every test a contract names
+// exists (novox/hq to-be 45 Phase 2, ADR 0227 rule 2).
+func TestEveryConsumedKindHasAContract(t *testing.T) {
+	// What the controller can be handed, from the subjects it is granted and the ones it derives.
+	subjects := []string{EnrolSubject, BuiltSubject, ReportSubject("anchor"), AliveSubject("anchor"),
+		ToolsAliveSubject("anchor"), "mesh.mod.postgres.event.provisioner.failing"}
+	subjects = append(subjects, broker.ControllerFollows...)
+	kinds := map[string]bool{}
+	for _, s := range subjects {
+		kind, known := kindOfSubject(s)
+		if !known {
+			if strings.Contains(s, "*") {
+				continue // a pattern among the follows; its concrete subject is listed above
+			}
+			t.Errorf("the controller follows %s and has no kind for it", s)
+			continue
+		}
+		kinds[kind] = true
+	}
+	for kind := range kinds {
+		c, ok := Contracts[kind]
+		switch {
+		case !ok:
+			t.Errorf("the controller consumes %s and no contract says how an older one is told from a newer", kind)
+		case (c.Ordered == "") == (c.Unordered == ""):
+			t.Errorf("%s's contract says neither, or both, how it is ordered and why it need not be: %+v", kind, c)
+		case c.Ordered != "" && len(c.Tests) == 0:
+			t.Errorf("%s is ordered and no test delivers the newer and then the older", kind)
+		}
+	}
+	for kind := range Contracts {
+		if !kinds[kind] {
+			t.Errorf("a contract for %s, which the controller does not consume", kind)
+		}
+	}
+
+	// Every test named exists in this repository.
+	exists := map[string]bool{}
+	fset := token.NewFileSet()
+	err := filepath.WalkDir("../..", func(path string, d fs.DirEntry, err error) error {
+		if err != nil {
+			return err
+		}
+		if d.IsDir() && (d.Name() == "vendor" || d.Name() == ".git") {
+			return filepath.SkipDir
+		}
+		if d.IsDir() || !strings.HasSuffix(path, "_test.go") {
+			return nil
+		}
+		f, err := parser.ParseFile(fset, path, nil, 0)
+		if err != nil {
+			return err
+		}
+		for name, obj := range f.Scope.Objects {
+			if obj.Kind.String() == "func" && strings.HasPrefix(name, "Test") {
+				exists[name] = true
+			}
+		}
+		return nil
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	for kind, c := range Contracts {
+		for _, name := range c.Tests {
+			if !exists[name] {
+				t.Errorf("%s's contract names %s, which no test in this repository is", kind, name)
+			}
+		}
+	}
+}

internal/link/declare.go

@@ -12,6 +12,10 @@ type Signer interface {
 	Sign(ctx context.Context, message []byte) ([]byte, error)
 }
 
+// ActingGate is what every send passes before it is made (novox/hq to-be 45 §6): whether this process
+// may act under the controller's lease. Set by the controller; nil passes every send — a test, a tool.
+var ActingGate func(ctx context.Context) error
+
 // Declare sends a node what it should be, signed.
 //
 // The signature is over the declaration exactly as it is published — the same bytes the node
@@ -25,6 +29,12 @@ func Declare(ctx context.Context, bus Bus, signer Signer, node string,
 	if !json.Valid(declaration) {
 		return fmt.Errorf("refusing to send %s something that is not a declaration", node)
 	}
+	// **At the send, not only where it was composed**: a lease lost between the two stops this one.
+	if ActingGate != nil {
+		if err := ActingGate(ctx); err != nil {
+			return fmt.Errorf("%s was not sent its declaration: %w", node, err)
+		}
+	}
 
 	signature, err := signer.Sign(ctx, declaration)
 	if err != nil {

internal/link/enrolment.go

@@ -11,6 +11,7 @@ import (
 	"fmt"
 	"log"
 	"sort"
+	"time"
 
 	"github.com/novox/mesh-controller/internal/broker"
 	"github.com/novox/mesh-controller/internal/identity"
@@ -421,23 +422,54 @@ func (e Enrolment) Heard(ctx context.Context, report Report) (news bool, err err
 	// the newer one arrived reported *after* the newer apply's report, and stored last, it read as the
 	// machine never having applied what it was sent — the plan waited until somebody pushed by hand.
 	// One row per node means the last write wins, so the order of arrival must not decide it.
-	if report.Declared != "" {
-		sent, err := e.Inventory.Outstanding(ctx, report.Node)
+	//
+	// **By order, from a node-engine that orders its reports** (novox/hq to-be 45 §6, the contract in
+	// order.go): the account is kept only if it is not older than the one kept — by the epoch and
+	// sequence of the declaration it is about, then the node-engine's own report sequence — and refused
+	// otherwise, said and counted (rule 2). One rule, whatever the account is about and whenever it
+	// arrives; the digest the mesh last sent decides nothing for it. **By digest, from an older
+	// node-engine**: its report claims no order, and the rule of issue 267 stands for it.
+	//
+	// And whether the node-engine reads an epoch in a declaration, from every account it gives: the
+	// mesh sends one only to a machine that said so, and one rolled back says so no longer.
+	if err := e.Inventory.RecordReadsEpoch(ctx, node.ID, report.ReadsEpoch()); err != nil {
+		return false, err
+	}
+	if report.Ordered() {
+		account := AccountOf(report)
+		news, err = e.Inventory.RecordOrderedDoing(ctx, node.ID, doing,
+			inventory.ReportOrder{Epoch: report.Epoch, Sequence: report.Sequence, ReportSequence: report.ReportSequence},
+			func(kept inventory.ReportOrder) bool {
+				return account.OlderThan(Account{Order: Order{Epoch: kept.Epoch, Sequence: kept.Sequence},
+					ReportSequence: kept.ReportSequence})
+			})
+		if errors.Is(err, inventory.ErrOlderAccount) {
+			log.Printf("kept what %s says about the machine, and refused its account of declaration %s (%s, report %d): "+
+				"the account kept is newer", report.Node, short(report.Declared), report.Order.Words(), report.ReportSequence)
+			StaleRefusals.Refused(Refusal{Writer: WriterNodeEngine(report.Node), Receiver: "controller", At: time.Now()})
+			return false, e.Inventory.Seen(ctx, node.ID)
+		}
 		if err != nil {
 			return false, err
 		}
-		if Superseded(report.Declared, sent) {
-			log.Printf("kept what %s says about the machine, and not its account of declaration %s: "+
-				"the mesh has sent it %s since", report.Node, short(report.Declared), short(sent))
-			return false, e.Inventory.Seen(ctx, node.ID)
+	} else {
+		if report.Declared != "" {
+			sent, err := e.Inventory.Outstanding(ctx, report.Node)
+			if err != nil {
+				return false, err
+			}
+			if Superseded(report.Declared, sent) {
+				log.Printf("kept what %s says about the machine, and not its account of declaration %s: "+
+					"the mesh has sent it %s since", report.Node, short(report.Declared), short(sent))
+				return false, e.Inventory.Seen(ctx, node.ID)
+			}
+		}
+		// **Whether this is news** is the store's answer: it holds the previous report, and a machine
+		// that reconciles every minute says the same thing until something changes (novox/hq ADR 0134).
+		news, err = e.Inventory.RecordDoing(ctx, node.ID, doing)
+		if err != nil {
+			return false, err
 		}
-	}
-
-	// **Whether this is news** is the store's answer: it holds the previous report, and a machine
-	// that reconciles every minute says the same thing until something changes (novox/hq ADR 0134).
-	news, err = e.Inventory.RecordDoing(ctx, node.ID, doing)
-	if err != nil {
-		return false, err
 	}
 	// And how long the machine took, from the send to this first account of it (novox/hq to-be 45
 	// Phase 0): what the sent-not-reported bound will be set from. Said if lost, never a failure.

internal/link/heard_order_test.go

@@ -0,0 +1,163 @@
+package link_test
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	"github.com/novox/mesh-controller/internal/inventory"
+	"github.com/novox/mesh-controller/internal/link"
+)
+
+// A report kept by its order (novox/hq to-be 45 §6, ADR 0227 rule 2): the contract test of the report,
+// the message kind the controller consumes from every machine. Deliver n, then n−1: refused and counted.
+// Of the same declaration, report r then r−1: refused. The digest the mesh last sent decides nothing
+// for an ordered report; an unordered one, from an older node-engine, is judged by it as before.
+
+func anOrderedMachine(t *testing.T) (*inventory.Inventory, link.Enrolment, string) {
+	t.Helper()
+	inv := inventory.ForTest(t)
+	node, err := inv.AddNode(context.Background(), "home-server")
+	if err != nil {
+		t.Fatal(err)
+	}
+	return inv, link.Enrolment{Inventory: inv}, node.ID
+}
+
+func ordered(declared string, epoch, sequence, report int64, applied ...string) link.Report {
+	return link.Report{Node: "home-server", Declared: declared, Applied: applied,
+		Order: link.Order{Epoch: epoch, Sequence: sequence}, ReportSequence: report}
+}
+
+// Issue 267, ordered: a reconcile's account of declaration 11 reaches the mesh after the apply's of 12.
+func TestAnAccountOfAnOlderDeclarationIsRefusedByItsSequence(t *testing.T) {
+	inv, heard, id := anOrderedMachine(t)
+	ctx := context.Background()
+	before := countFor(link.WriterNodeEngine("home-server"))
+	if _, err := heard.Heard(ctx, ordered("d12", 57, 12, 41, "a", "b")); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := heard.Heard(ctx, ordered("d11", 57, 11, 40, "a")); err != nil {
+		t.Fatal(err)
+	}
+	doing, _, err := inv.DoingOf(ctx, "home-server")
+	if err != nil || doing.Declared != "d12" || doing.Applied != 2 {
+		t.Fatalf("the older account replaced the newer: %+v (%v)", doing, err)
+	}
+	if got := countFor(link.WriterNodeEngine("home-server")) - before; got != 1 {
+		t.Fatalf("the refusal was counted %d times, want once", got)
+	}
+	kept, err := inv.KeptOrder(ctx, id)
+	if err != nil || kept != (inventory.ReportOrder{Epoch: 57, Sequence: 12, ReportSequence: 41}) {
+		t.Fatalf("the order kept is %+v (%v)", kept, err)
+	}
+}
+
+func TestAnOlderReportOfTheSameDeclarationIsRefused(t *testing.T) {
+	inv, heard, _ := anOrderedMachine(t)
+	ctx := context.Background()
+	if _, err := heard.Heard(ctx, ordered("d12", 57, 12, 41, "a", "b")); err != nil {
+		t.Fatal(err)
+	}
+	failed := ordered("d12", 57, 12, 40)
+	failed.Failed = map[string]string{"b": "it failed a moment before it applied"}
+	if _, err := heard.Heard(ctx, failed); err != nil {
+		t.Fatal(err)
+	}
+	doing, _, _ := inv.DoingOf(ctx, "home-server")
+	if doing.Outcome != inventory.OutcomeApplied {
+		t.Fatalf("an older report of the same declaration replaced the newer: %+v", doing)
+	}
+	// A newer report of it — the next reconcile — is kept.
+	again := ordered("d12", 57, 12, 42)
+	again.Failed = map[string]string{"b": "it failed since"}
+	if _, err := heard.Heard(ctx, again); err != nil {
+		t.Fatal(err)
+	}
+	if doing, _, _ := inv.DoingOf(ctx, "home-server"); doing.Outcome != inventory.OutcomeFailed {
+		t.Fatalf("a newer report of the same declaration was not kept: %+v", doing)
+	}
+}
+
+// An ordered account is judged by its order, not by the digest the mesh last sent: the account of 12
+// is kept although the mesh has sent 13 since — it is still the newest account of the machine.
+func TestAnOrderedAccountIsNotJudgedByTheDigestSent(t *testing.T) {
+	inv, heard, id := anOrderedMachine(t)
+	ctx := context.Background()
+	if err := inv.RecordSent(ctx, id, "d13", nil); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := heard.Heard(ctx, ordered("d12", 57, 12, 41, "a")); err != nil {
+		t.Fatal(err)
+	}
+	if doing, said, _ := inv.DoingOf(ctx, "home-server"); !said || doing.Declared != "d12" {
+		t.Fatalf("the newest account the machine gave was set aside by the digest: %+v", doing)
+	}
+	// And the machine reads an epoch: it orders its reports.
+	if reads, err := inv.ReadsEpoch(ctx, id); err != nil || !reads {
+		t.Fatalf("a machine whose reports carry a report sequence is not recorded as reading an epoch: %v", err)
+	}
+}
+
+// A node-engine rolled back to one that orders nothing: its account is taken by the digest rule, the
+// order kept is cleared, and it is no longer sent an epoch.
+func TestAnUnorderedAccountClearsTheOrderAndTheEpoch(t *testing.T) {
+	inv, heard, id := anOrderedMachine(t)
+	ctx := context.Background()
+	if _, err := heard.Heard(ctx, ordered("d12", 57, 12, 41, "a")); err != nil {
+		t.Fatal(err)
+	}
+	if err := inv.RecordSent(ctx, id, "d13", nil); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := heard.Heard(ctx, link.Report{Node: "home-server", Declared: "d13", Applied: []string{"a", "b"}}); err != nil {
+		t.Fatal(err)
+	}
+	if doing, _, _ := inv.DoingOf(ctx, "home-server"); doing.Declared != "d13" {
+		t.Fatalf("an older node-engine's account of what was sent was not kept: %+v", doing)
+	}
+	if kept, _ := inv.KeptOrder(ctx, id); kept != (inventory.ReportOrder{}) {
+		t.Fatalf("an unordered account left the order kept: %+v", kept)
+	}
+	if reads, _ := inv.ReadsEpoch(ctx, id); reads {
+		t.Fatal("a node-engine that orders nothing is still sent an epoch")
+	}
+	// The next ordered account, whatever its numbers, has nothing to be older than.
+	if _, err := heard.Heard(ctx, ordered("d14", 58, 14, 1, "a")); err != nil {
+		t.Fatal(err)
+	}
+	if doing, _, _ := inv.DoingOf(ctx, "home-server"); doing.Declared != "d14" {
+		t.Fatalf("the first ordered account after an unordered one was refused: %+v", doing)
+	}
+}
+
+// A stale refusal names its writer by the epoch the refused declaration claimed, and a refusal whose
+// own report was lost is still counted from the machine's own tally.
+func TestAStaleRefusalIsCountedByItsWriter(t *testing.T) {
+	count := link.NewRefusalCount()
+	now := time.Now()
+	refusal := link.Report{Node: "anchor", Refused: "older", Order: link.Order{Epoch: 41, Sequence: 11},
+		OlderThan: &link.Order{Epoch: 57, Sequence: 12}, ReportSequence: 9, RefusedOlder: 3}
+	if !refusal.StaleRefusalOf() {
+		t.Fatal("a refusal naming the order it holds is not stale")
+	}
+	count.Lifetime("anchor", 2, now) // the machine's tally, as the controller first heard it
+	count.Refused(link.Refusal{Writer: link.WriterEpoch(refusal.Epoch), Epoch: refusal.Epoch, Receiver: "anchor", At: now})
+	count.Lifetime("anchor", refusal.RefusedOlder, now)
+	count.Lifetime("anchor", 5, now) // two more refused, their reports lost
+	got := count.Within(now.Add(-time.Minute))
+	if len(got) != 2 || got[0].Count != 2 || got[0].Epoch != 0 || got[1].Writer != "the controller of epoch 41" ||
+		got[1].Count != 1 || got[1].Receivers[0] != "anchor" {
+		t.Fatalf("the refusals by writer are %+v", got)
+	}
+}
+
+// countFor is how many refusals of a writer this process has heard in the last minute.
+func countFor(writer string) int {
+	for _, w := range link.StaleRefusals.Within(time.Now().Add(-time.Minute)) {
+		if w.Writer == writer {
+			return w.Count
+		}
+	}
+	return 0
+}

internal/link/order.go

@@ -0,0 +1,118 @@
+package link
+
+import "strconv"
+
+// The order a declaration carries, and the order a report about one carries back (novox/hq to-be 45
+// §6, ADR 0227 rule 2).
+//
+// **This file is the controller's side of the contract with the node-engine for Phase 2**, and the one
+// place it is written here. It mirrors mesh-host internal/link/messages.go (`Order`, and the report's
+// `report_sequence`, `older_than`, `refused_older`) field for field and rule for rule, as Report has
+// always mirrored the host's report: a key added on one side and not the other is a key the other side
+// does not know exists.
+//
+// **The wire.**
+//
+//   - A declaration's order is two top-level keys of its signed envelope, beside "declaration" and
+//     "resources": `sequence` (since issue 107) and `epoch` (new). Inside the signed bytes, so a message
+//     cannot be given a newer order than the controller gave it.
+//   - A report carries the order of the declaration it is about as the same two top-level keys, beside
+//     "declared"; `report_sequence`, the node-engine's own number for the report; `older_than`, on a
+//     refusal of a declaration older than one it applied, the order of the one it holds; and
+//     `refused_older`, how many it has refused so, ever, on every report.
+//
+// Every key is optional and absent when zero, and zero is "no order claimed", never "first". So each
+// side reads the other's older shape:
+//
+//   - **An older node-engine with this controller.** It decodes a declaration strictly and refuses a key
+//     it does not know, whole — so `epoch` is sent only to a machine whose latest report carried a
+//     `report_sequence`, which a node-engine that reads the epoch always does. Until then it is sent the
+//     sequence alone, as today; its reports carry no report sequence and are judged by the digest they
+//     name, as today (issue 267).
+//   - **A newer node-engine with an older controller.** It is sent no epoch, which claims none: it
+//     refuses nothing by epoch. The keys it adds to a report are fields the older controller ignores.
+//   - **A controller rolled back to a build without the lease** sends no epoch again and is not refused
+//     for it: a node-engine refuses by epoch only when both declarations claim one. That gives up the
+//     epoch's protection while such a build runs — the price of a rollback that cannot strand every
+//     machine.
+//
+// **Epoch** is the controller lease's epoch (to-be 45 §6): the lease bucket's revision at which the
+// instance that composed the declaration took the lease. It only grows: a later holder's is higher, and
+// the controller never issues one at or under the highest it has issued (internal/lease, the floor).
+// **Sequence** is the declaration's number for that machine, one higher every send, taken from the
+// controller's store before the declaration is composed (issues 107, 204).
+
+// Order is where a declaration stands among everything the mesh has sent a machine: the lease epoch it
+// was sent under and its sequence. Zero in either claims none.
+type Order struct {
+	Epoch    int64 `json:"epoch,omitempty"`
+	Sequence int64 `json:"sequence,omitempty"`
+}
+
+// Older is the node-engine's rule, as mesh-host states it: a declaration of this order is older than
+// one of order `than` the machine applied **only when both claim an epoch** — and then when its epoch
+// is lower, or its epoch is the same and its sequence lower (both claimed). A higher epoch is a new lease
+// holder and is never older, whatever its sequence. Here so the controller's tests state what the
+// machines will do with what it sends.
+func (o Order) Older(than Order) bool {
+	if o.Epoch <= 0 || than.Epoch <= 0 {
+		return false
+	}
+	if o.Epoch != than.Epoch {
+		return o.Epoch < than.Epoch
+	}
+	return o.Sequence > 0 && than.Sequence > 0 && o.Sequence < than.Sequence
+}
+
+// Words is an order as a person reads it in a log line. Not String: Report embeds Order, and a Stringer
+// promoted onto every report would print each one as its order alone.
+func (o Order) Words() string {
+	switch {
+	case o.Epoch > 0:
+		return "epoch " + strconv.FormatInt(o.Epoch, 10) + ", sequence " + strconv.FormatInt(o.Sequence, 10)
+	case o.Sequence > 0:
+		return "sequence " + strconv.FormatInt(o.Sequence, 10) + ", no epoch"
+	}
+	return "no order"
+}
+
+// Account is the order of one account of a machine — a report about a declaration: that declaration's
+// order and the node-engine's own number for the report.
+type Account struct {
+	Order
+	ReportSequence int64
+}
+
+// AccountOf is a report's account order.
+func AccountOf(r Report) Account { return Account{Order: r.Order, ReportSequence: r.ReportSequence} }
+
+// Ordered says a report comes from a node-engine that orders its reports: it carries a report sequence.
+// Such a report is judged by its order (OlderThan); one without is judged by the digest it names, as
+// before (issue 267).
+func (r Report) Ordered() bool { return r.ReportSequence > 0 }
+
+// ReadsEpoch says the node-engine that made the report takes an epoch in a declaration: one that orders
+// its reports does (mesh-host: "its presence also says this host reads a declaration's epoch").
+func (r Report) ReadsEpoch() bool { return r.ReportSequence > 0 }
+
+// StaleRefusalOf says a report refuses a declaration older than one the machine applied: the node-engine
+// names the order it holds (`older_than`), or, from a node-engine older than that, says so in the words
+// of its sequence refusal.
+func (r Report) StaleRefusalOf() bool {
+	return r.OlderThan != nil || (r.Refused != "" && IsStaleRefusal(r.Refused))
+}
+
+// OlderThan is the controller's rule (to-be 45 §6): whether this account is older than the one kept for
+// the machine — **by epoch, then sequence, then report sequence**, each compared only where both claim
+// one. An account of a declaration of an older epoch is refused; of an older sequence, refused; an older
+// report of the same declaration (a reconcile's, overtaken by the apply's — issue 267), refused. Equal is
+// the same report again, redelivered, and is not refused.
+func (a Account) OlderThan(kept Account) bool {
+	if a.Epoch > 0 && kept.Epoch > 0 && a.Epoch != kept.Epoch {
+		return a.Epoch < kept.Epoch
+	}
+	if a.Sequence > 0 && kept.Sequence > 0 && a.Sequence != kept.Sequence {
+		return a.Sequence < kept.Sequence
+	}
+	return a.ReportSequence > 0 && kept.ReportSequence > 0 && a.ReportSequence < kept.ReportSequence
+}

internal/link/order_test.go

@@ -0,0 +1,113 @@
+package link
+
+import (
+	"encoding/json"
+	"testing"
+)
+
+// The contract of order.go, case by case: what a node-engine refuses of a declaration (the rule
+// mesh-host states, restated so the controller's tests say what the machines do with what it sends),
+// what the controller refuses of a report, and the bytes on the wire both ways.
+
+func TestADeclarationIsOlderOnlyWhenBothClaimAnEpoch(t *testing.T) {
+	for _, c := range []struct {
+		name     string
+		arriving Order
+		held     Order
+		older    bool
+	}{
+		{"a newer sequence of the same epoch", Order{57, 12}, Order{57, 11}, false},
+		{"the same declaration again (a reconcile)", Order{57, 12}, Order{57, 12}, false},
+		{"a lower sequence of the same epoch", Order{57, 11}, Order{57, 12}, true},
+		// Issue 204: a controller that lost its lease goes on sending.
+		{"an older epoch, whatever its sequence", Order{41, 99}, Order{57, 12}, true},
+		{"a newer epoch, whatever its sequence", Order{57, 3}, Order{41, 99}, false},
+		{"no epoch arriving: a rolled-back controller is not stranded", Order{0, 11}, Order{57, 12}, false},
+		{"no epoch held: what the machine held before any lease", Order{57, 11}, Order{0, 12}, false},
+		{"no order at all", Order{}, Order{57, 12}, false},
+	} {
+		t.Run(c.name, func(t *testing.T) {
+			if got := c.arriving.Older(c.held); got != c.older {
+				t.Fatalf("%s against %s: older %v, want %v", c.arriving.Words(), c.held.Words(), got, c.older)
+			}
+		})
+	}
+}
+
+func TestAnAccountIsOlderByEpochThenSequenceThenReportSequence(t *testing.T) {
+	for _, c := range []struct {
+		name     string
+		arriving Account
+		kept     Account
+		older    bool
+	}{
+		// Issue 267: a reconcile's account of declaration 11 arrives after the apply's of 12.
+		{"an account of an older declaration", Account{Order{57, 11}, 40}, Account{Order{57, 12}, 41}, true},
+		{"an older report of the same declaration", Account{Order{57, 12}, 40}, Account{Order{57, 12}, 41}, true},
+		{"a newer report of the same declaration (a reconcile after the apply)",
+			Account{Order{57, 12}, 42}, Account{Order{57, 12}, 41}, false},
+		{"the same report again (redelivered)", Account{Order{57, 12}, 41}, Account{Order{57, 12}, 41}, false},
+		{"an account of a newer declaration, whatever its report sequence",
+			Account{Order{57, 13}, 1}, Account{Order{57, 12}, 41}, false},
+		{"an account of an older epoch", Account{Order{41, 99}, 50}, Account{Order{57, 12}, 41}, true},
+		{"an account of a newer epoch", Account{Order{58, 1}, 2}, Account{Order{57, 12}, 41}, false},
+		{"no epoch either side: the sequences", Account{Order{0, 11}, 50}, Account{Order{0, 12}, 41}, true},
+		{"no report sequence: the declaration's alone", Account{Order{57, 12}, 0}, Account{Order{57, 12}, 41}, false},
+		{"an unordered declaration: the report sequences", Account{Order{}, 40}, Account{Order{}, 41}, true},
+		{"nothing kept yet", Account{Order{57, 3}, 1}, Account{}, false},
+	} {
+		t.Run(c.name, func(t *testing.T) {
+			if got := c.arriving.OlderThan(c.kept); got != c.older {
+				t.Fatalf("%+v against %+v: older %v, want %v", c.arriving, c.kept, got, c.older)
+			}
+		})
+	}
+}
+
+// The bytes, as mesh-host's report writes them: every key optional, absent when zero.
+func TestTheOrderOnTheWire(t *testing.T) {
+	// A stale refusal, as the node-engine says it: the refused declaration's order at the top, the one
+	// it holds in older_than, and how many it has refused ever.
+	raw := []byte(`{"node":"anchor","refused":"older","declared":"d1","epoch":41,"sequence":11,` +
+		`"report_sequence":7,"older_than":{"epoch":57,"sequence":12},"refused_older":3}`)
+	var r Report
+	if err := json.Unmarshal(raw, &r); err != nil {
+		t.Fatal(err)
+	}
+	if r.Order != (Order{41, 11}) || r.ReportSequence != 7 || r.OlderThan == nil || *r.OlderThan != (Order{57, 12}) ||
+		r.RefusedOlder != 3 {
+		t.Fatalf("a node-engine's stale refusal reads as %+v", r)
+	}
+	if !r.Ordered() || !r.ReadsEpoch() || !r.StaleRefusalOf() {
+		t.Fatalf("a node-engine that orders its reports reads as one that does not: %+v", r)
+	}
+
+	// A report from a node-engine older than the contract says none of it.
+	var older Report
+	if err := json.Unmarshal([]byte(`{"node":"anchor","applied":["a"],"declared":"d1"}`), &older); err != nil {
+		t.Fatal(err)
+	}
+	if older.Ordered() || older.ReadsEpoch() || older.StaleRefusalOf() {
+		t.Fatalf("an older node-engine's report reads as ordered: %+v", older)
+	}
+	// Its stale refusal, in the words it has always used, is still one.
+	older.Refused = "this declaration " + StaleRefusal + ": it is sequence 3"
+	if !older.StaleRefusalOf() {
+		t.Fatal("an older node-engine's refusal of an older sequence does not read as stale")
+	}
+
+	// And a report with no order puts no order key on the wire.
+	raw, err := json.Marshal(Report{Node: "anchor", Declared: "d1"})
+	if err != nil {
+		t.Fatal(err)
+	}
+	var keys map[string]any
+	if err := json.Unmarshal(raw, &keys); err != nil {
+		t.Fatal(err)
+	}
+	for _, key := range []string{"epoch", "sequence", "report_sequence", "older_than", "refused_older"} {
+		if _, there := keys[key]; there {
+			t.Fatalf("an unordered report carries %s: %s", key, raw)
+		}
+	}
+}

internal/link/protocol.go

@@ -178,6 +178,23 @@ type Report struct {
 	// the same way, as the `sent` digest the mesh recorded. Which declaration, not when.
 	Declared string `json:"declared,omitempty"`
 
+	// Order is where the declaration this report is about stands — its `epoch` and `sequence` as the
+	// declaration carried them — so the mesh keeps accounts by what they are about rather than by when
+	// they arrived (novox/hq to-be 45 §6; the contract is order.go). Two top-level keys; absent for a
+	// declaration that claimed no order, and from a node-engine older than the contract.
+	Order
+	// ReportSequence is the node-engine's own number for this report: one higher for every report it
+	// makes, kept on disk across restarts and self-updates. Zero claims none — every report an older
+	// node-engine makes. Its presence also says the node-engine reads a declaration's epoch.
+	ReportSequence int64 `json:"report_sequence,omitempty"`
+	// OlderThan is set on a report refusing a declaration older than one the machine applied: the order
+	// of the one it holds. The refused declaration is the report's own Declared and Order — whose epoch
+	// names the controller that sent it (S13).
+	OlderThan *Order `json:"older_than,omitempty"`
+	// RefusedOlder is how many declarations the node-engine has refused as older, ever, on every report:
+	// a refusal whose own report was lost is still counted from the next.
+	RefusedOlder int64 `json:"refused_older,omitempty"`
+
 	// Held is what an adopted node found and is keeping as it was until its module is taken
 	// (novox/hq ADR 0100). Without it an adopted node reads as converged.
 	Held []Held `json:"held,omitempty"`

internal/link/serve.go

@@ -317,6 +317,11 @@ func (s *Server) reported(ctx context.Context, m Control) {
 		// the server's, it comes back after the newer was applied whatever the controller does.
 		outstanding := s.outstanding(ctx, report.Node)
 		declaredIn := staleAgainst(report)
+		if report.Ordered() {
+			// Kept by its order, not by the digest the mesh last sent (novox/hq to-be 45 §6): the
+			// listener refuses an older account under the one rule, whatever it is about.
+			declaredIn = ""
+		}
 		if Superseded(declaredIn, outstanding) {
 			s.log.Printf("set aside %s: the mesh has moved past that declaration", what)
 			_ = m.Took()

internal/link/watched.go

@@ -5,6 +5,8 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"slices"
+	"sort"
 	"strings"
 	"sync"
 	"sync/atomic"
@@ -176,48 +178,130 @@ const StaleRefusal = "is older than what the mesh last said to this node"
 // IsStaleRefusal says a report's refusal is the node-engine refusing a declaration older than it holds.
 func IsStaleRefusal(refused string) bool { return strings.Contains(refused, StaleRefusal) }
 
-// RefusalCount keeps when each machine refused a stale declaration, for S13 (novox/hq to-be 45 §3):
-// more than five from one writer in five minutes is a writer sending what it has moved past.
+// A Refusal is one receiver refusing something older than what it holds (novox/hq to-be 45 §6, ADR 0227
+// rule 2): a machine refusing a declaration, or this controller refusing an account.
+type Refusal struct {
+	// Writer is who sent what was refused, in the mesh's words: WriterEpoch for a declaration, a
+	// machine's node-engine for an account.
+	Writer string
+	// Epoch is the lease epoch the refused declaration claimed, for a controller to be named by; zero
+	// for a declaration that claimed none, and for an account.
+	Epoch int64
+	// Receiver is the machine, or "controller", that refused it.
+	Receiver string
+	At       time.Time
+}
+
+// WriterEpoch is how a refused declaration's writer is said before it is named: by the epoch it
+// claimed, or as a controller that claimed none.
+func WriterEpoch(epoch int64) string {
+	if epoch > 0 {
+		return fmt.Sprintf("the controller of epoch %d", epoch)
+	}
+	return "a controller that claimed no epoch"
+}
+
+// WriterNodeEngine is a machine's node-engine as the writer of an account the controller refused.
+func WriterNodeEngine(node string) string { return "the node-engine on " + node }
+
+// RefusalCount keeps every stale refusal heard, for S13 (novox/hq to-be 45 §3): more than five from
+// one writer in five minutes is a writer sending what it — or the mesh — has moved past.
 type RefusalCount struct {
-	mu  sync.Mutex
-	per map[string][]time.Time
+	mu   sync.Mutex
+	list []Refusal
+	// lifetime is each machine's own count of declarations it refused as older, as its last report
+	// said it (`refused_older`), and named how many of those this process heard as refusal reports
+	// since: what the count rose by beyond them is refusals whose reports never arrived.
+	lifetime map[string]int64
+	named    map[string]int64
 }
 
 // StaleRefusals is this process's count.
-var StaleRefusals = &RefusalCount{per: map[string][]time.Time{}}
+var StaleRefusals = NewRefusalCount()
 
-// Refused records one refusal by a machine.
-func (r *RefusalCount) Refused(node string, at time.Time) {
-	r.mu.Lock()
-	defer r.mu.Unlock()
-	r.per[node] = append(r.per[node], at)
+// NewRefusalCount is an empty count.
+func NewRefusalCount() *RefusalCount {
+	return &RefusalCount{lifetime: map[string]int64{}, named: map[string]int64{}}
 }
 
-// Within is how many refusals each machine made since a moment; older ones are forgotten.
-func (r *RefusalCount) Within(since time.Time) map[string]int {
+// Refused records one refusal.
+func (r *RefusalCount) Refused(f Refusal) {
 	r.mu.Lock()
 	defer r.mu.Unlock()
-	out := map[string]int{}
-	for node, times := range r.per {
-		var kept []time.Time
-		for _, t := range times {
-			if !t.Before(since) {
-				kept = append(kept, t)
-			}
-		}
-		if len(kept) == 0 {
-			delete(r.per, node)
+	r.list = append(r.list, f)
+	if f.Writer != WriterNodeEngine(f.Receiver) {
+		r.named[f.Receiver]++
+	}
+}
+
+// Lifetime reads a machine's own count of declarations it refused as older, from any report: what it
+// rose by since the last, beyond the refusals heard by name, is recorded as refused by a writer the
+// mesh never heard named — the refusal's report was lost, and the count is still a fact.
+func (r *RefusalCount) Lifetime(node string, total int64, at time.Time) {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	before, known := r.lifetime[node]
+	r.lifetime[node] = total
+	named := r.named[node]
+	r.named[node] = 0
+	if !known || total <= before {
+		return // the first word since this controller started, or a node-engine that started over
+	}
+	for missing := total - before - named; missing > 0; missing-- {
+		r.list = append(r.list, Refusal{Writer: "a writer whose refused declaration was never reported",
+			Receiver: node, At: at})
+	}
+}
+
+// WriterRefusals is one writer's refusals within a window.
+type WriterRefusals struct {
+	Writer    string
+	Epoch     int64
+	Count     int
+	Receivers []string
+	Last      time.Time
+}
+
+// Within is every writer's refusals since a moment, most first; older ones are forgotten.
+func (r *RefusalCount) Within(since time.Time) []WriterRefusals {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	kept := r.list[:0]
+	by := map[string]*WriterRefusals{}
+	var order []string
+	for _, f := range r.list {
+		if f.At.Before(since) {
 			continue
 		}
-		r.per[node] = kept
-		out[node] = len(kept)
+		kept = append(kept, f)
+		w, ok := by[f.Writer]
+		if !ok {
+			w = &WriterRefusals{Writer: f.Writer, Epoch: f.Epoch}
+			by[f.Writer] = w
+			order = append(order, f.Writer)
+		}
+		w.Count++
+		if !slices.Contains(w.Receivers, f.Receiver) {
+			w.Receivers = append(w.Receivers, f.Receiver)
+		}
+		if f.At.After(w.Last) {
+			w.Last = f.At
+		}
 	}
+	r.list = kept
+	out := make([]WriterRefusals, 0, len(order))
+	for _, writer := range order {
+		w := by[writer]
+		sort.Strings(w.Receivers)
+		out = append(out, *w)
+	}
+	sort.SliceStable(out, func(i, j int) bool { return out[i].Count > out[j].Count })
 	return out
 }
 
 // holding is whether this process holds the controller's consumer of what nodes say: the controller
-// acting, not one standing by for another (issue 213). Until the lease (to-be 45 §6) it is how a
-// watchdog knows it is the one that hears.
+// acting, not one standing by for another (issue 213). Beside the lease (to-be 45 §6), which decides
+// who may act, it is how a watchdog knows this process is the one that hears.
 var holding atomic.Bool
 
 // Holding says this process is the controller acting now.