Act under a lease, keep accounts by order, one writer at composition (hq to-be 45 Phase 2)

Two controllers could both act (issue 204), a reconcile's report could
overtake the apply after it and the digest decided (issue 267), and a grant
could make a second writer of a machine's report.

- The lease (internal/lease, ADR 0229): mesh-controller_lease key `holder`,
  15 s age, renewed every 5 s by compare-and-set; the epoch is the revision
  it was taken at. The gate is the clock (stops 3 s before expiry); a refused
  renewal is a loss and the process exits; a holder that stops gives it back.
  serve takes it before asserting the bus. Epochs kept in the store
  (migration 0068 controller_epoch) as a floor: a bucket raised from nothing
  is compacted past it. Unleased (no epoch, S12 urgent) only when nobody
  holds it and the bus will not let it be written. A shell command acts
  under the holder's epoch, or its own lease when none.
- Declarations carry `epoch` inside the signed envelope, only to a machine
  whose latest account carried a report_sequence (mesh-host #35); would-send
  is composed with the epoch last sent. Allot and the send both pass the gate.
- Reports: contract in internal/link/order.go (epoch, sequence,
  report_sequence, older_than, refused_older). Accounts kept by epoch, then
  sequence, then report sequence; older refused, counted; unordered reports
  keep the digest rule. Plans by compare-and-set on a revision, with epoch.
  Conditions and calls carry the epoch and are not written off the lease.
- S12 and S13 (naming the writer by epoch) watched, D5 run; reset of the
  bucket said. Writers table compiled in and enforced in PermissionsFor; the
  controller no longer publishes mesh.control.>. A contract per consumed
  kind, and the empty-on-error lint over the repository.
- mesh-host pinned to its main with the epoch in the validator (D1 validates
  the envelope as sent).

Needs mesh-host's genesis lock with the lease grant (mesh-host PR) for
TestTheInstallersFirstUserListIsWhatTheControllerWouldCompose.

internal/inventory/durations_test.go

@@ -68,25 +68,25 @@ func TestAPlansTierIsMeasuredWhenItIsLeft(t *testing.T) {
 	p := Plan{ID: "plan-1", Repository: "novox/mesh-tools", Commit: "abc", Created: time.Now().UTC(),
 		State: PlanBuilding, Tiers: [][]string{{"mesh-tools"}, {"builder"}},
 		Modules: map[string]*PlanModule{"mesh-tools": {}, "builder": {}}}
-	if err := inv.SavePlan(ctx, p); err != nil {
+	if err := inv.SavePlan(ctx, &p); err != nil {
 		t.Fatal(err)
 	}
 	p.State = PlanRolling // the same tier, saved again
-	if err := inv.SavePlan(ctx, p); err != nil {
+	if err := inv.SavePlan(ctx, &p); err != nil {
 		t.Fatal(err)
 	}
 	if ds, _ := inv.Durations(ctx, DurationPlanTier, time.Now().Add(-time.Hour)); len(ds) != 0 {
 		t.Fatalf("a tier not left was measured: %+v", ds)
 	}
 	p.Tier = 1
-	if err := inv.SavePlan(ctx, p); err != nil {
+	if err := inv.SavePlan(ctx, &p); err != nil {
 		t.Fatal(err)
 	}
 	p.State = PlanDone
-	if err := inv.SavePlan(ctx, p); err != nil {
+	if err := inv.SavePlan(ctx, &p); err != nil {
 		t.Fatal(err)
 	}
-	if err := inv.SavePlan(ctx, p); err != nil { // saved again once ended: nothing more to measure
+	if err := inv.SavePlan(ctx, &p); err != nil { // saved again once ended: nothing more to measure
 		t.Fatal(err)
 	}
 	ds, err := inv.Durations(ctx, DurationPlanTier, time.Now().Add(-time.Hour))

internal/inventory/migrations/0068-order-and-one-writer.sql

@@ -0,0 +1,40 @@
+-- Order and one writer (novox/hq to-be 45 §6, Phase 2).
+--
+-- The controller acts only while it holds its lease, and every act carries the lease's epoch: a
+-- declaration, a plan write. A report carries the order of the declaration it accounts for, and the
+-- controller keeps the highest per machine, refusing an older account rather than letting the last
+-- one written win (issue 267). These are the records each of those needs.
+
+-- Every epoch the mesh issued: which controller instance held it, from when, and how it ended —
+-- given back, or lost (its key expired, or a renewal was refused). The highest one is the floor no
+-- new epoch may be at or under, even when the lease's bucket was raised again from nothing; and a
+-- stale refusal names its writer from here.
+create table controller_epoch (
+    epoch    bigint      primary key,
+    instance text        not null,
+    host     text        not null default '',
+    build    text        not null default '',
+    taken    timestamptz not null default now(),
+    ended    timestamptz,
+    -- how: 'released' (given back), 'lost' (its own renewal was refused or failed), 'expired'
+    -- (found expired by the next holder: it stopped renewing without saying so).
+    how      text        not null default ''
+);
+
+-- The epoch a machine was last sent, so what the mesh WOULD send is composed with it and reads as
+-- byte for byte what it DID send when nothing else changed — a new holder's epoch is not a change
+-- of the machine. Null for a declaration sent without one.
+alter table node add column sent_epoch bigint;
+-- Whether the machine's node-engine said it reads an epoch in a declaration (its report's `reads`):
+-- until it has, it is sent none, because an older node-engine refuses a key it does not know, whole.
+alter table node add column reads_epoch boolean not null default false;
+
+-- The order of the account kept for each machine: the declaration's sequence and epoch it is about
+-- and the node-engine's own report sequence. Null where the kept account carried none.
+alter table node_report add column reported_sequence bigint;
+alter table node_report add column reported_epoch bigint;
+alter table node_report add column report_sequence bigint;
+
+-- A plan is written by compare-and-set on its revision, and says the epoch that wrote it last.
+alter table release_plan add column revision bigint not null default 0;
+alter table release_plan add column epoch bigint;

internal/inventory/nodes.go

@@ -14,12 +14,17 @@ import (
 	"time"
 
 	"github.com/jackc/pgx/v5"
+	"github.com/jackc/pgx/v5/pgconn"
 	"github.com/novox/mesh-controller/internal/broker"
 	"github.com/novox/mesh-controller/internal/store"
 )
 
 // Inventory is this context, holding the store it exclusively owns.
-type Inventory struct{ store *store.Store }
+type Inventory struct {
+	store *store.Store
+	// acting is the gate a write that acts passes (ActsUnder, novox/hq to-be 45 §6); nil passes all.
+	acting func(ctx context.Context) (uint64, error)
+}
 
 // Open connects to the inventory store.
 func Open(ctx context.Context) (*Inventory, error) {
@@ -763,14 +768,35 @@ func sameFailure(a, b Doing) bool {
 // ADR 0134). A machine reconciles continuously and reports each time; the same outcome about the same
 // declaration is the same state said again, and a fact per report would be a fact per minute per
 // machine that tells nobody anything. Read here because the previous row is read here anyway.
+//
+// **An account that claims no order clears the order kept** (novox/hq to-be 45 §6): the account kept is
+// then one the next ordered report has nothing to compare against, and it is taken as it was before
+// reports carried an order. RecordOrderedDoing keeps an ordered one.
 func (i *Inventory) RecordDoing(ctx context.Context, node string, d Doing) (news bool, err error) {
+	news, err = recordDoing(ctx, i.store.Pool(), node, d)
+	if err != nil {
+		return false, err
+	}
+	_, err = i.store.Pool().Exec(ctx,
+		`update node_report set reported_sequence = null, reported_epoch = null, report_sequence = null
+		  where node = $1`, node)
+	return news, err
+}
+
+// queries is what recordDoing writes through: the pool, or a transaction an ordered account holds.
+type queries interface {
+	QueryRow(ctx context.Context, sql string, args ...any) pgx.Row
+	Exec(ctx context.Context, sql string, args ...any) (pgconn.CommandTag, error)
+}
+
+func recordDoing(ctx context.Context, q queries, node string, d Doing) (news bool, err error) {
 	failed, err := json.Marshal(d.Failed)
 	if err != nil {
 		return false, err
 	}
 	var before Doing
 	var beforeFailed []byte
-	found := i.store.Pool().QueryRow(ctx,
+	found := q.QueryRow(ctx,
 		`select outcome, refused, failed, failing_since, failures, coalesce(declared,'')
 		   from node_report where node = $1`,
 		node).Scan(&before.Outcome, &before.Refused, &beforeFailed, &before.Since, &before.Times,
@@ -795,7 +821,7 @@ func (i *Inventory) RecordDoing(ctx context.Context, node string, d Doing) (news
 			since, times = before.Since, before.Times+1
 		}
 	}
-	_, err = i.store.Pool().Exec(ctx,
+	_, err = q.Exec(ctx,
 		`insert into node_report (node, outcome, refused, failed, applied, at, declared,
 		                          failing_since, failures)
 		 values ($1, $2, $3, $4, $5, now(), $6, $7, $8)
@@ -923,6 +949,18 @@ func (i *Inventory) LastReports(ctx context.Context) ([]Reported, error) {
 // is a push's to send to a machine it did not name. Nil records that it is not known, as for a
 // declaration sent by hand.
 func (i *Inventory) RecordSent(ctx context.Context, node, digest string, builds map[string]string) error {
+	return i.RecordSentUnder(ctx, node, digest, builds, 0)
+}
+
+// RecordSentUnder is RecordSent for a declaration that carried an epoch (novox/hq to-be 45 §6): kept
+// beside the digest, so what the mesh would send is composed with it. Zero is one that carried none.
+func (i *Inventory) RecordSentUnder(ctx context.Context, node, digest string, builds map[string]string,
+	epoch uint64) error {
+	var sentEpoch *int64
+	if epoch > 0 {
+		e := int64(epoch)
+		sentEpoch = &e
+	}
 	var carried *string
 	if builds != nil {
 		raw, err := json.Marshal(builds)
@@ -933,7 +971,8 @@ func (i *Inventory) RecordSent(ctx context.Context, node, digest string, builds
 		carried = &text
 	}
 	_, err := i.store.Pool().Exec(ctx,
-		`update node set sent = $2, sent_at = now(), sent_builds = $3::jsonb where id = $1`, node, digest, carried)
+		`update node set sent = $2, sent_at = now(), sent_builds = $3::jsonb, sent_epoch = $4 where id = $1`,
+		node, digest, carried, sentEpoch)
 	return err
 }
 

internal/inventory/order.go

@@ -0,0 +1,250 @@
+package inventory
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"time"
+
+	"github.com/jackc/pgx/v5"
+)
+
+// Order and one writer, as this context keeps them (novox/hq to-be 45 §6, Phase 2): the epochs the
+// mesh issued, the epoch each machine was sent, whether its node-engine reads one, and the order of
+// the account kept for it.
+
+// ActsUnder gives this inventory the gate every write that acts passes (a plan's): the epoch of the
+// controller lease this process acts under, or why it may not act. Nil — a test, a command reading —
+// writes with no epoch and refuses nothing.
+func (i *Inventory) ActsUnder(epoch func(ctx context.Context) (uint64, error)) { i.acting = epoch }
+
+// actingEpoch is the epoch a write carries, nil when there is no gate or it claims none.
+func (i *Inventory) actingEpoch(ctx context.Context) (*int64, error) {
+	if i.acting == nil {
+		return nil, nil
+	}
+	epoch, err := i.acting(ctx)
+	if err != nil {
+		return nil, err
+	}
+	if epoch == 0 {
+		return nil, nil
+	}
+	e := int64(epoch)
+	return &e, nil
+}
+
+// Epoch is one epoch the mesh issued.
+type Epoch struct {
+	Epoch    uint64
+	Instance string
+	Host     string
+	Build    string
+	Taken    time.Time
+	// Ended is when it ended, nil while it is held; How is how: EpochReleased, EpochLost, EpochExpired.
+	Ended *time.Time
+	How   string
+}
+
+// How an epoch ends.
+const (
+	// EpochReleased is a holder that gave the lease back.
+	EpochReleased = "released"
+	// EpochLost is a holder whose own renewal was refused or failed, and which said so.
+	EpochLost = "lost"
+	// EpochExpired is a holder the next one found gone: it stopped renewing without saying anything.
+	EpochExpired = "expired"
+)
+
+// HighestEpoch is the highest epoch the mesh has issued, zero before the first: the floor no new one
+// may be at or under.
+func (i *Inventory) HighestEpoch(ctx context.Context) (uint64, error) {
+	var highest int64
+	if err := i.store.Pool().QueryRow(ctx, `select coalesce(max(epoch), 0) from controller_epoch`).Scan(&highest); err != nil {
+		return 0, fmt.Errorf("reading the highest epoch issued: %w", err)
+	}
+	return uint64(highest), nil
+}
+
+// TookEpoch records an epoch taken, and ends every earlier one still open as found expired: the lease
+// was free to take, so whoever held it last neither holds it nor said it let go. Answers the epochs it
+// ended that way, newest first.
+func (i *Inventory) TookEpoch(ctx context.Context, e Epoch) ([]Epoch, error) {
+	tx, err := i.store.Pool().Begin(ctx)
+	if err != nil {
+		return nil, err
+	}
+	defer func() { _ = tx.Rollback(ctx) }()
+	rows, err := tx.Query(ctx,
+		`update controller_epoch set ended = now(), how = $2
+		  where ended is null and epoch < $1
+		  returning epoch, instance, host, build, taken, ended, how`, int64(e.Epoch), EpochExpired)
+	if err != nil {
+		return nil, err
+	}
+	expired, err := scanEpochs(rows)
+	if err != nil {
+		return nil, err
+	}
+	if _, err := tx.Exec(ctx,
+		`insert into controller_epoch (epoch, instance, host, build, taken) values ($1, $2, $3, $4, $5)
+		 on conflict (epoch) do nothing`,
+		int64(e.Epoch), e.Instance, e.Host, e.Build, e.Taken); err != nil {
+		return nil, err
+	}
+	return expired, tx.Commit(ctx)
+}
+
+// EndEpoch records how an epoch ended. One already ended keeps its first word.
+func (i *Inventory) EndEpoch(ctx context.Context, epoch uint64, how string) error {
+	_, err := i.store.Pool().Exec(ctx,
+		`update controller_epoch set ended = now(), how = $2 where epoch = $1 and ended is null`, int64(epoch), how)
+	return err
+}
+
+// EpochOf is the epoch issued at a number; false when the mesh issued none there.
+func (i *Inventory) EpochOf(ctx context.Context, epoch uint64) (Epoch, bool, error) {
+	rows, err := i.store.Pool().Query(ctx,
+		`select epoch, instance, host, build, taken, ended, how from controller_epoch where epoch = $1`, int64(epoch))
+	if err != nil {
+		return Epoch{}, false, err
+	}
+	found, err := scanEpochs(rows)
+	if err != nil || len(found) == 0 {
+		return Epoch{}, false, err
+	}
+	return found[0], true, nil
+}
+
+// EpochsSince is every epoch taken or ended since a moment, newest first.
+func (i *Inventory) EpochsSince(ctx context.Context, since time.Time) ([]Epoch, error) {
+	rows, err := i.store.Pool().Query(ctx,
+		`select epoch, instance, host, build, taken, ended, how from controller_epoch
+		  where taken >= $1 or ended >= $1 or ended is null order by epoch desc`, since)
+	if err != nil {
+		return nil, err
+	}
+	return scanEpochs(rows)
+}
+
+func scanEpochs(rows pgx.Rows) ([]Epoch, error) {
+	defer rows.Close()
+	var out []Epoch
+	for rows.Next() {
+		var e Epoch
+		var epoch int64
+		if err := rows.Scan(&epoch, &e.Instance, &e.Host, &e.Build, &e.Taken, &e.Ended, &e.How); err != nil {
+			return nil, err
+		}
+		e.Epoch = uint64(epoch)
+		out = append(out, e)
+	}
+	return out, rows.Err()
+}
+
+// SentEpoch is the epoch a machine was last sent, by its id; zero for one sent without.
+func (i *Inventory) SentEpoch(ctx context.Context, id string) (uint64, error) {
+	var epoch *int64
+	if err := i.store.Pool().QueryRow(ctx, `select sent_epoch from node where id = $1`, id).Scan(&epoch); err != nil {
+		return 0, fmt.Errorf("reading the epoch %s was last sent: %w", id, err)
+	}
+	if epoch == nil {
+		return 0, nil
+	}
+	return uint64(*epoch), nil
+}
+
+// ReadsEpoch says a machine's node-engine said it reads an epoch in a declaration, by its id.
+func (i *Inventory) ReadsEpoch(ctx context.Context, id string) (bool, error) {
+	var reads bool
+	if err := i.store.Pool().QueryRow(ctx, `select reads_epoch from node where id = $1`, id).Scan(&reads); err != nil {
+		return false, fmt.Errorf("reading whether %s reads an epoch: %w", id, err)
+	}
+	return reads, nil
+}
+
+// RecordReadsEpoch keeps what a machine's latest report said of reading an epoch. Every report of a
+// node-engine that orders its reports says it, so a node-engine rolled back says it no longer does.
+func (i *Inventory) RecordReadsEpoch(ctx context.Context, id string, reads bool) error {
+	_, err := i.store.Pool().Exec(ctx, `update node set reads_epoch = $2 where id = $1`, id, reads)
+	return err
+}
+
+// ReportOrder is the order of an account: the declaration's epoch and sequence it is about, and the
+// node-engine's own report sequence. Zero in any claims none.
+type ReportOrder struct {
+	Epoch          int64
+	Sequence       int64
+	ReportSequence int64
+}
+
+// ErrOlderAccount is an account refused because the one kept is newer.
+var ErrOlderAccount = errors.New("an older account than the one kept")
+
+// KeptOrder is the order of the account kept for a machine, by its id; zero when it carried none.
+func (i *Inventory) KeptOrder(ctx context.Context, id string) (ReportOrder, error) {
+	o, err := keptOrder(ctx, i.store.Pool(), id, "")
+	if errors.Is(err, pgx.ErrNoRows) {
+		return ReportOrder{}, nil
+	}
+	return o, err
+}
+
+func keptOrder(ctx context.Context, q queries, id, lock string) (ReportOrder, error) {
+	var epoch, seq, rseq *int64
+	err := q.QueryRow(ctx,
+		`select reported_epoch, reported_sequence, report_sequence from node_report where node = $1`+lock, id).
+		Scan(&epoch, &seq, &rseq)
+	if err != nil {
+		return ReportOrder{}, err
+	}
+	var o ReportOrder
+	for _, f := range []struct {
+		from *int64
+		to   *int64
+	}{{epoch, &o.Epoch}, {seq, &o.Sequence}, {rseq, &o.ReportSequence}} {
+		if f.from != nil {
+			*f.to = *f.from
+		}
+	}
+	return o, nil
+}
+
+// RecordOrderedDoing is RecordDoing for an account that carries its order (novox/hq to-be 45 §6):
+// written only when it is not older than the account kept, as olderThan judges — the rule is the
+// link's (link.Account), stated once — and in one transaction with the row locked, so two accounts
+// arriving together cannot both win. ErrOlderAccount when it is older; nothing is written then.
+func (i *Inventory) RecordOrderedDoing(ctx context.Context, id string, d Doing, o ReportOrder,
+	olderThan func(kept ReportOrder) bool) (news bool, err error) {
+	tx, err := i.store.Pool().Begin(ctx)
+	if err != nil {
+		return false, err
+	}
+	defer func() { _ = tx.Rollback(ctx) }()
+	kept, err := keptOrder(ctx, tx, id, " for update")
+	switch {
+	case errors.Is(err, pgx.ErrNoRows):
+	case err != nil:
+		return false, err
+	case olderThan(kept):
+		return false, ErrOlderAccount
+	}
+	news, err = recordDoing(ctx, tx, id, d)
+	if err != nil {
+		return false, err
+	}
+	if _, err := tx.Exec(ctx,
+		`update node_report set reported_epoch = $2, reported_sequence = $3, report_sequence = $4 where node = $1`,
+		id, nullIfZero(o.Epoch), nullIfZero(o.Sequence), nullIfZero(o.ReportSequence)); err != nil {
+		return false, err
+	}
+	return news, tx.Commit(ctx)
+}
+
+// nullIfZero is a claimed order or none.
+func nullIfZero(n int64) *int64 {
+	if n == 0 {
+		return nil
+	}
+	return &n
+}

internal/inventory/order_test.go

@@ -0,0 +1,126 @@
+package inventory
+
+import (
+	"context"
+	"errors"
+	"testing"
+	"time"
+)
+
+// Order and one writer, as the store keeps them (novox/hq to-be 45 §6).
+
+// **A plan is written by compare-and-set on its revision, carrying the epoch**: a write against a plan
+// another writer moved since is refused, and nothing is written by a process that may not act.
+func TestAPlanIsWrittenByCompareAndSetUnderTheLease(t *testing.T) {
+	inv := ForTest(t)
+	ctx := context.Background()
+	epoch := uint64(57)
+	inv.ActsUnder(func(context.Context) (uint64, error) { return epoch, nil })
+
+	p := Plan{ID: "plan-cas", Repository: "novox/app", Commit: "c0ffee00", Created: time.Now(), State: PlanBuilding,
+		Tiers: [][]string{{"app"}}, Modules: map[string]*PlanModule{"app": {}}}
+	if err := inv.SavePlan(ctx, &p); err != nil {
+		t.Fatal(err)
+	}
+	if p.Revision != 1 || p.Epoch != 57 {
+		t.Fatalf("a new plan was written at revision %d, epoch %d", p.Revision, p.Epoch)
+	}
+	// Two readers of revision 1: the first write wins, the second is refused and writes nothing.
+	first, err := inv.PlanByID(ctx, "plan-cas")
+	if err != nil {
+		t.Fatal(err)
+	}
+	second := first
+	first.Note = "the newer word"
+	if err := inv.SavePlan(ctx, &first); err != nil {
+		t.Fatal(err)
+	}
+	second.State = PlanFailed
+	if err := inv.SavePlan(ctx, &second); !errors.Is(err, ErrPlanMoved) {
+		t.Fatalf("a write against a plan moved since it was read was not refused: %v", err)
+	}
+	kept, _ := inv.PlanByID(ctx, "plan-cas")
+	if kept.State != PlanBuilding || kept.Note != "the newer word" || kept.Revision != 2 {
+		t.Fatalf("the refused write reached the plan: %+v", kept)
+	}
+	// The writer saves its own plan again without reading it: its revision moved with its write.
+	first.Tier = 0
+	if err := inv.SavePlan(ctx, &first); err != nil {
+		t.Fatalf("a writer could not save its own plan twice: %v", err)
+	}
+	// A new plan under an id already written is refused, not laid over it.
+	again := Plan{ID: "plan-cas", Repository: "novox/app", Commit: "deadbeef", Created: time.Now(), State: PlanBuilding}
+	if err := inv.SavePlan(ctx, &again); !errors.Is(err, ErrPlanMoved) {
+		t.Fatalf("a second plan under one id was written: %v", err)
+	}
+	// And a process that does not hold the lease writes nothing.
+	inv.ActsUnder(func(context.Context) (uint64, error) { return 0, errors.New("this controller lost the lease") })
+	first.Note = "from a controller that lost the lease"
+	if err := inv.SavePlan(ctx, &first); err == nil {
+		t.Fatal("a plan was written by a controller that does not hold the lease")
+	}
+	if kept, _ := inv.PlanByID(ctx, "plan-cas"); kept.Note != "the newer word" || kept.Epoch != 57 {
+		t.Fatalf("a controller without the lease wrote the plan: %+v", kept)
+	}
+}
+
+// The epochs the mesh issued: the highest is the floor; taking one ends every earlier one nobody gave
+// back as found expired, and says which; one given back says so and is not said again.
+func TestTheEpochsIssuedAreKept(t *testing.T) {
+	inv := ForTest(t)
+	ctx := context.Background()
+	if highest, err := inv.HighestEpoch(ctx); err != nil || highest != 0 {
+		t.Fatalf("a mesh that issued none has %d (%v)", highest, err)
+	}
+	if _, err := inv.TookEpoch(ctx, Epoch{Epoch: 41, Instance: "a", Taken: time.Now()}); err != nil {
+		t.Fatal(err)
+	}
+	// a stopped renewing and said nothing; b takes the lease.
+	expired, err := inv.TookEpoch(ctx, Epoch{Epoch: 57, Instance: "b", Taken: time.Now()})
+	if err != nil || len(expired) != 1 || expired[0].Epoch != 41 || expired[0].How != EpochExpired {
+		t.Fatalf("taking 57 ended %+v (%v), want 41 found expired", expired, err)
+	}
+	if err := inv.EndEpoch(ctx, 57, EpochReleased); err != nil {
+		t.Fatal(err)
+	}
+	expired, err = inv.TookEpoch(ctx, Epoch{Epoch: 60, Instance: "c", Taken: time.Now()})
+	if err != nil || len(expired) != 0 {
+		t.Fatalf("a lease given back was found expired: %+v (%v)", expired, err)
+	}
+	if highest, _ := inv.HighestEpoch(ctx); highest != 60 {
+		t.Fatalf("the highest epoch issued is %d, want 60", highest)
+	}
+	e, found, err := inv.EpochOf(ctx, 57)
+	if err != nil || !found || e.Instance != "b" || e.How != EpochReleased || e.Ended == nil {
+		t.Fatalf("epoch 57 reads %+v (%v, %v)", e, found, err)
+	}
+	recent, err := inv.EpochsSince(ctx, time.Now().Add(-time.Hour))
+	if err != nil || len(recent) != 3 || recent[0].Epoch != 60 || recent[0].Ended != nil {
+		t.Fatalf("the epochs of the last hour read %+v (%v)", recent, err)
+	}
+}
+
+// What a machine was sent under, and whether it reads an epoch.
+func TestTheEpochAMachineWasSentIsKept(t *testing.T) {
+	inv := ForTest(t)
+	ctx := context.Background()
+	node, err := inv.AddNode(ctx, "anchor")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := inv.RecordSentUnder(ctx, node.ID, "d1", nil, 57); err != nil {
+		t.Fatal(err)
+	}
+	if e, err := inv.SentEpoch(ctx, node.ID); err != nil || e != 57 {
+		t.Fatalf("sent under %d (%v)", e, err)
+	}
+	if err := inv.RecordSent(ctx, node.ID, "d2", nil); err != nil {
+		t.Fatal(err)
+	}
+	if e, _ := inv.SentEpoch(ctx, node.ID); e != 0 {
+		t.Fatalf("a declaration sent without an epoch left %d kept", e)
+	}
+	if reads, _ := inv.ReadsEpoch(ctx, node.ID); reads {
+		t.Fatal("a machine that never said so reads an epoch")
+	}
+}

internal/inventory/plans.go

@@ -32,8 +32,17 @@ type Plan struct {
 	// never written from here: a save measures the tier it leaves and stamps the next. What the
 	// watchdog of a plan's progress (S3) reads.
 	TierEntered time.Time `json:"tier_entered,omitempty"`
+	// Revision is the plan's as it was read, and the one a save must find (novox/hq to-be 45 §6): a
+	// plan is written by compare-and-set, so a write against a plan another writer moved since is
+	// refused rather than laid over it. Zero is a plan never saved. SavePlan moves it.
+	Revision int64 `json:"revision"`
+	// Epoch is the controller lease epoch that wrote it last; zero for a write that claimed none.
+	Epoch uint64 `json:"epoch,omitempty"`
 }
 
+// ErrPlanMoved is a save against a plan written by somebody else since it was read.
+var ErrPlanMoved = errors.New("the plan was written by somebody else since it was read")
+
 // PlanModule is one module's state within a plan.
 type PlanModule struct {
 	// State: asked, built, failed; empty for a module whose tier has not been asked yet.
@@ -75,7 +84,16 @@ const (
 func (p Plan) Open() bool { return p.State == PlanBuilding || p.State == PlanRolling }
 
 // SavePlan writes a plan, new or changed, whole: the plan is small and read as one thing.
-func (i *Inventory) SavePlan(ctx context.Context, p Plan) error {
+//
+// **By compare-and-set on its revision, carrying the epoch** (novox/hq to-be 45 §6): written only if
+// the plan is still at the revision it was read at — a new one only if it does not exist — and refused
+// with ErrPlanMoved otherwise; and only by a process that may act (ActsUnder), whose epoch it records.
+// On success p's revision and epoch are the ones written, so the caller may save it again.
+func (i *Inventory) SavePlan(ctx context.Context, p *Plan) error {
+	epoch, err := i.actingEpoch(ctx)
+	if err != nil {
+		return fmt.Errorf("the plan for %s %s is not written: %w", p.Repository, p.Commit, err)
+	}
 	tiers, err := json.Marshal(p.Tiers)
 	if err != nil {
 		return err
@@ -92,21 +110,49 @@ func (i *Inventory) SavePlan(ctx context.Context, p Plan) error {
 		return err
 	}
 	defer func() { _ = tx.Rollback(ctx) }()
-	entered, err := planTierLeft(ctx, tx, p, time.Now())
+	entered, err := planTierLeft(ctx, tx, *p, time.Now())
 	if err != nil {
 		return err
 	}
-	_, err = tx.Exec(ctx,
-		`insert into release_plan (id, repository, commit_hash, created, updated, state, tier, tiers, modules, note, branch, tier_entered)
-		 values ($1, $2, $3, $4, now(), $5, $6, $7, $8, $9, $10, $11)
+	var revision int64
+	err = tx.QueryRow(ctx,
+		`insert into release_plan (id, repository, commit_hash, created, updated, state, tier, tiers, modules, note,
+		                           branch, tier_entered, revision, epoch)
+		 values ($1, $2, $3, $4, now(), $5, $6, $7, $8, $9, $10, $11, 1, $13)
 		 on conflict (id) do update set updated = now(), state = excluded.state, tier = excluded.tier,
 		        tiers = excluded.tiers, modules = excluded.modules, note = excluded.note, branch = excluded.branch,
-		        tier_entered = excluded.tier_entered`,
-		p.ID, p.Repository, p.Commit, p.Created, p.State, p.Tier, tiers, modules, p.Note, p.Branch, entered)
+		        tier_entered = excluded.tier_entered, revision = release_plan.revision + 1, epoch = excluded.epoch
+		  where release_plan.revision = $12
+		 returning revision`,
+		p.ID, p.Repository, p.Commit, p.Created, p.State, p.Tier, tiers, modules, p.Note, p.Branch, entered,
+		p.Revision, epoch).Scan(&revision)
+	if errors.Is(err, pgx.ErrNoRows) {
+		// The row is there and at another revision — moved since this was read, or there already
+		// when this one is new: either way not this writer's to overwrite. (A plan saved before plans
+		// had revisions is at zero, and its first save here is from a read at zero.)
+		return fmt.Errorf("the plan for %s %s (%s) is not written: %w", p.Repository, short(p.Commit), p.ID, ErrPlanMoved)
+	}
 	if err != nil {
 		return err
 	}
-	return tx.Commit(ctx)
+	if err := tx.Commit(ctx); err != nil {
+		return err
+	}
+	p.Revision, p.TierEntered = revision, entered
+	if epoch != nil {
+		p.Epoch = uint64(*epoch)
+	} else {
+		p.Epoch = 0
+	}
+	return nil
+}
+
+// short is a commit as a person reads it.
+func short(commit string) string {
+	if len(commit) > 8 {
+		return commit[:8]
+	}
+	return commit
 }
 
 // OpenPlans is every plan still being worked, oldest first.
@@ -134,7 +180,7 @@ func (i *Inventory) PlanByID(ctx context.Context, id string) (Plan, error) {
 func (i *Inventory) plans(ctx context.Context, tail string) ([]Plan, error) {
 	rows, err := i.store.Pool().Query(ctx,
 		`select id, repository, commit_hash, created, updated, state, tier, tiers, modules, note, branch,
-		        coalesce(tier_entered, created)
+		        coalesce(tier_entered, created), revision, coalesce(epoch, 0)
 		   from release_plan `+tail)
 	if err != nil {
 		return nil, err
@@ -144,10 +190,12 @@ func (i *Inventory) plans(ctx context.Context, tail string) ([]Plan, error) {
 	for rows.Next() {
 		var p Plan
 		var tiers, modules []byte
+		var epoch int64
 		if err := rows.Scan(&p.ID, &p.Repository, &p.Commit, &p.Created, &p.Updated, &p.State,
-			&p.Tier, &tiers, &modules, &p.Note, &p.Branch, &p.TierEntered); err != nil {
+			&p.Tier, &tiers, &modules, &p.Note, &p.Branch, &p.TierEntered, &p.Revision, &epoch); err != nil {
 			return nil, err
 		}
+		p.Epoch = uint64(epoch)
 		if err := json.Unmarshal(tiers, &p.Tiers); err != nil {
 			return nil, err
 		}

internal/inventory/plans_test.go

@@ -13,7 +13,7 @@ func TestAPlanIsKeptAdvancedAndResumedFromTheStore(t *testing.T) {
 	p := Plan{ID: "plan-1", Repository: "novox/mesh-tools", Commit: "abc", Created: time.Now().UTC(),
 		State: PlanBuilding, Tiers: [][]string{{"mesh-tools"}, {"builder"}, {"shop"}},
 		Modules: map[string]*PlanModule{"mesh-tools": {}, "builder": {}, "shop": {}}}
-	if err := inv.SavePlan(ctx, p); err != nil {
+	if err := inv.SavePlan(ctx, &p); err != nil {
 		t.Fatal(err)
 	}
 	open, err := inv.OpenPlans(ctx)
@@ -28,7 +28,7 @@ func TestAPlanIsKeptAdvancedAndResumedFromTheStore(t *testing.T) {
 	resumed.Modules["mesh-tools"].BuiltAt = &now
 	resumed.State = PlanRolling
 	resumed.Note = "tier 0 built; waiting for builder on anchor to be applied"
-	if err := inv.SavePlan(ctx, resumed); err != nil {
+	if err := inv.SavePlan(ctx, &resumed); err != nil {
 		t.Fatal(err)
 	}
 	again, err := inv.PlanByID(ctx, "plan-1")
@@ -36,7 +36,7 @@ func TestAPlanIsKeptAdvancedAndResumedFromTheStore(t *testing.T) {
 		t.Fatalf("the advanced plan did not come back as left: %v %+v", err, again)
 	}
 	again.State = PlanDone
-	if err := inv.SavePlan(ctx, again); err != nil {
+	if err := inv.SavePlan(ctx, &again); err != nil {
 		t.Fatal(err)
 	}
 	if open, _ = inv.OpenPlans(ctx); len(open) != 0 {
@@ -54,7 +54,7 @@ func TestASupersededPlanIsNotOpen(t *testing.T) {
 	p := Plan{ID: "plan-1", Repository: "novox/mesh-catalog", Branch: "main", Commit: "abc",
 		Created: time.Now().UTC(), State: PlanBuilding, Tiers: [][]string{{"gitea"}},
 		Modules: map[string]*PlanModule{"gitea": {}}}
-	if err := inv.SavePlan(ctx, p); err != nil {
+	if err := inv.SavePlan(ctx, &p); err != nil {
 		t.Fatal(err)
 	}
 	kept, err := inv.PlanByID(ctx, "plan-1")
@@ -63,7 +63,7 @@ func TestASupersededPlanIsNotOpen(t *testing.T) {
 	}
 	kept.State = PlanSuperseded
 	kept.Note = "superseded at tier 0 by plan-2"
-	if err := inv.SavePlan(ctx, kept); err != nil {
+	if err := inv.SavePlan(ctx, &kept); err != nil {
 		t.Fatal(err)
 	}
 	if open, err := inv.OpenPlans(ctx); err != nil || len(open) != 0 {