Merge main (#227) into the grants step's branch

cmd/mesh-controller/facts.go

@@ -514,6 +514,10 @@ func composedAndValidated(ctx context.Context, open *stores, node string, gens m
 	if declared.Epoch, err = open.inventory.SentEpoch(ctx, record.ID); err != nil {
 		return sendable{}, nil, err
 	}
+	// And the generation it was last sent, as the would-send is (novox/hq issue 234).
+	if declared.Generation, err = open.inventory.SentGeneration(ctx, record.ID); err != nil {
+		return sendable{}, nil, err
+	}
 	body, err := declared.Body()
 	if err != nil {
 		return sendable{}, nil, err

cmd/mesh-controller/generation.go

@@ -0,0 +1,222 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"os"
+	"sort"
+	"strings"
+	"time"
+
+	"github.com/novox/mesh-controller/internal/catalogue"
+	"github.com/novox/mesh-controller/internal/conditions"
+	"github.com/novox/mesh-controller/internal/inventory"
+	"github.com/novox/mesh-controller/internal/link"
+)
+
+// The assignment generation a declaration was composed from, and the record of every send (novox/hq
+// issue 234).
+//
+// On 2026-10-04 a declaration newer in sequence than every other named four fewer modules than the
+// assignments held, and a machine applied it and undeclared all four. The sequence orders arrival and
+// cannot tell a later send that carries an older view of the assignments. So a declaration now carries
+// the generation of the assignments it was composed from — a counter the store raises in the same
+// transaction as every assignment change — and a machine refuses one older than it applied, unless it is
+// a gate's put-back. And every send is written down with who sent it, from which generation and naming
+// which modules: the controller logged no send then, and which process sent the stale declaration could
+// not be read back from anything the mesh kept.
+
+// kindOlderGeneration is S20's kind: a machine refused a send for the generation it was composed from.
+const kindOlderGeneration = "older-generation"
+
+// generationRefusalsSaid is how long a refused send is said after its refusal: an hour, as an advisory is.
+const generationRefusalsSaid = advisoryQuiet
+
+// stamp gives a composed declaration the order allotted to it before it was composed: its sequence, and
+// the epoch and generation when the machine reads them — and keeps the generation and acting epoch for the
+// record of the send whether or not the machine is sent them.
+//
+// **No put-back mark is sent.** A gate puts a machine back by composing it again (sendRollout), so its
+// declaration is allotted here like any other and carries the generation as it stands — never older than
+// what the machine applied. The node-engine reads a `put_back` key (mesh-host#82); this controller never
+// needs to send one.
+func (o order) stamp(d *sendable) {
+	d.Sequence, d.Epoch = o.sequence, o.epoch
+	d.composedFrom, d.actingEpoch = o.generation, o.acting
+	d.Generation = 0
+	if o.readsGeneration && o.generation > 0 {
+		d.Generation = o.generation
+	}
+}
+
+// sentRecord is what the record of a send keeps beside its digest.
+type sentRecord struct {
+	sequence   int64
+	epoch      uint64
+	generation int64
+	// toldGeneration is the generation on the wire, which the would-send is stamped with: zero for a machine
+	// whose node-engine has not said it reads one.
+	toldGeneration int64
+	sender         string
+	modules        []string
+}
+
+// sentRecordOf is a composed send's record: its sender is the caller this process acts for.
+func sentRecordOf(ctx context.Context, d sendable) sentRecord {
+	return sentRecord{sequence: d.Sequence, epoch: d.actingEpoch, generation: d.composedFrom,
+		toldGeneration: d.Generation, sender: senderOf(callerOf(ctx)), modules: d.modules}
+}
+
+// callerOf is who asked for what this process does: the seat call's caller when ctx belongs to one, the
+// caller the controller ran this command for, or the account at the shell.
+func callerOf(ctx context.Context) string {
+	if c := link.CallerIn(ctx); c != "" {
+		return c
+	}
+	return link.Caller()
+}
+
+// senderOf is a send's sender in words: who asked, and the process and build that composed it — the
+// answer issue 234 could not find, because two controllers and a one-shot push were all sending then.
+func senderOf(caller string) string {
+	host, _ := os.Hostname()
+	return fmt.Sprintf("%s (pid %d on %s, build %s)", caller, os.Getpid(), host, version)
+}
+
+// namedModules are the modules a declaration names: its machine's set, less what was left out of it.
+func namedModules(plan catalogue.Resolution, leftOut map[string]string) []string {
+	out := make([]string, 0, len(plan.Modules))
+	for _, m := range plan.Modules {
+		if _, left := leftOut[m.Module]; !left {
+			out = append(out, m.Module)
+		}
+	}
+	sort.Strings(out)
+	return out
+}
+
+// heardGenerationRefusal keeps a machine's refusal of a send for its generation, so S20 names its sender
+// and says it (novox/hq issue 234): on the send it refused or — when the mesh has no record of sending that
+// sequence — as a refusal of its own, naming the sender as unknown. Either way S20 is raised: a refusal the
+// mesh cannot attribute is the louder fact, not a quieter one.
+//
+// **And raises the mesh's counter past what the machine applied, when the refused send was composed from
+// the counter as it stands** (counterBehind): then the sender's view was not stale — the counter is behind
+// the machine, which is a store put back from a backup — and every send after would be refused for ever. A
+// stale sender's generation is below the counter, and the counter is left alone for it.
+//
+// Nothing is sent from here. This runs in the controller's receive loop, and a push from it would hold that
+// loop, and the machine's hold, for as long as the push takes — the deaf controller of issues 184 and 185.
+// S20 names `push <node>` for that case instead.
+func heardGenerationRefusal(ctx context.Context, inv *inventory.Inventory, report link.Report) {
+	r := report.OlderGeneration
+	if r == nil || inv == nil || report.Node == "" {
+		return
+	}
+	node, err := inv.NodeByName(ctx, report.Node)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "mesh-controller: %s refused a send for its generation, and the machine cannot be "+
+			"read, so it is not raised: %v\n", report.Node, err)
+		return
+	}
+	var raised int64
+	if now, err := inv.AssignmentGeneration(ctx); err != nil {
+		fmt.Fprintf(os.Stderr, "mesh-controller: %s refused a send for its generation, and the mesh's own cannot be "+
+			"read: %v\n", report.Node, err)
+	} else if counterBehind(*r, now) {
+		if raised, err = inv.RaiseAssignmentGeneration(ctx, r.Applied); err != nil {
+			fmt.Fprintf(os.Stderr, "mesh-controller: the assignment generation (%d) is behind what %s applied (%d), "+
+				"and could not be raised: %v\n", now, report.Node, r.Applied, err)
+			raised = 0
+		} else {
+			fmt.Fprintf(os.Stderr, "mesh-controller: the assignment generation was %d, behind what %s applied (%d) — "+
+				"a store put back from a backup — and is raised to %d; `push %s` sends it what the mesh holds now\n",
+				now, report.Node, r.Applied, raised, report.Node)
+		}
+	}
+	send, found, err := inv.RefusedSend(ctx, node.ID, report.Sequence, r.Applied, raised)
+	if err == nil && !found {
+		send, err = inv.RecordUnrecordedRefusal(ctx, inventory.Send{Node: node.ID, Sequence: report.Sequence,
+			Epoch: report.Epoch, Generation: r.Generation, Digest: report.Declared, RefusedApplied: r.Applied,
+			CounterRaisedTo: raised})
+	}
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "mesh-controller: %s refused send %d for its generation, and the refusal could not be "+
+			"kept, so it is not raised: %v\n", report.Node, report.Sequence, err)
+		return
+	}
+	fmt.Fprintf(os.Stderr, "mesh-controller: %s refused send %d from %s: composed from assignment generation %d, "+
+		"and it applied %d\n", report.Node, report.Sequence, send.Sender, r.Generation, r.Applied)
+}
+
+// counterBehind says a refusal shows the mesh's counter behind the machine rather than a stale sender: the
+// refused send carried the counter as it stands now, and the machine applied more than that.
+func counterBehind(r link.GenerationRefusal, now int64) bool {
+	return r.Generation >= now && r.Applied > r.Generation
+}
+
+// watchGenerationRefusals is S20: every send a machine refused within the hour for the generation it was
+// composed from, naming who sent it (novox/hq issue 234). One per machine, the newest refusal.
+func watchGenerationRefusals(f *signalFacts) []conditions.Observation {
+	seen := map[string]bool{}
+	var out []conditions.Observation
+	for _, s := range f.refusedSends {
+		if s.RefusedAt == nil || f.now.Sub(*s.RefusedAt) > generationRefusalsSaid || seen[s.NodeName] {
+			continue
+		}
+		seen[s.NodeName] = true
+		o := conditions.Observation{Scope: conditions.ScopeMachine, ID: s.NodeName, Kind: kindOlderGeneration,
+			Machine: s.NodeName, Severity: conditions.Warning,
+			Summary: fmt.Sprintf("%s refused sequence %d from %s: it was composed from assignment generation %d, and "+
+				"%s applied generation %d — a sender composing from a view of the assignments the mesh has moved "+
+				"past; it named %s", s.NodeName, s.Sequence, s.Sender, s.Generation, s.NodeName, s.RefusedApplied,
+				modulesWords(s.Modules)),
+			Said:     fmt.Sprintf("refused at %s", s.RefusedAt.UTC().Format(time.RFC3339)),
+			Headline: fmt.Sprintf("%s refused an out-of-date update", s.NodeName),
+			Explanation: fmt.Sprintf("Something sent %s an update made from an older list of what runs there. %s "+
+				"refused it, so nothing was removed. Nothing for you to do unless it repeats.", s.NodeName, s.NodeName),
+			Resolved: fmt.Sprintf("%s has had no out-of-date update for an hour", s.NodeName)}
+		if s.CounterRaisedTo > 0 {
+			// Not a stale sender: the mesh's counter was behind the machine (a store put back from a backup) and
+			// was raised; nothing has sent the machine its declaration since, so a person is asked to.
+			o.Summary = fmt.Sprintf("%s refused sequence %d from %s: it was composed from assignment generation %d, "+
+				"the mesh's own, and %s applied generation %d — the mesh's counter was behind the machine (a store "+
+				"put back from a backup?) and is raised to %d; `push %s` sends it what the mesh holds now",
+				s.NodeName, s.Sequence, s.Sender, s.Generation, s.NodeName, s.RefusedApplied, s.CounterRaisedTo,
+				s.NodeName)
+			o.Explanation = fmt.Sprintf("Needs you: push %s. The mesh's record was older than %s, so %s refused its "+
+				"update and kept what it had. The record is repaired; a push sends the update again.",
+				s.NodeName, s.NodeName, s.NodeName)
+		}
+		out = append(out, o)
+	}
+	return out
+}
+
+// modulesWords is a send's modules as a sentence says them.
+func modulesWords(modules []string) string {
+	if len(modules) == 0 {
+		return "no module the mesh recorded"
+	}
+	return strings.Join(modules, ", ")
+}
+
+// writeLastSend says what a machine was last told, by whom and from which generation (novox/hq issue 234):
+// nothing when the mesh has not recorded a send to it.
+func writeLastSend(ctx context.Context, w io.Writer, inv *inventory.Inventory, node string) error {
+	s, found, err := inv.LastSend(ctx, node)
+	if err != nil || !found {
+		return err
+	}
+	generation := "no generation recorded"
+	if s.Generation > 0 {
+		generation = fmt.Sprintf("assignment generation %d", s.Generation)
+	}
+	fmt.Fprintf(w, "%s was last sent sequence %d at %s by %s, composed from %s, naming %s\n", node, s.Sequence, s.SentAt.Local().Format("2006-01-02 15:04:05"), s.Sender, generation, modulesWords(s.Modules))
+	if s.RefusedAt != nil {
+		fmt.Fprintf(w, "  and refused it at %s: it had applied generation %d\n",
+			s.RefusedAt.Local().Format("2006-01-02 15:04:05"), s.RefusedApplied)
+	}
+	return nil
+}

cmd/mesh-controller/generation_test.go

@@ -0,0 +1,148 @@
+package main
+
+import (
+	"encoding/json"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/novox/mesh-controller/internal/inventory"
+	"github.com/novox/mesh-controller/internal/link"
+)
+
+// The assignment generation a declaration was composed from (novox/hq issue 234).
+
+func bodyKeys(t *testing.T, s sendable) map[string]any {
+	t.Helper()
+	raw, err := s.Body()
+	if err != nil {
+		t.Fatal(err)
+	}
+	var keys map[string]any
+	if err := json.Unmarshal(raw, &keys); err != nil {
+		t.Fatal(err)
+	}
+	return keys
+}
+
+// **The generation goes on the wire only to a machine whose node-engine said it reads one**: an older
+// node-engine decodes strictly and refuses an unknown key, whole. No put-back mark is ever sent: a gate
+// composes its put-back afresh, with the generation as it stands.
+func TestTheGenerationIsSentOnlyToAMachineThatReadsOne(t *testing.T) {
+	resources := []map[string]any{{"id": "a", "type": "file", "path": "/etc/a", "content": "x\n"}}
+	reads := order{sequence: 12, epoch: 57, generation: 40, readsGeneration: true, acting: 57}
+	var told sendable
+	told.Resources = resources
+	reads.stamp(&told)
+	keys := bodyKeys(t, told)
+	if keys["generation"] != float64(40) || keys["sequence"] != float64(12) {
+		t.Fatalf("a machine that reads a generation was sent %v", keys)
+	}
+	if _, there := keys["put_back"]; there {
+		t.Fatalf("a put-back mark was sent: %v", keys)
+	}
+	if told.composedFrom != 40 || told.actingEpoch != 57 {
+		t.Errorf("the send does not keep what it was composed from for its record: %+v", told)
+	}
+
+	older := order{sequence: 12, epoch: 57, generation: 40, readsGeneration: false, acting: 57}
+	var untold sendable
+	untold.Resources = resources
+	older.stamp(&untold)
+	if _, there := bodyKeys(t, untold)["generation"]; there {
+		t.Fatal("a machine whose node-engine never said it reads a generation was sent one")
+	}
+	if untold.composedFrom != 40 {
+		t.Errorf("the generation it was composed from is recorded whether or not it was sent: %+v", untold)
+	}
+}
+
+// **The sender is named**: the caller of the seat call when there is one, else the shell's account, and the
+// process and build that composed it.
+func TestASendNamesItsSender(t *testing.T) {
+	said := senderOf("g14.node-tools, through the mesh-controller seat")
+	if !strings.Contains(said, "g14.node-tools") || !strings.Contains(said, "pid ") || !strings.Contains(said, "build ") {
+		t.Fatalf("the sender reads %q", said)
+	}
+}
+
+// refusedSend is a send a machine refused for its generation at a moment.
+func refusedSend(at time.Time) inventory.Send {
+	return inventory.Send{NodeName: "anchor", Sequence: 12, Epoch: 57, Generation: 38, RefusedApplied: 40,
+		Sender:  "a one-shot push by jochen at a shell on anchor (pid 4242 on anchor, build 2026.10.11)",
+		Modules: []string{"docker"}, SentAt: at.Add(-time.Second), RefusedAt: &at, Recorded: true}
+}
+
+// **A refused send is raised naming its sender** (novox/hq issue 234): who sent it, from which generation,
+// against which the machine applied, and its sequence — the facts that took a morning to look for.
+func TestARefusedSendIsRaisedNamingItsSender(t *testing.T) {
+	now := time.Date(2026, 10, 11, 12, 0, 0, 0, time.UTC)
+	f := calm(now)
+	f.refusedSends = []inventory.Send{refusedSend(now.Add(-time.Minute))}
+	got := watchGenerationRefusals(f)
+	if len(got) != 1 {
+		t.Fatalf("%+v", got)
+	}
+	o := got[0]
+	if o.Key() != "machine.anchor.older-generation" || o.Machine != "anchor" {
+		t.Errorf("raised as %s about %q", o.Key(), o.Machine)
+	}
+	for _, want := range []string{"a one-shot push by jochen", "generation 38", "generation 40", "sequence 12"} {
+		if !strings.Contains(o.Summary, want) {
+			t.Errorf("the summary does not say %q: %s", want, o.Summary)
+		}
+	}
+	if o.Headline == "" || o.Explanation == "" || o.Resolved == "" {
+		t.Errorf("the condition is not worded for the operator: %+v", o)
+	}
+}
+
+// **A refusal of a send the mesh has no record of is raised too, its sender said to be unknown** — never
+// only a line on stderr.
+func TestARefusalOfAnUnrecordedSendIsRaisedSayingTheSenderIsUnknown(t *testing.T) {
+	now := time.Date(2026, 10, 11, 12, 0, 0, 0, time.UTC)
+	f := calm(now)
+	s := refusedSend(now.Add(-time.Minute))
+	s.Recorded, s.Sender, s.Modules = false, "a sender the mesh has no record of (no send of this sequence was recorded)", nil
+	f.refusedSends = []inventory.Send{s}
+	got := watchGenerationRefusals(f)
+	if len(got) != 1 || !strings.Contains(got[0].Summary, "no record of") {
+		t.Fatalf("an unattributed refusal was not raised as one: %+v", got)
+	}
+}
+
+// **When the refusal showed the mesh's counter behind the machine, the condition asks for a push** — it was
+// raised, and nothing has sent the machine its declaration since — and does not say there is nothing to do.
+func TestACounterBehindTheMachineAsksForAPush(t *testing.T) {
+	now := time.Date(2026, 10, 11, 12, 0, 0, 0, time.UTC)
+	f := calm(now)
+	s := refusedSend(now.Add(-time.Minute))
+	s.Generation, s.RefusedApplied, s.CounterRaisedTo = 12, 40, 41
+	f.refusedSends = []inventory.Send{s}
+	got := watchGenerationRefusals(f)
+	if len(got) != 1 {
+		t.Fatalf("%+v", got)
+	}
+	if !strings.Contains(got[0].Summary, "`push anchor`") || !strings.Contains(got[0].Explanation, "push anchor") ||
+		strings.Contains(got[0].Explanation, "Nothing for you to do") {
+		t.Errorf("a counter behind the machine does not ask for a push: %s / %s", got[0].Summary, got[0].Explanation)
+	}
+}
+
+// **Only a refusal of the counter as it stands is the counter behind**: a stale sender's generation is below
+// it, and the counter is left alone for that.
+func TestOnlyARefusalOfTheCounterAsItStandsRaisesIt(t *testing.T) {
+	for _, c := range []struct {
+		refused, applied, now int64
+		behind                bool
+	}{
+		{refused: 12, applied: 40, now: 12, behind: true},  // the store was put back: the mesh says 12, the machine had 40
+		{refused: 38, applied: 40, now: 41, behind: false}, // a stale sender: the counter is already past
+		{refused: 38, applied: 40, now: 40, behind: false}, // a stale sender: the counter is where the machine is
+	} {
+		r := link.GenerationRefusal{Generation: c.refused, Applied: c.applied}
+		if got := counterBehind(r, c.now); got != c.behind {
+			t.Errorf("refused %d, applied %d, counter %d: behind %v, want %v", c.refused, c.applied, c.now, got, c.behind)
+		}
+	}
+}

cmd/mesh-controller/held_back_test.go

@@ -128,7 +128,7 @@ func (r *recordedDelivery) grant(context.Context, []readyNode) error { return ni
 
 func (r *recordedDelivery) declare(ctx context.Context, s readyNode, body []byte) (string, error) {
 	r.declared = append(r.declared, s.node)
-	return recordSent(ctx, r.inv, s.node, body, s.declared.Builds, s.declared.Epoch)
+	return recordSent(ctx, r.inv, s.node, body, s.declared.Builds, s.declared.Epoch, sentRecordOf(ctx, s.declared))
 }
 
 // aResolver is a module built from a repository, at a commit, with something on the machine that

cmd/mesh-controller/issue_204_test.go

@@ -77,7 +77,7 @@ func TestASendIsRecordedEvenWhenTheSenderIsBeingCancelled(t *testing.T) {
 	}
 	cancel() // the sender is going away: its context is cancelled between the send and the record
 	body := []byte(`{"declaration":1,"resources":[]}`)
-	digest, err := recordSent(ctx, inv, "anchor", body, nil, 0)
+	digest, err := recordSent(ctx, inv, "anchor", body, nil, 0, sentRecord{sender: "a test"})
 	if err != nil {
 		// NodeByName on the cancelled context may itself refuse; the record must still be possible
 		// through the detached context, so look the node up again on a live one.

cmd/mesh-controller/plan.go

@@ -420,7 +420,8 @@ func declarationWith(ctx context.Context, open *stores, node string,
 	out := sendable{Resources: composed.Resources, Adoption: adoption,
 		Received: composed.Received, Mesh: with.Mesh, BusUsers: with.BusUsers,
 		LeftOut: sortedKeysOf(composed.LeftOut), leftOutWhy: composed.LeftOut, withheld: with.Withheld,
-		unbound: with.Unbound, foreseen: composed.Foreseen, unplaced: composed.Unplaced}
+		unbound: with.Unbound, foreseen: composed.Foreseen, unplaced: composed.Unplaced,
+		modules: namedModules(plan, composed.LeftOut)}
 	// And which build of each module it carries, for the send to record (novox/hq issue 259, ADR
 	// 0221). Read only on the send path: a question about what would be sent records nothing.
 	if choosing == Allocating {
@@ -1290,6 +1291,11 @@ func planCommand(ctx context.Context, args []string) error {
 		return nil
 	}
 
+	// What the machine was last told, by whom and from which assignment generation (novox/hq issue 234),
+	// beside what it would be told now. A record that cannot be read is said, not taken for none.
+	if err := writeLastSend(ctx, os.Stdout, open.inventory, args[0]); err != nil {
+		fmt.Printf("what %s was last sent could not be read: %v\n", args[0], err)
+	}
 	// Which modules a push would leave out, and why — said before the plan, since the plan is of
 	// what the machine would be told (novox/hq ADR 0163, rule 6). Judged, never composed: `plan`
 	// without --json allocates nothing.

cmd/mesh-controller/push.go

@@ -356,10 +356,16 @@ func declare(ctx context.Context, args []string) error {
 	// the mesh did not compose it, so a push that does not name this machine treats it as held.
 	// The epoch it carried, if a person wrote one in, is what the machine heard.
 	var carried struct {
-		Epoch uint64 `json:"epoch"`
+		Epoch      uint64 `json:"epoch"`
+		Sequence   int64  `json:"sequence"`
+		Generation int64  `json:"generation"`
 	}
 	_ = json.Unmarshal(raw, &carried)
-	if _, err := recordSent(ctx, inv, node, raw, nil, carried.Epoch); err != nil {
+	// And recorded as every send is (novox/hq issue 234): by hand, from what it carried; the modules it
+	// named are not known, since the mesh did not compose it.
+	if _, err := recordSent(ctx, inv, node, raw, nil, carried.Epoch, sentRecord{sequence: carried.Sequence,
+		epoch: carried.Epoch, generation: carried.Generation, toldGeneration: carried.Generation,
+		sender: senderOf(callerOf(ctx)) + ", a declaration sent by hand"}); err != nil {
 		return err
 	}
 	fmt.Printf("sent %s a signed declaration (%d bytes)\n", node, len(raw))
@@ -798,7 +804,7 @@ func composeEach(names []string, allot func(node string) (order, error),
 			refusals = append(refusals, fmt.Sprintf("%s:\n%v", name, err))
 			continue
 		}
-		declared.Sequence, declared.Epoch = numbered.sequence, numbered.epoch
+		numbered.stamp(&declared)
 		if len(declared.Resources) == 0 {
 			// Sent, not skipped (novox/hq issue 127). A node whose declaration composes to
 			// nothing may have HELD something before — the broker opening a placement gave it,
@@ -1016,7 +1022,8 @@ func (b overTheBus) declare(ctx context.Context, s readyNode, body []byte) (stri
 	}
 	// After it is away, not before. A digest recorded for something that failed to send would make
 	// the machine look current for a declaration it never received.
-	digest, err := recordSent(ctx, b.open.inventory, s.node, body, s.declared.Builds, s.declared.Epoch)
+	digest, err := recordSent(ctx, b.open.inventory, s.node, body, s.declared.Builds, s.declared.Epoch,
+		sentRecordOf(ctx, s.declared))
 	if err != nil {
 		return "", err
 	}
@@ -1228,7 +1235,7 @@ func sendToEach(ctx context.Context, open *stores, names []string) ([]string, er
 			refusals = append(refusals, fmt.Sprintf("%s:\n%v", name, err))
 			continue
 		}
-		declared.Sequence, declared.Epoch = numbered.sequence, numbered.epoch
+		numbered.stamp(&declared)
 		reportLeftOut(name, declared)
 		sending = append(sending, readyNode{name, declared})
 	}
@@ -1393,6 +1400,11 @@ func wouldSendFrom(ctx context.Context, open *stores,
 		if declared.Epoch, err = open.inventory.SentEpoch(ctx, n.ID); err != nil {
 			return nil, err
 		}
+		// And the generation it was last sent, for the same reason (novox/hq issue 234): an assignment
+		// elsewhere in the mesh is not a change of this machine.
+		if declared.Generation, err = open.inventory.SentGeneration(ctx, n.ID); err != nil {
+			return nil, err
+		}
 		body, err := declared.Body()
 		if err != nil {
 			return nil, err
@@ -1531,6 +1543,12 @@ var epochForActs = func(ctx context.Context) (uint64, error) { return theLease.e
 type order struct {
 	sequence int64
 	epoch    uint64
+	// generation is the assignment generation read before the composition, and readsGeneration whether the
+	// machine's node-engine said it reads one (novox/hq issue 234); acting is the lease epoch the sender acts
+	// under, whether or not the machine is sent it — both kept for the record of the send.
+	generation      int64
+	readsGeneration bool
+	acting          uint64
 }
 
 // allot takes the next sequence for a machine — the number its next declaration carries — under the
@@ -1544,6 +1562,7 @@ func allot(ctx context.Context, inv *inventory.Inventory, node string) (order, e
 	if err != nil {
 		return order{}, err
 	}
+	acting := epoch
 	if epoch > 0 {
 		reads, err := inv.ReadsEpoch(ctx, record.ID)
 		if err != nil {
@@ -1553,11 +1572,24 @@ func allot(ctx context.Context, inv *inventory.Inventory, node string) (order, e
 			epoch = 0
 		}
 	}
+	// **The generation is read here, before the composition reads a single assignment** (novox/hq issue
+	// 234). A generation only grows, so one read before is never newer than the view composed after it:
+	// the declaration may claim a generation older than its content, never newer — and a claim newer than
+	// the content is exactly the stale send the machine must be able to refuse.
+	generation, err := inv.AssignmentGeneration(ctx)
+	if err != nil {
+		return order{}, err
+	}
+	readsGeneration, err := inv.ReadsGeneration(ctx, record.ID)
+	if err != nil {
+		return order{}, err
+	}
 	seq, err := inv.NextSequence(ctx, record.ID)
 	if err != nil {
 		return order{}, err
 	}
-	return order{sequence: seq, epoch: epoch}, nil
+	return order{sequence: seq, epoch: epoch, generation: generation, readsGeneration: readsGeneration,
+		acting: acting}, nil
 }
 
 // recordSent writes down what a machine was just sent, and returns the digest.
@@ -1572,7 +1604,7 @@ func allot(ctx context.Context, inv *inventory.Inventory, node string) (order, e
 // And the build of each module it carried (novox/hq issue 259, ADR 0221), nil when that is not known:
 // what tells a machine held back by a policy or a plan from one a push left behind.
 func recordSent(ctx context.Context, inv *inventory.Inventory, node string, body []byte,
-	builds map[string]string, epoch uint64) (string, error) {
+	builds map[string]string, epoch uint64, sent sentRecord) (string, error) {
 	kept, cancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second)
 	defer cancel()
 	record, err := inv.NodeByName(kept, node)
@@ -1580,7 +1612,21 @@ func recordSent(ctx context.Context, inv *inventory.Inventory, node string, body
 		return "", err
 	}
 	digest := digestOf(body)
-	if err := inv.RecordSentUnder(kept, record.ID, digest, builds, epoch); err != nil {
+	// The digest and the generation it carried are written in one transaction (novox/hq issue 234), so the
+	// would-send is never stamped with a generation the machine was not sent; and the send itself, who sent
+	// it and from which generation, beside them. A record of the send that cannot be written does not make a
+	// send that is away look failed: it is said, loudly, and the machine's own record stands.
+	err = inv.RecordSentWith(kept, record.ID, digest, builds, epoch, sent.toldGeneration, inventory.Send{
+		Sequence: sent.sequence, Epoch: int64(sent.epoch), Sender: sent.sender, Generation: sent.generation,
+		Digest: digest, Modules: sent.modules})
+	var unrecorded *inventory.SendNotRecordedError
+	switch {
+	case errors.As(err, &unrecorded):
+		fmt.Fprintf(os.Stderr, "mesh-controller: SEND NOT RECORDED: %s was sent declaration %s (sequence %d), and who "+
+			"sent it and from which generation could not be written down, so a refusal of it will name no sender "+
+			"(novox/hq issue 234): %v\n", node, short(digest), sent.sequence, unrecorded.Err)
+		fmt.Printf("%s: sent, and the record of who sent it could NOT be written: %v\n", node, unrecorded.Err)
+	case err != nil:
 		return "", err
 	}
 	// And what it was, summarised, so the next push can be compared with it (novox/hq ADR 0217).

cmd/mesh-controller/sendable.go

@@ -27,6 +27,19 @@ type sendable struct {
 	// said it reads one is sent none, because an older node-engine refuses a key it does not know, whole
 	// (link/order.go, the contract).
 	Epoch uint64
+	// Generation is the assignment generation it was composed from (novox/hq issue 234): a machine that
+	// applied a later one refuses it, so a send composed from a view of the assignments the mesh has moved
+	// past cannot undeclare what is still assigned. Zero is not sent at all — every machine whose
+	// node-engine has not said it reads one is sent none, for the reason the epoch is not (an older
+	// node-engine refuses a key it does not know, whole).
+	Generation int64
+	// composedFrom is the generation read before this declaration was composed, and actingEpoch the
+	// lease epoch its sender acted under — whether or not the machine is sent either — for the record of
+	// the send; never on the wire.
+	composedFrom int64
+	actingEpoch  uint64
+	// modules are the modules this declaration names, for the record of the send; never on the wire.
+	modules []string
 	// Adoption is nil for a converged node, and then the body is byte for byte what it was before
 	// adoption existed: an older host parses the envelope strictly and would refuse the key.
 	Adoption *adoptionEnvelope
@@ -94,6 +107,9 @@ func (s sendable) Body() ([]byte, error) {
 	if len(s.LeftOut) > 0 {
 		envelope["left_out"] = s.LeftOut
 	}
+	if s.Generation > 0 {
+		envelope["generation"] = s.Generation
+	}
 	// An empty declaration is deliberate here — the node owns nothing the mesh put there
 	// (novox/hq issue 127) — and the host refuses an empty body unless it is told the emptiness
 	// is meant, so a truncated or mis-composed body is never mistaken for "own nothing".

cmd/mesh-controller/signals.go

@@ -226,6 +226,20 @@ var signalsTable = []signalRow{
 		newest: func(f *signalFacts) time.Time {
 			return newestOf(f.batches, func(b batchFacts) time.Time { return b.closed })
 		}},
+	{Row: "S20", Signal: "a send refused for the assignment generation it was composed from",
+		Emitter: "node-engine", Trigger: "each refusal (novox/hq issue 234)",
+		Bound: "none: raised at the first refusal, naming its sender, the generation it came from and the one the " +
+			"machine applied; cleared an hour after the last",
+		Kind: kindOlderGeneration, Severity: conditions.Warning, Phase: 2,
+		needs: func(f *signalFacts) error { return f.refusedSendsErr }, watch: watchGenerationRefusals,
+		newest: func(f *signalFacts) time.Time {
+			return newestOf(f.refusedSends, func(s inventory.Send) time.Time {
+				if s.RefusedAt == nil {
+					return time.Time{}
+				}
+				return *s.RefusedAt
+			})
+		}},
 	{Row: "S17", Signal: "a send held for the bus's planned step is told to a person", Emitter: "controller's plan",
 		Trigger: "each send refused because it would replace the bus outside its step (novox/hq issue 336)",
 		Bound: "none: raised at the first refusal, for the operator, naming what waits, the bus build from and to, " +

cmd/mesh-controller/signals_test.go

@@ -178,6 +178,14 @@ var suppressions = map[string]suppression{
 					commit: "c0ffee001122", modules: []string{"app"}, since: f.now.Add(-time.Second)}}}
 		},
 	},
+	// A send refused by its machine for the generation it was composed from (novox/hq issue 234): said at
+	// once, naming its sender, for an hour after. Inside: the last such refusal was more than an hour ago.
+	"S20": {
+		inside: func(f *signalFacts) {
+			f.refusedSends = []inventory.Send{refusedSend(f.now.Add(-61 * time.Minute))}
+		},
+		past: func(f *signalFacts) { f.refusedSends = []inventory.Send{refusedSend(f.now.Add(-time.Second))} },
+	},
 	// Twice by hand within a fortnight is a healer wanted; once, or the first of two a day too old, is not.
 	"S15": {
 		inside: func(f *signalFacts) {

cmd/mesh-controller/status_summary.go

@@ -196,6 +196,11 @@ func (l nudgingListener) Heard(ctx context.Context, report link.Report) (bool, e
 	if report.Ordered() {
 		link.StaleRefusals.Lifetime(report.Node, report.RefusedOlder, now)
 	}
+	// A send refused for the generation it was composed from is kept on the send it refused, so S20 names
+	// its sender (novox/hq issue 234).
+	if report.OlderGeneration != nil {
+		heardGenerationRefusal(ctx, l.Enrolment.Inventory, report)
+	}
 	// What the machine's witnesses put back and stand by (novox/hq ADR 0236): read by the gate and its
 	// probe. Only from an account of the machine — not a word that a declaration was set aside, nor a rekey.
 	if report.Superseded == "" && report.Rekey == nil && report.Node != "" {

cmd/mesh-controller/watchdogs.go

@@ -55,6 +55,10 @@ type signalFacts struct {
 	waits []waitFacts
 	// batches are the batches not yet cut (S18, S19, novox/hq ADR 0276).
 	batches []batchFacts
+	// refusedSends are the sends a machine refused within the hour for the generation they were composed
+	// from, each naming its sender (S20, novox/hq issue 234).
+	refusedSends    []inventory.Send
+	refusedSendsErr error
 
 	loop    loopFacts
 	loopErr error
@@ -364,6 +368,7 @@ func (w *watchdogs) gather(ctx context.Context) *signalFacts {
 		}
 	}
 	f.handActs, f.handActsErr = w.gatherHandActs(ctx, now)
+	f.refusedSends, f.refusedSendsErr = inv.RefusedSendsSince(ctx, now.Add(-generationRefusalsSaid))
 	f.facts.taken, _, f.facts.began, f.facts.err = exportedFacts.last()
 	return f
 }