Say in every declaration the assignment generation it came from, and record every send (hq issue 234)

A declaration newer in sequence than every other named four fewer modules
than the assignments held, a machine applied it, and nothing the mesh kept
said who sent it or from what view. The store now raises an assignment
generation in the same transaction as every assignment change (a trigger,
so a cascade counts too); a send reads it before composing, carries it to
engines that said they read it, marks a gate's put-back, and is recorded
with its sequence, sender, generation and modules. The would-send is
stamped with what was last sent, so nothing reads behind for it; a refusal
is kept on the send it refused and raised as S20 naming the sender; plan
says what the machine was last told.

cmd/mesh-controller/generation.go

@@ -0,0 +1,195 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"os"
+	"sort"
+	"strings"
+	"time"
+
+	"github.com/novox/mesh-controller/internal/catalogue"
+	"github.com/novox/mesh-controller/internal/conditions"
+	"github.com/novox/mesh-controller/internal/inventory"
+	"github.com/novox/mesh-controller/internal/link"
+)
+
+// The assignment generation a declaration was composed from, and the record of every send (novox/hq
+// issue 234).
+//
+// On 2026-10-04 a declaration newer in sequence than every other named four fewer modules than the
+// assignments held, and a machine applied it and undeclared all four. The sequence orders arrival and
+// cannot tell a later send that carries an older view of the assignments. So a declaration now carries
+// the generation of the assignments it was composed from — a counter the store raises in the same
+// transaction as every assignment change — and a machine refuses one older than it applied, unless it is
+// a gate's put-back. And every send is written down with who sent it, from which generation and naming
+// which modules: the controller logged no send then, and which process sent the stale declaration could
+// not be read back from anything the mesh kept.
+
+// kindOlderGeneration is S20's kind: a machine refused a send for the generation it was composed from.
+const kindOlderGeneration = "older-generation"
+
+// generationRefusalsSaid is how long a refused send is said after its refusal: an hour, as an advisory is.
+const generationRefusalsSaid = advisoryQuiet
+
+// stamp gives a composed declaration the order allotted to it before it was composed: its sequence, the
+// epoch and generation when the machine reads them, and the put-back mark beside a generation — and keeps
+// the generation and acting epoch for the record of the send whether or not the machine is sent them.
+func (o order) stamp(d *sendable) {
+	d.Sequence, d.Epoch = o.sequence, o.epoch
+	d.composedFrom, d.actingEpoch, d.putBackSend = o.generation, o.acting, o.putBack
+	d.Generation, d.PutBack = 0, false
+	if o.readsGeneration && o.generation > 0 {
+		d.Generation, d.PutBack = o.generation, o.putBack
+	}
+}
+
+// sentRecord is what the record of a send keeps beside its digest.
+type sentRecord struct {
+	sequence   int64
+	epoch      uint64
+	generation int64
+	putBack    bool
+	// told is the generation and put-back mark on the wire, which the would-send is stamped with.
+	toldGeneration int64
+	toldPutBack    bool
+	sender         string
+	modules        []string
+}
+
+// sentRecordOf is a composed send's record: its sender is the caller this process acts for.
+func sentRecordOf(ctx context.Context, d sendable) sentRecord {
+	return sentRecord{sequence: d.Sequence, epoch: d.actingEpoch, generation: d.composedFrom, putBack: d.putBackSend,
+		toldGeneration: d.Generation, toldPutBack: d.PutBack, sender: senderOf(callerOf(ctx)), modules: d.modules}
+}
+
+// callerOf is who asked for what this process does: the seat call's caller when ctx belongs to one, the
+// caller the controller ran this command for, or the account at the shell.
+func callerOf(ctx context.Context) string {
+	if c := link.CallerIn(ctx); c != "" {
+		return c
+	}
+	return link.Caller()
+}
+
+// senderOf is a send's sender in words: who asked, and the process and build that composed it — the
+// answer issue 234 could not find, because two controllers and a one-shot push were all sending then.
+func senderOf(caller string) string {
+	host, _ := os.Hostname()
+	return fmt.Sprintf("%s (pid %d on %s, build %s)", caller, os.Getpid(), host, version)
+}
+
+// namedModules are the modules a declaration names: its machine's set, less what was left out of it.
+func namedModules(plan catalogue.Resolution, leftOut map[string]string) []string {
+	out := make([]string, 0, len(plan.Modules))
+	for _, m := range plan.Modules {
+		if _, left := leftOut[m.Module]; !left {
+			out = append(out, m.Module)
+		}
+	}
+	sort.Strings(out)
+	return out
+}
+
+// heardGenerationRefusal keeps a machine's refusal of a send for its generation on the send it refused,
+// so S20 names its sender, and says it (novox/hq issue 234).
+//
+// **And raises the mesh's counter past what the machine applied, when the refused send was composed from
+// the counter as it stands**: then the sender's view was not stale — the counter is behind the machine,
+// which is a store put back from a backup — and every send after would be refused for ever. A stale
+// sender's generation is below the counter, and the counter is left alone for it.
+func heardGenerationRefusal(ctx context.Context, inv *inventory.Inventory, report link.Report) {
+	r := report.OlderGeneration
+	if r == nil || inv == nil || report.Node == "" {
+		return
+	}
+	node, err := inv.NodeByName(ctx, report.Node)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "mesh-controller: %s refused a send for its generation, and the machine cannot be read: %v\n",
+			report.Node, err)
+		return
+	}
+	send, found, err := inv.RefusedSend(ctx, node.ID, report.Sequence, r.Applied)
+	switch {
+	case err != nil:
+		fmt.Fprintf(os.Stderr, "mesh-controller: %s refused send %d for its generation, and the refusal could not be "+
+			"kept: %v\n", report.Node, report.Sequence, err)
+	case !found:
+		fmt.Fprintf(os.Stderr, "mesh-controller: %s refused send %d (generation %d; it applied %d), which the mesh "+
+			"has no record of sending\n", report.Node, report.Sequence, r.Generation, r.Applied)
+	default:
+		fmt.Fprintf(os.Stderr, "mesh-controller: %s refused send %d from %s: composed from assignment generation %d, "+
+			"and it applied %d\n", report.Node, report.Sequence, send.Sender, r.Generation, r.Applied)
+	}
+	now, err := inv.AssignmentGeneration(ctx)
+	if err != nil || r.Generation < now || r.Applied < now {
+		return
+	}
+	raised, err := inv.RaiseAssignmentGeneration(ctx, r.Applied)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "mesh-controller: the assignment generation (%d) is behind what %s applied (%d), and "+
+			"could not be raised: %v\n", now, report.Node, r.Applied, err)
+		return
+	}
+	fmt.Fprintf(os.Stderr, "mesh-controller: the assignment generation was %d, behind what %s applied (%d) — a store "+
+		"put back from a backup — and is raised to %d, so the next send is not refused\n", now, report.Node, r.Applied,
+		raised)
+}
+
+// watchGenerationRefusals is S20: every send a machine refused within the hour for the generation it was
+// composed from, naming who sent it (novox/hq issue 234). One per machine, the newest refusal.
+func watchGenerationRefusals(f *signalFacts) []conditions.Observation {
+	seen := map[string]bool{}
+	var out []conditions.Observation
+	for _, s := range f.refusedSends {
+		if s.RefusedAt == nil || f.now.Sub(*s.RefusedAt) > generationRefusalsSaid || seen[s.NodeName] {
+			continue
+		}
+		seen[s.NodeName] = true
+		out = append(out, conditions.Observation{Scope: conditions.ScopeMachine, ID: s.NodeName, Kind: kindOlderGeneration,
+			Machine: s.NodeName, Severity: conditions.Warning,
+			Summary: fmt.Sprintf("%s refused sequence %d from %s: it was composed from assignment generation %d, and "+
+				"%s applied generation %d — a sender composing from a view of the assignments the mesh has moved "+
+				"past; it named %s", s.NodeName, s.Sequence, s.Sender, s.Generation, s.NodeName, s.RefusedApplied,
+				modulesWords(s.Modules)),
+			Said:     fmt.Sprintf("refused at %s", s.RefusedAt.UTC().Format(time.RFC3339)),
+			Headline: fmt.Sprintf("%s refused an out-of-date update", s.NodeName),
+			Explanation: fmt.Sprintf("Something sent %s an update made from an older list of what runs there. %s "+
+				"refused it, so nothing was removed. Nothing for you to do unless it repeats.", s.NodeName, s.NodeName),
+			Resolved: fmt.Sprintf("%s has had no out-of-date update for an hour", s.NodeName)})
+	}
+	return out
+}
+
+// modulesWords is a send's modules as a sentence says them.
+func modulesWords(modules []string) string {
+	if len(modules) == 0 {
+		return "no module the mesh recorded"
+	}
+	return strings.Join(modules, ", ")
+}
+
+// writeLastSend says what a machine was last told, by whom and from which generation (novox/hq issue 234):
+// nothing when the mesh has not recorded a send to it.
+func writeLastSend(ctx context.Context, w io.Writer, inv *inventory.Inventory, node string) error {
+	s, found, err := inv.LastSend(ctx, node)
+	if err != nil || !found {
+		return err
+	}
+	generation := "no generation recorded"
+	if s.Generation > 0 {
+		generation = fmt.Sprintf("assignment generation %d", s.Generation)
+	}
+	kind := "sent"
+	if s.PutBack {
+		kind = "put back"
+	}
+	fmt.Fprintf(w, "%s was last %s sequence %d at %s by %s, composed from %s, naming %s\n", node, kind,
+		s.Sequence, s.SentAt.Local().Format("2006-01-02 15:04:05"), s.Sender, generation, modulesWords(s.Modules))
+	if s.RefusedAt != nil {
+		fmt.Fprintf(w, "  and refused it at %s: it had applied generation %d\n",
+			s.RefusedAt.Local().Format("2006-01-02 15:04:05"), s.RefusedApplied)
+	}
+	return nil
+}