Merge remote-tracking branch 'origin/main' into feat/459-a-stop-re-walks-the-others

# Conflicts:
#	testdata/store-tests

cmd/mesh-controller/build.go

@@ -866,6 +866,10 @@ type answers struct {
 	// applied, is not well: an assignment somebody made is not made yet, or will not be.
 	pending       []inventory.PendingAssignment
 	pendingUnread string
+	// lastSent is every machine's last recorded send, by name, with what the machine answered of it (novox/hq
+	// issue 485); lastSentUnread why it could not be read.
+	lastSent       map[string]inventory.Send
+	lastSentUnread string
 }
 
 // heldBy is every artifact this mesh has built, for a build that may need one as its base.

cmd/mesh-controller/generation.go

@@ -202,21 +202,99 @@ func modulesWords(modules []string) string {
 	return strings.Join(modules, ", ")
 }
 
-// writeLastSend says what a machine was last told, by whom and from which generation (novox/hq issue 234):
-// nothing when the mesh has not recorded a send to it.
+// writeLastSend says what a machine was last told, by whom and from which generation (novox/hq issue 234), and
+// what it answered (issue 485): nothing when the mesh has not recorded a send to it.
 func writeLastSend(ctx context.Context, w io.Writer, inv *inventory.Inventory, node string) error {
 	s, found, err := inv.LastSend(ctx, node)
 	if err != nil || !found {
 		return err
 	}
-	generation := "no generation recorded"
-	if s.Generation > 0 {
-		generation = fmt.Sprintf("assignment generation %d", s.Generation)
-	}
-	fmt.Fprintf(w, "%s was last sent sequence %d at %s by %s, composed from %s, naming %s\n", node, s.Sequence, s.SentAt.Local().Format("2006-01-02 15:04:05"), s.Sender, generation, modulesWords(s.Modules))
-	if s.RefusedAt != nil {
-		fmt.Fprintf(w, "  and refused it at %s: it had applied generation %d\n",
-			s.RefusedAt.Local().Format("2006-01-02 15:04:05"), s.RefusedApplied)
+	fmt.Fprintf(w, "%s:\n", node)
+	for _, line := range lastSendLines(s) {
+		fmt.Fprintln(w, line)
 	}
 	return nil
 }
+
+// lastSendLines is a machine's last send in full, as `node show` and `plan` say it (novox/hq issue 485): when,
+// by whom, under which lease epoch, from which assignment generation, naming which modules, and what the
+// machine answered of it — the refusal in its own words.
+func lastSendLines(s inventory.Send) []string {
+	const stamp = "2006-01-02 15:04:05"
+	epoch := "under no lease epoch"
+	if s.Epoch > 0 {
+		epoch = fmt.Sprintf("under lease epoch %d", s.Epoch)
+	}
+	generation := "no assignment generation recorded"
+	if s.Generation > 0 {
+		generation = fmt.Sprintf("assignment generation %d", s.Generation)
+	}
+	lines := []string{
+		fmt.Sprintf("  last sent         sequence %d at %s", s.Sequence, s.SentAt.Local().Format(stamp)),
+		fmt.Sprintf("                    by %s", s.Sender),
+		fmt.Sprintf("                    %s, composed from %s", epoch, generation),
+		fmt.Sprintf("                    naming %s", modulesWords(s.Modules)),
+	}
+	answer := func(words string) { lines = append(lines, "  its answer        "+words) }
+	switch a := s.Answer; {
+	case s.RefusedAt != nil:
+		// Before the report: a refusal for the generation is kept on the send itself and is the machine's
+		// answer to this send in particular.
+		answer(fmt.Sprintf("refused it at %s: it had applied assignment generation %d, newer than this one",
+			s.RefusedAt.Local().Format(stamp), s.RefusedApplied))
+	case a.Outcome == "" || a.At == nil:
+		answer("the machine has not reported on it yet")
+	case a.Outcome == inventory.OutcomeApplied:
+		answer("applied it at " + a.At.Local().Format(stamp))
+	case a.Outcome == inventory.OutcomeRefused:
+		answer("refused it at " + a.At.Local().Format(stamp) + ", in its words:")
+		for _, line := range strings.Split(strings.TrimRight(a.Refused, "\n"), "\n") {
+			lines = append(lines, "                      "+strings.TrimSpace(line))
+		}
+	default:
+		answer(fmt.Sprintf("%s at %s: %d resource(s) failed — `status` says which", a.Outcome,
+			a.At.Local().Format(stamp), a.Failed))
+	}
+	return lines
+}
+
+// sendAnswerWord is what a machine answered of a send, in a word or few, as status says it in one line.
+func sendAnswerWord(s inventory.Send) string {
+	switch a := s.Answer; {
+	case s.RefusedAt != nil:
+		return fmt.Sprintf("refused: it had applied generation %d", s.RefusedApplied)
+	case a.Outcome == "" || a.At == nil:
+		return "not reported on yet"
+	case a.Outcome == inventory.OutcomeFailed:
+		return fmt.Sprintf("failed (%d resource(s))", a.Failed)
+	default:
+		return a.Outcome
+	}
+}
+
+// printLastSends says, one line per machine, when it was last sent a declaration, by whom, from which assignment
+// generation and what it answered (novox/hq issue 485). `node show <node>` says the send in full.
+func printLastSends(nodes []inventory.Node, sends map[string]inventory.Send, unread string) {
+	if unread != "" {
+		fmt.Printf("what each machine was last sent could NOT be read: %s\n\n", unread)
+		return
+	}
+	if len(nodes) == 0 {
+		return
+	}
+	fmt.Println("last sent, per machine:")
+	for _, n := range nodes {
+		s, found := sends[n.Name]
+		if !found {
+			fmt.Printf("  %-12s no send recorded\n", n.Name)
+			continue
+		}
+		generation := "no generation"
+		if s.Generation > 0 {
+			generation = fmt.Sprintf("generation %d", s.Generation)
+		}
+		fmt.Printf("  %-12s %s by %s, %s — %s\n", n.Name, s.SentAt.Local().Format("2006-01-02 15:04"), s.Sender,
+			generation, sendAnswerWord(s))
+	}
+	fmt.Printf("\n  `node show <node>` says a machine's last send in full\n\n")
+}

cmd/mesh-controller/last_send_test.go

@@ -0,0 +1,160 @@
+package main
+
+import (
+	"encoding/json"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/novox/mesh-controller/internal/inventory"
+)
+
+// novox/hq issue 485: the mesh records every send — sender, lease epoch, generation, modules — and only `plan`
+// said it. `status` says per machine when it was last sent a declaration and by whom, in one line; `node show`
+// says the last send in full and what the machine answered; and `status --json` carries it as fields, since
+// the store's reader may not read the controller's tables and these verbs are the only window onto them.
+
+func aLastSend(answer inventory.SendAnswer) inventory.Send {
+	sent := time.Date(2026, 10, 11, 18, 40, 5, 0, time.Local)
+	return inventory.Send{NodeName: "laptop", Sequence: 12, Epoch: 57,
+		Sender: "a one-shot push by jochen at a shell on anchor", Generation: 40, Digest: "d12",
+		Modules: []string{"docker", "sudo"}, SentAt: sent, Recorded: true, Answer: answer}
+}
+
+func TestStatusSaysEachMachinesLastSendInOneLine(t *testing.T) {
+	at := time.Date(2026, 10, 11, 18, 41, 0, 0, time.Local)
+	asked := answers{
+		nodes: []inventory.Node{{Name: "anchor", LastSeen: time.Now()}, {Name: "laptop", LastSeen: time.Now()}},
+		lastSent: map[string]inventory.Send{"laptop": aLastSend(inventory.SendAnswer{
+			Outcome: inventory.OutcomeApplied, At: &at})},
+	}
+	shown := printed(t, func() error { return printStatus(asked) })
+	var laptop, anchor []string
+	for _, line := range strings.Split(shown, "\n") {
+		fields := strings.Fields(line)
+		if len(fields) > 0 && fields[0] == "laptop" {
+			laptop = append(laptop, line)
+		}
+		if len(fields) > 0 && fields[0] == "anchor" {
+			anchor = append(anchor, line)
+		}
+	}
+	if len(laptop) != 1 || !strings.Contains(laptop[0], "2026-10-11 18:40") ||
+		!strings.Contains(laptop[0], "by a one-shot push by jochen at a shell on anchor") ||
+		!strings.Contains(laptop[0], "generation 40") || !strings.Contains(laptop[0], "applied") {
+		t.Fatalf("status does not say laptop's last send in one line:\n%s", shown)
+	}
+	if len(anchor) != 1 || !strings.Contains(anchor[0], "no send recorded") {
+		t.Fatalf("status does not say anchor has no send recorded, in one line:\n%s", shown)
+	}
+}
+
+func TestStatusSaysTheLastSendsCouldNotBeRead(t *testing.T) {
+	asked := answers{nodes: []inventory.Node{{Name: "laptop", LastSeen: time.Now()}},
+		lastSentUnread: "the store went away"}
+	if shown := printed(t, func() error { return printStatus(asked) }); !strings.Contains(shown, "the store went away") {
+		t.Fatalf("a record of sends that could not be read is not said:\n%s", shown)
+	}
+	if asked.well() {
+		t.Error("a mesh whose sends could not be read is called well")
+	}
+}
+
+func TestNodeShowSaysTheLastSendInFull(t *testing.T) {
+	at := time.Date(2026, 10, 11, 18, 41, 0, 0, time.Local)
+	refused := strings.Join(lastSendLines(aLastSend(inventory.SendAnswer{Outcome: inventory.OutcomeRefused,
+		Refused: "unknown field \"generation\"\nsecond line", At: &at})), "\n")
+	for _, want := range []string{"sequence 12", "2026-10-11 18:40:05", "a one-shot push by jochen at a shell on anchor",
+		"lease epoch 57", "generation 40", "docker, sudo", "refused", "unknown field \"generation\""} {
+		if !strings.Contains(refused, want) {
+			t.Errorf("node show's last send does not say %q:\n%s", want, refused)
+		}
+	}
+	if applied := strings.Join(lastSendLines(aLastSend(inventory.SendAnswer{Outcome: inventory.OutcomeApplied,
+		At: &at})), "\n"); !strings.Contains(applied, "applied it at 2026-10-11 18:41:00") {
+		t.Errorf("an applied send is not said applied:\n%s", applied)
+	}
+	if waiting := strings.Join(lastSendLines(aLastSend(inventory.SendAnswer{})), "\n"); !strings.Contains(waiting,
+		"not reported on it yet") {
+		t.Errorf("a send not reported on is not said so:\n%s", waiting)
+	}
+	stale := aLastSend(inventory.SendAnswer{})
+	when := at
+	stale.RefusedAt, stale.RefusedApplied = &when, 44
+	if s := strings.Join(lastSendLines(stale), "\n"); !strings.Contains(s, "refused it at 2026-10-11 18:41:00") ||
+		!strings.Contains(s, "generation 44") {
+		t.Errorf("a send refused for its generation is not said so:\n%s", s)
+	}
+}
+
+func TestStatusJSONCarriesEachMachinesLastSend(t *testing.T) {
+	at := time.Date(2026, 10, 11, 18, 41, 0, 0, time.UTC)
+	asked := answers{nodes: []inventory.Node{{Name: "anchor"}, {Name: "laptop"}},
+		lastSent: map[string]inventory.Send{"laptop": aLastSend(inventory.SendAnswer{Outcome: inventory.OutcomeRefused,
+			Refused: "unknown field", At: &at})}}
+	body, err := statusAsJSON(asked)
+	if err != nil {
+		t.Fatal(err)
+	}
+	var got struct {
+		LastSent []map[string]any `json:"lastSent"`
+	}
+	if err := json.Unmarshal(body, &got); err != nil {
+		t.Fatal(err)
+	}
+	if len(got.LastSent) != 1 {
+		t.Fatalf("status --json carries %d last sends, want laptop's only:\n%s", len(got.LastSent), body)
+	}
+	s := got.LastSent[0]
+	if s["node"] != "laptop" || s["sequence"] != float64(12) || s["epoch"] != float64(57) ||
+		s["generation"] != float64(40) || s["sender"] != "a one-shot push by jochen at a shell on anchor" ||
+		s["digest"] != "d12" || s["sentAt"] == nil || s["answer"] != "refused" || s["refused"] != "unknown field" ||
+		s["answeredAt"] == nil {
+		t.Fatalf("laptop's last send reads %v", s)
+	}
+	if modules, _ := s["modules"].([]any); len(modules) != 2 || modules[0] != "docker" {
+		t.Fatalf("the modules it named read %v", s["modules"])
+	}
+}
+
+// And through the store: a send recorded and refused by the machine is what `node show` and `status --json`
+// read back.
+func TestNodeShowAndStatusReadTheLastSendFromTheStore(t *testing.T) {
+	open := aMesh(t)
+	ctx := t.Context()
+	record, err := open.inventory.NodeByName(ctx, "laptop")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := open.inventory.RecordSentWith(ctx, record.ID, "d12", nil, 57, 40, inventory.Send{Sequence: 12,
+		Epoch: 57, Sender: "a one-shot push by jochen at a shell on anchor", Generation: 40, Digest: "d12",
+		Modules: []string{"docker", "sudo"}}); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := open.inventory.RecordDoing(ctx, record.ID, inventory.Doing{Outcome: inventory.OutcomeRefused,
+		Declared: "d12", Refused: "unknown field \"generation\""}); err != nil {
+		t.Fatal(err)
+	}
+	shown := printed(t, func() error { return showNode(ctx, open, "laptop") })
+	for _, want := range []string{"a one-shot push by jochen at a shell on anchor", "lease epoch 57", "generation 40",
+		"docker, sudo", "unknown field \"generation\""} {
+		if !strings.Contains(shown, want) {
+			t.Errorf("node show laptop does not say %q:\n%s", want, shown)
+		}
+	}
+	body := printed(t, func() error { return statusFor(ctx, open, true) })
+	var got struct {
+		LastSent []struct {
+			Node   string `json:"node"`
+			Sender string `json:"sender"`
+			Answer string `json:"answer"`
+		} `json:"lastSent"`
+	}
+	if err := json.Unmarshal([]byte(body), &got); err != nil {
+		t.Fatalf("%v:\n%s", err, body)
+	}
+	if len(got.LastSent) != 1 || got.LastSent[0].Node != "laptop" || got.LastSent[0].Answer != "refused" ||
+		got.LastSent[0].Sender != "a one-shot push by jochen at a shell on anchor" {
+		t.Fatalf("status --json's last sends read %+v", got.LastSent)
+	}
+}

cmd/mesh-controller/nodes.go

@@ -804,6 +804,19 @@ func showNode(ctx context.Context, stored *stores, name string) error {
 	if err := showMode(ctx, inv, node); err != nil {
 		return err
 	}
+	// What it was last sent, by whom, under which epoch, from which generation, naming which modules, and what
+	// it answered (novox/hq issue 485): the record of sends is read through this verb, never the store's reader.
+	// Unreadable is said, not taken for none.
+	switch last, found, err := inv.LastSend(ctx, name); {
+	case err != nil:
+		fmt.Printf("  what it was last sent could NOT be read: %v\n", err)
+	case !found:
+		fmt.Printf("  last sent         no send recorded since the mesh began keeping them\n")
+	default:
+		for _, line := range lastSendLines(last) {
+			fmt.Println(line)
+		}
+	}
 	// Whom agents run as here, and whether that account can become root without a person (ADR 0266).
 	for _, line := range agentAccountLines(ctx, inv, node) {
 		fmt.Println(line)

cmd/mesh-controller/readable.go

@@ -109,6 +109,33 @@ type meshStatus struct {
 	// why they could not be read.
 	Pending       []pendingStatus `json:"pending,omitempty"`
 	PendingUnread string          `json:"pendingUnread,omitempty"`
+	// LastSent is every machine's last recorded send — who sent it, under which lease epoch, from which
+	// assignment generation, naming which modules — and what the machine answered of it (novox/hq issue 485).
+	// A machine the mesh has recorded no send to is absent. LastSentUnread says why it could not be read.
+	LastSent       []machineLastSent `json:"lastSent"`
+	LastSentUnread string            `json:"lastSentUnread,omitempty"`
+}
+
+// machineLastSent is one machine's last send as status says it.
+type machineLastSent struct {
+	Node       string    `json:"node"`
+	Sequence   int64     `json:"sequence"`
+	SentAt     time.Time `json:"sentAt"`
+	Sender     string    `json:"sender"`
+	Epoch      int64     `json:"epoch"`
+	Generation int64     `json:"generation"`
+	Digest     string    `json:"digest"`
+	Modules    []string  `json:"modules"`
+	// Answer is what the machine said of it: applied, failed or refused, as its report has it; empty when it
+	// has not reported on it. Refused is its words when it refused it whole, Failed how many resources failed.
+	Answer     string     `json:"answer"`
+	Refused    string     `json:"refused,omitempty"`
+	Failed     int        `json:"failed,omitempty"`
+	AnsweredAt *time.Time `json:"answeredAt,omitempty"`
+	// RefusedForGeneration is when the machine refused it for coming from an older assignment generation than
+	// it applied, and AppliedGeneration that generation; absent when it did not.
+	RefusedForGeneration *time.Time `json:"refusedForGeneration,omitempty"`
+	AppliedGeneration    int64      `json:"appliedGeneration,omitempty"`
 }
 
 // pendingStatus is one pending assignment as status says it.
@@ -267,6 +294,7 @@ func statusAsJSON(asked answers) ([]byte, error) {
 		out.Pending = append(out.Pending, pendingStatus{Node: p.Node, Module: p.Module, State: p.State,
 			Build: p.Build, Repository: p.Repository, Path: p.Path, Since: p.Since, Settled: p.Settled, Note: p.Note})
 	}
+	out.LastSent, out.LastSentUnread = lastSentStatus(asked.lastSent), asked.lastSentUnread
 	for name := range asked.refused {
 		out.Unresolved = append(out.Unresolved, machineUnresolved{
 			Node: name, Problem: asked.refused[name]})
@@ -323,3 +351,20 @@ func say(value any) error {
 	fmt.Println(string(body))
 	return nil
 }
+
+// lastSentStatus is every machine's last send as status --json says it, by machine name (novox/hq issue 485).
+func lastSentStatus(sends map[string]inventory.Send) []machineLastSent {
+	out := []machineLastSent{}
+	for _, s := range sends {
+		modules := s.Modules
+		if modules == nil {
+			modules = []string{}
+		}
+		out = append(out, machineLastSent{Node: s.NodeName, Sequence: s.Sequence, SentAt: s.SentAt, Sender: s.Sender,
+			Epoch: s.Epoch, Generation: s.Generation, Digest: s.Digest, Modules: modules, Answer: s.Answer.Outcome,
+			Refused: s.Answer.Refused, Failed: s.Answer.Failed, AnsweredAt: s.Answer.At,
+			RefusedForGeneration: s.RefusedAt, AppliedGeneration: s.RefusedApplied})
+	}
+	sort.Slice(out, func(i, j int) bool { return out[i].Node < out[j].Node })
+	return out
+}

cmd/mesh-controller/status.go

@@ -341,6 +341,10 @@ func printStatus(asked answers) error {
 			"lists them, and each is in its condition's tried\n\n", asked.heals.Acts, asked.heals.Escalated)
 	}
 
+	// What each machine was last sent and by whom, one line each (novox/hq issue 485). Not a fault, so it does not
+	// break "all well"; a record that could not be read does.
+	printLastSends(nodes, asked.lastSent, asked.lastSentUnread)
+
 	if adopted := adoptedNodes(nodes); len(adopted) > 0 {
 		// Said, because nothing forces the flip: a node left adopted is visible here rather than
 		// read as converged (novox/hq ADR 0100). Not a fault, so it does not break "all well".
@@ -485,6 +489,12 @@ func theThreeQuestions(ctx context.Context, open *stores) (answers, error) {
 		out.pendingUnread = err.Error()
 		err = nil
 	}
+	// And what each machine was last sent, by whom (novox/hq issue 485): the record of sends is read through
+	// the controller's verbs alone, the store's reader being shut out of its tables. Unreadable is said.
+	if out.lastSent, err = inv.LastSends(ctx); err != nil {
+		out.lastSentUnread = err.Error()
+		err = nil
+	}
 	out.plans, err = inv.RecentPlans(ctx, 5)
 	if err != nil {
 		return answers{}, err
@@ -617,7 +627,8 @@ func (a answers) well() bool {
 	return len(a.wrong) == 0 && len(a.quiet) == 0 && len(a.behind) == 0 &&
 		len(a.waiting) == 0 && len(a.refused) == 0 && a.network == "" && len(a.untaken) == 0 && len(a.leftOut) == 0 &&
 		len(a.filtered) == 0 && len(a.unheld) == 0 && len(a.overflowing) == 0 &&
-		len(a.conditions) == 0 && a.conditionsUnread == "" && a.pendingUnread == "" && !pendingOpen(a.pending)
+		len(a.conditions) == 0 && a.conditionsUnread == "" && a.pendingUnread == "" && !pendingOpen(a.pending) &&
+		a.lastSentUnread == ""
 }
 
 // pendingOpen is whether any pending assignment is still to be made. One that ended without being made is