Say each machine's last send in status and node show, and as fields in status --json (novox/hq issue 485)
mesh/delivery superseded: a newer head of the same pull request
mesh/merge-gate pass: builds build-agent, mesh-controller → ace, g14, novox, shanks; no bus step; every machine composes with the change as it did without …
mesh/repo-check pass: its merge-check.sh passed

The send record of issue 234 is now readable only through the controller's
verbs (issue 506), and only plan said it. status says per machine when it was
last sent a declaration, by whom, from which generation and what it answered;
node show says it in full with the refusal's words. A machine's answer is read
from its report only when that report is about the send and came after it.
This commit is contained in:
2026-10-11 20:40:48 +02:00
parent afc7dd8f68
commit fae6bd0a87
9 changed files with 455 additions and 20 deletions
+4
View File
@@ -866,6 +866,10 @@ type answers struct {
// applied, is not well: an assignment somebody made is not made yet, or will not be.
pending []inventory.PendingAssignment
pendingUnread string
// lastSent is every machine's last recorded send, by name, with what the machine answered of it (novox/hq
// issue 485); lastSentUnread why it could not be read.
lastSent map[string]inventory.Send
lastSentUnread string
}
// heldBy is every artifact this mesh has built, for a build that may need one as its base.
+88 -10
View File
@@ -202,21 +202,99 @@ func modulesWords(modules []string) string {
return strings.Join(modules, ", ")
}
// writeLastSend says what a machine was last told, by whom and from which generation (novox/hq issue 234):
// nothing when the mesh has not recorded a send to it.
// writeLastSend says what a machine was last told, by whom and from which generation (novox/hq issue 234), and
// what it answered (issue 485): nothing when the mesh has not recorded a send to it.
func writeLastSend(ctx context.Context, w io.Writer, inv *inventory.Inventory, node string) error {
s, found, err := inv.LastSend(ctx, node)
if err != nil || !found {
return err
}
generation := "no generation recorded"
if s.Generation > 0 {
generation = fmt.Sprintf("assignment generation %d", s.Generation)
}
fmt.Fprintf(w, "%s was last sent sequence %d at %s by %s, composed from %s, naming %s\n", node, s.Sequence, s.SentAt.Local().Format("2006-01-02 15:04:05"), s.Sender, generation, modulesWords(s.Modules))
if s.RefusedAt != nil {
fmt.Fprintf(w, " and refused it at %s: it had applied generation %d\n",
s.RefusedAt.Local().Format("2006-01-02 15:04:05"), s.RefusedApplied)
fmt.Fprintf(w, "%s:\n", node)
for _, line := range lastSendLines(s) {
fmt.Fprintln(w, line)
}
return nil
}
// lastSendLines is a machine's last send in full, as `node show` and `plan` say it (novox/hq issue 485): when,
// by whom, under which lease epoch, from which assignment generation, naming which modules, and what the
// machine answered of it — the refusal in its own words.
func lastSendLines(s inventory.Send) []string {
const stamp = "2006-01-02 15:04:05"
epoch := "under no lease epoch"
if s.Epoch > 0 {
epoch = fmt.Sprintf("under lease epoch %d", s.Epoch)
}
generation := "no assignment generation recorded"
if s.Generation > 0 {
generation = fmt.Sprintf("assignment generation %d", s.Generation)
}
lines := []string{
fmt.Sprintf(" last sent sequence %d at %s", s.Sequence, s.SentAt.Local().Format(stamp)),
fmt.Sprintf(" by %s", s.Sender),
fmt.Sprintf(" %s, composed from %s", epoch, generation),
fmt.Sprintf(" naming %s", modulesWords(s.Modules)),
}
answer := func(words string) { lines = append(lines, " its answer "+words) }
switch a := s.Answer; {
case s.RefusedAt != nil:
// Before the report: a refusal for the generation is kept on the send itself and is the machine's
// answer to this send in particular.
answer(fmt.Sprintf("refused it at %s: it had applied assignment generation %d, newer than this one",
s.RefusedAt.Local().Format(stamp), s.RefusedApplied))
case a.Outcome == "" || a.At == nil:
answer("the machine has not reported on it yet")
case a.Outcome == inventory.OutcomeApplied:
answer("applied it at " + a.At.Local().Format(stamp))
case a.Outcome == inventory.OutcomeRefused:
answer("refused it at " + a.At.Local().Format(stamp) + ", in its words:")
for _, line := range strings.Split(strings.TrimRight(a.Refused, "\n"), "\n") {
lines = append(lines, " "+strings.TrimSpace(line))
}
default:
answer(fmt.Sprintf("%s at %s: %d resource(s) failed — `status` says which", a.Outcome,
a.At.Local().Format(stamp), a.Failed))
}
return lines
}
// sendAnswerWord is what a machine answered of a send, in a word or few, as status says it in one line.
func sendAnswerWord(s inventory.Send) string {
switch a := s.Answer; {
case s.RefusedAt != nil:
return fmt.Sprintf("refused: it had applied generation %d", s.RefusedApplied)
case a.Outcome == "" || a.At == nil:
return "not reported on yet"
case a.Outcome == inventory.OutcomeFailed:
return fmt.Sprintf("failed (%d resource(s))", a.Failed)
default:
return a.Outcome
}
}
// printLastSends says, one line per machine, when it was last sent a declaration, by whom, from which assignment
// generation and what it answered (novox/hq issue 485). `node show <node>` says the send in full.
func printLastSends(nodes []inventory.Node, sends map[string]inventory.Send, unread string) {
if unread != "" {
fmt.Printf("what each machine was last sent could NOT be read: %s\n\n", unread)
return
}
if len(nodes) == 0 {
return
}
fmt.Println("last sent, per machine:")
for _, n := range nodes {
s, found := sends[n.Name]
if !found {
fmt.Printf(" %-12s no send recorded\n", n.Name)
continue
}
generation := "no generation"
if s.Generation > 0 {
generation = fmt.Sprintf("generation %d", s.Generation)
}
fmt.Printf(" %-12s %s by %s, %s — %s\n", n.Name, s.SentAt.Local().Format("2006-01-02 15:04"), s.Sender,
generation, sendAnswerWord(s))
}
fmt.Printf("\n `node show <node>` says a machine's last send in full\n\n")
}
+160
View File
@@ -0,0 +1,160 @@
package main
import (
"encoding/json"
"strings"
"testing"
"time"
"github.com/novox/mesh-controller/internal/inventory"
)
// novox/hq issue 485: the mesh records every send — sender, lease epoch, generation, modules — and only `plan`
// said it. `status` says per machine when it was last sent a declaration and by whom, in one line; `node show`
// says the last send in full and what the machine answered; and `status --json` carries it as fields, since
// the store's reader may not read the controller's tables and these verbs are the only window onto them.
func aLastSend(answer inventory.SendAnswer) inventory.Send {
sent := time.Date(2026, 10, 11, 18, 40, 5, 0, time.Local)
return inventory.Send{NodeName: "laptop", Sequence: 12, Epoch: 57,
Sender: "a one-shot push by jochen at a shell on anchor", Generation: 40, Digest: "d12",
Modules: []string{"docker", "sudo"}, SentAt: sent, Recorded: true, Answer: answer}
}
func TestStatusSaysEachMachinesLastSendInOneLine(t *testing.T) {
at := time.Date(2026, 10, 11, 18, 41, 0, 0, time.Local)
asked := answers{
nodes: []inventory.Node{{Name: "anchor", LastSeen: time.Now()}, {Name: "laptop", LastSeen: time.Now()}},
lastSent: map[string]inventory.Send{"laptop": aLastSend(inventory.SendAnswer{
Outcome: inventory.OutcomeApplied, At: &at})},
}
shown := printed(t, func() error { return printStatus(asked) })
var laptop, anchor []string
for _, line := range strings.Split(shown, "\n") {
fields := strings.Fields(line)
if len(fields) > 0 && fields[0] == "laptop" {
laptop = append(laptop, line)
}
if len(fields) > 0 && fields[0] == "anchor" {
anchor = append(anchor, line)
}
}
if len(laptop) != 1 || !strings.Contains(laptop[0], "2026-10-11 18:40") ||
!strings.Contains(laptop[0], "by a one-shot push by jochen at a shell on anchor") ||
!strings.Contains(laptop[0], "generation 40") || !strings.Contains(laptop[0], "applied") {
t.Fatalf("status does not say laptop's last send in one line:\n%s", shown)
}
if len(anchor) != 1 || !strings.Contains(anchor[0], "no send recorded") {
t.Fatalf("status does not say anchor has no send recorded, in one line:\n%s", shown)
}
}
func TestStatusSaysTheLastSendsCouldNotBeRead(t *testing.T) {
asked := answers{nodes: []inventory.Node{{Name: "laptop", LastSeen: time.Now()}},
lastSentUnread: "the store went away"}
if shown := printed(t, func() error { return printStatus(asked) }); !strings.Contains(shown, "the store went away") {
t.Fatalf("a record of sends that could not be read is not said:\n%s", shown)
}
if asked.well() {
t.Error("a mesh whose sends could not be read is called well")
}
}
func TestNodeShowSaysTheLastSendInFull(t *testing.T) {
at := time.Date(2026, 10, 11, 18, 41, 0, 0, time.Local)
refused := strings.Join(lastSendLines(aLastSend(inventory.SendAnswer{Outcome: inventory.OutcomeRefused,
Refused: "unknown field \"generation\"\nsecond line", At: &at})), "\n")
for _, want := range []string{"sequence 12", "2026-10-11 18:40:05", "a one-shot push by jochen at a shell on anchor",
"lease epoch 57", "generation 40", "docker, sudo", "refused", "unknown field \"generation\""} {
if !strings.Contains(refused, want) {
t.Errorf("node show's last send does not say %q:\n%s", want, refused)
}
}
if applied := strings.Join(lastSendLines(aLastSend(inventory.SendAnswer{Outcome: inventory.OutcomeApplied,
At: &at})), "\n"); !strings.Contains(applied, "applied it at 2026-10-11 18:41:00") {
t.Errorf("an applied send is not said applied:\n%s", applied)
}
if waiting := strings.Join(lastSendLines(aLastSend(inventory.SendAnswer{})), "\n"); !strings.Contains(waiting,
"not reported on it yet") {
t.Errorf("a send not reported on is not said so:\n%s", waiting)
}
stale := aLastSend(inventory.SendAnswer{})
when := at
stale.RefusedAt, stale.RefusedApplied = &when, 44
if s := strings.Join(lastSendLines(stale), "\n"); !strings.Contains(s, "refused it at 2026-10-11 18:41:00") ||
!strings.Contains(s, "generation 44") {
t.Errorf("a send refused for its generation is not said so:\n%s", s)
}
}
func TestStatusJSONCarriesEachMachinesLastSend(t *testing.T) {
at := time.Date(2026, 10, 11, 18, 41, 0, 0, time.UTC)
asked := answers{nodes: []inventory.Node{{Name: "anchor"}, {Name: "laptop"}},
lastSent: map[string]inventory.Send{"laptop": aLastSend(inventory.SendAnswer{Outcome: inventory.OutcomeRefused,
Refused: "unknown field", At: &at})}}
body, err := statusAsJSON(asked)
if err != nil {
t.Fatal(err)
}
var got struct {
LastSent []map[string]any `json:"lastSent"`
}
if err := json.Unmarshal(body, &got); err != nil {
t.Fatal(err)
}
if len(got.LastSent) != 1 {
t.Fatalf("status --json carries %d last sends, want laptop's only:\n%s", len(got.LastSent), body)
}
s := got.LastSent[0]
if s["node"] != "laptop" || s["sequence"] != float64(12) || s["epoch"] != float64(57) ||
s["generation"] != float64(40) || s["sender"] != "a one-shot push by jochen at a shell on anchor" ||
s["digest"] != "d12" || s["sentAt"] == nil || s["answer"] != "refused" || s["refused"] != "unknown field" ||
s["answeredAt"] == nil {
t.Fatalf("laptop's last send reads %v", s)
}
if modules, _ := s["modules"].([]any); len(modules) != 2 || modules[0] != "docker" {
t.Fatalf("the modules it named read %v", s["modules"])
}
}
// And through the store: a send recorded and refused by the machine is what `node show` and `status --json`
// read back.
func TestNodeShowAndStatusReadTheLastSendFromTheStore(t *testing.T) {
open := aMesh(t)
ctx := t.Context()
record, err := open.inventory.NodeByName(ctx, "laptop")
if err != nil {
t.Fatal(err)
}
if err := open.inventory.RecordSentWith(ctx, record.ID, "d12", nil, 57, 40, inventory.Send{Sequence: 12,
Epoch: 57, Sender: "a one-shot push by jochen at a shell on anchor", Generation: 40, Digest: "d12",
Modules: []string{"docker", "sudo"}}); err != nil {
t.Fatal(err)
}
if _, err := open.inventory.RecordDoing(ctx, record.ID, inventory.Doing{Outcome: inventory.OutcomeRefused,
Declared: "d12", Refused: "unknown field \"generation\""}); err != nil {
t.Fatal(err)
}
shown := printed(t, func() error { return showNode(ctx, open, "laptop") })
for _, want := range []string{"a one-shot push by jochen at a shell on anchor", "lease epoch 57", "generation 40",
"docker, sudo", "unknown field \"generation\""} {
if !strings.Contains(shown, want) {
t.Errorf("node show laptop does not say %q:\n%s", want, shown)
}
}
body := printed(t, func() error { return statusFor(ctx, open, true) })
var got struct {
LastSent []struct {
Node string `json:"node"`
Sender string `json:"sender"`
Answer string `json:"answer"`
} `json:"lastSent"`
}
if err := json.Unmarshal([]byte(body), &got); err != nil {
t.Fatalf("%v:\n%s", err, body)
}
if len(got.LastSent) != 1 || got.LastSent[0].Node != "laptop" || got.LastSent[0].Answer != "refused" ||
got.LastSent[0].Sender != "a one-shot push by jochen at a shell on anchor" {
t.Fatalf("status --json's last sends read %+v", got.LastSent)
}
}
+13
View File
@@ -804,6 +804,19 @@ func showNode(ctx context.Context, stored *stores, name string) error {
if err := showMode(ctx, inv, node); err != nil {
return err
}
// What it was last sent, by whom, under which epoch, from which generation, naming which modules, and what
// it answered (novox/hq issue 485): the record of sends is read through this verb, never the store's reader.
// Unreadable is said, not taken for none.
switch last, found, err := inv.LastSend(ctx, name); {
case err != nil:
fmt.Printf(" what it was last sent could NOT be read: %v\n", err)
case !found:
fmt.Printf(" last sent no send recorded since the mesh began keeping them\n")
default:
for _, line := range lastSendLines(last) {
fmt.Println(line)
}
}
// Whom agents run as here, and whether that account can become root without a person (ADR 0266).
for _, line := range agentAccountLines(ctx, inv, node) {
fmt.Println(line)
+45
View File
@@ -109,6 +109,33 @@ type meshStatus struct {
// why they could not be read.
Pending []pendingStatus `json:"pending,omitempty"`
PendingUnread string `json:"pendingUnread,omitempty"`
// LastSent is every machine's last recorded send — who sent it, under which lease epoch, from which
// assignment generation, naming which modules — and what the machine answered of it (novox/hq issue 485).
// A machine the mesh has recorded no send to is absent. LastSentUnread says why it could not be read.
LastSent []machineLastSent `json:"lastSent"`
LastSentUnread string `json:"lastSentUnread,omitempty"`
}
// machineLastSent is one machine's last send as status says it.
type machineLastSent struct {
Node string `json:"node"`
Sequence int64 `json:"sequence"`
SentAt time.Time `json:"sentAt"`
Sender string `json:"sender"`
Epoch int64 `json:"epoch"`
Generation int64 `json:"generation"`
Digest string `json:"digest"`
Modules []string `json:"modules"`
// Answer is what the machine said of it: applied, failed or refused, as its report has it; empty when it
// has not reported on it. Refused is its words when it refused it whole, Failed how many resources failed.
Answer string `json:"answer"`
Refused string `json:"refused,omitempty"`
Failed int `json:"failed,omitempty"`
AnsweredAt *time.Time `json:"answeredAt,omitempty"`
// RefusedForGeneration is when the machine refused it for coming from an older assignment generation than
// it applied, and AppliedGeneration that generation; absent when it did not.
RefusedForGeneration *time.Time `json:"refusedForGeneration,omitempty"`
AppliedGeneration int64 `json:"appliedGeneration,omitempty"`
}
// pendingStatus is one pending assignment as status says it.
@@ -267,6 +294,7 @@ func statusAsJSON(asked answers) ([]byte, error) {
out.Pending = append(out.Pending, pendingStatus{Node: p.Node, Module: p.Module, State: p.State,
Build: p.Build, Repository: p.Repository, Path: p.Path, Since: p.Since, Settled: p.Settled, Note: p.Note})
}
out.LastSent, out.LastSentUnread = lastSentStatus(asked.lastSent), asked.lastSentUnread
for name := range asked.refused {
out.Unresolved = append(out.Unresolved, machineUnresolved{
Node: name, Problem: asked.refused[name]})
@@ -323,3 +351,20 @@ func say(value any) error {
fmt.Println(string(body))
return nil
}
// lastSentStatus is every machine's last send as status --json says it, by machine name (novox/hq issue 485).
func lastSentStatus(sends map[string]inventory.Send) []machineLastSent {
out := []machineLastSent{}
for _, s := range sends {
modules := s.Modules
if modules == nil {
modules = []string{}
}
out = append(out, machineLastSent{Node: s.NodeName, Sequence: s.Sequence, SentAt: s.SentAt, Sender: s.Sender,
Epoch: s.Epoch, Generation: s.Generation, Digest: s.Digest, Modules: modules, Answer: s.Answer.Outcome,
Refused: s.Answer.Refused, Failed: s.Answer.Failed, AnsweredAt: s.Answer.At,
RefusedForGeneration: s.RefusedAt, AppliedGeneration: s.RefusedApplied})
}
sort.Slice(out, func(i, j int) bool { return out[i].Node < out[j].Node })
return out
}
+12 -1
View File
@@ -341,6 +341,10 @@ func printStatus(asked answers) error {
"lists them, and each is in its condition's tried\n\n", asked.heals.Acts, asked.heals.Escalated)
}
// What each machine was last sent and by whom, one line each (novox/hq issue 485). Not a fault, so it does not
// break "all well"; a record that could not be read does.
printLastSends(nodes, asked.lastSent, asked.lastSentUnread)
if adopted := adoptedNodes(nodes); len(adopted) > 0 {
// Said, because nothing forces the flip: a node left adopted is visible here rather than
// read as converged (novox/hq ADR 0100). Not a fault, so it does not break "all well".
@@ -485,6 +489,12 @@ func theThreeQuestions(ctx context.Context, open *stores) (answers, error) {
out.pendingUnread = err.Error()
err = nil
}
// And what each machine was last sent, by whom (novox/hq issue 485): the record of sends is read through
// the controller's verbs alone, the store's reader being shut out of its tables. Unreadable is said.
if out.lastSent, err = inv.LastSends(ctx); err != nil {
out.lastSentUnread = err.Error()
err = nil
}
out.plans, err = inv.RecentPlans(ctx, 5)
if err != nil {
return answers{}, err
@@ -617,7 +627,8 @@ func (a answers) well() bool {
return len(a.wrong) == 0 && len(a.quiet) == 0 && len(a.behind) == 0 &&
len(a.waiting) == 0 && len(a.refused) == 0 && a.network == "" && len(a.untaken) == 0 && len(a.leftOut) == 0 &&
len(a.filtered) == 0 && len(a.unheld) == 0 && len(a.overflowing) == 0 &&
len(a.conditions) == 0 && a.conditionsUnread == "" && a.pendingUnread == "" && !pendingOpen(a.pending)
len(a.conditions) == 0 && a.conditionsUnread == "" && a.pendingUnread == "" && !pendingOpen(a.pending) &&
a.lastSentUnread == ""
}
// pendingOpen is whether any pending assignment is still to be made. One that ended without being made is