Merge remote-tracking branch 'origin/main' into feat/459-a-stop-re-walks-the-others
mesh/merge-gate pass: builds build-agent, mesh-controller → ace, g14, novox, shanks; no bus step; every machine composes with the change as it did without …
mesh/repo-check pass: its merge-check.sh passed
mesh/delivery-group group feat/459-a-stop-re-walks-the-others delivering: 1 of 3 delivered
mesh/delivery delivered

# Conflicts:
#	testdata/store-tests
This commit is contained in:
2026-10-11 21:06:47 +02:00
9 changed files with 528 additions and 20 deletions
+4
View File
@@ -866,6 +866,10 @@ type answers struct {
// applied, is not well: an assignment somebody made is not made yet, or will not be.
pending []inventory.PendingAssignment
pendingUnread string
// lastSent is every machine's last recorded send, by name, with what the machine answered of it (novox/hq
// issue 485); lastSentUnread why it could not be read.
lastSent map[string]inventory.Send
lastSentUnread string
}
// heldBy is every artifact this mesh has built, for a build that may need one as its base.
+88 -10
View File
@@ -202,21 +202,99 @@ func modulesWords(modules []string) string {
return strings.Join(modules, ", ")
}
// writeLastSend says what a machine was last told, by whom and from which generation (novox/hq issue 234):
// nothing when the mesh has not recorded a send to it.
// writeLastSend says what a machine was last told, by whom and from which generation (novox/hq issue 234), and
// what it answered (issue 485): nothing when the mesh has not recorded a send to it.
func writeLastSend(ctx context.Context, w io.Writer, inv *inventory.Inventory, node string) error {
s, found, err := inv.LastSend(ctx, node)
if err != nil || !found {
return err
}
generation := "no generation recorded"
if s.Generation > 0 {
generation = fmt.Sprintf("assignment generation %d", s.Generation)
}
fmt.Fprintf(w, "%s was last sent sequence %d at %s by %s, composed from %s, naming %s\n", node, s.Sequence, s.SentAt.Local().Format("2006-01-02 15:04:05"), s.Sender, generation, modulesWords(s.Modules))
if s.RefusedAt != nil {
fmt.Fprintf(w, " and refused it at %s: it had applied generation %d\n",
s.RefusedAt.Local().Format("2006-01-02 15:04:05"), s.RefusedApplied)
fmt.Fprintf(w, "%s:\n", node)
for _, line := range lastSendLines(s) {
fmt.Fprintln(w, line)
}
return nil
}
// lastSendLines is a machine's last send in full, as `node show` and `plan` say it (novox/hq issue 485): when,
// by whom, under which lease epoch, from which assignment generation, naming which modules, and what the
// machine answered of it — the refusal in its own words.
func lastSendLines(s inventory.Send) []string {
const stamp = "2006-01-02 15:04:05"
epoch := "under no lease epoch"
if s.Epoch > 0 {
epoch = fmt.Sprintf("under lease epoch %d", s.Epoch)
}
generation := "no assignment generation recorded"
if s.Generation > 0 {
generation = fmt.Sprintf("assignment generation %d", s.Generation)
}
lines := []string{
fmt.Sprintf(" last sent sequence %d at %s", s.Sequence, s.SentAt.Local().Format(stamp)),
fmt.Sprintf(" by %s", s.Sender),
fmt.Sprintf(" %s, composed from %s", epoch, generation),
fmt.Sprintf(" naming %s", modulesWords(s.Modules)),
}
answer := func(words string) { lines = append(lines, " its answer "+words) }
switch a := s.Answer; {
case s.RefusedAt != nil:
// Before the report: a refusal for the generation is kept on the send itself and is the machine's
// answer to this send in particular.
answer(fmt.Sprintf("refused it at %s: it had applied assignment generation %d, newer than this one",
s.RefusedAt.Local().Format(stamp), s.RefusedApplied))
case a.Outcome == "" || a.At == nil:
answer("the machine has not reported on it yet")
case a.Outcome == inventory.OutcomeApplied:
answer("applied it at " + a.At.Local().Format(stamp))
case a.Outcome == inventory.OutcomeRefused:
answer("refused it at " + a.At.Local().Format(stamp) + ", in its words:")
for _, line := range strings.Split(strings.TrimRight(a.Refused, "\n"), "\n") {
lines = append(lines, " "+strings.TrimSpace(line))
}
default:
answer(fmt.Sprintf("%s at %s: %d resource(s) failed — `status` says which", a.Outcome,
a.At.Local().Format(stamp), a.Failed))
}
return lines
}
// sendAnswerWord is what a machine answered of a send, in a word or few, as status says it in one line.
func sendAnswerWord(s inventory.Send) string {
switch a := s.Answer; {
case s.RefusedAt != nil:
return fmt.Sprintf("refused: it had applied generation %d", s.RefusedApplied)
case a.Outcome == "" || a.At == nil:
return "not reported on yet"
case a.Outcome == inventory.OutcomeFailed:
return fmt.Sprintf("failed (%d resource(s))", a.Failed)
default:
return a.Outcome
}
}
// printLastSends says, one line per machine, when it was last sent a declaration, by whom, from which assignment
// generation and what it answered (novox/hq issue 485). `node show <node>` says the send in full.
func printLastSends(nodes []inventory.Node, sends map[string]inventory.Send, unread string) {
if unread != "" {
fmt.Printf("what each machine was last sent could NOT be read: %s\n\n", unread)
return
}
if len(nodes) == 0 {
return
}
fmt.Println("last sent, per machine:")
for _, n := range nodes {
s, found := sends[n.Name]
if !found {
fmt.Printf(" %-12s no send recorded\n", n.Name)
continue
}
generation := "no generation"
if s.Generation > 0 {
generation = fmt.Sprintf("generation %d", s.Generation)
}
fmt.Printf(" %-12s %s by %s, %s — %s\n", n.Name, s.SentAt.Local().Format("2006-01-02 15:04"), s.Sender,
generation, sendAnswerWord(s))
}
fmt.Printf("\n `node show <node>` says a machine's last send in full\n\n")
}
+160
View File
@@ -0,0 +1,160 @@
package main
import (
"encoding/json"
"strings"
"testing"
"time"
"github.com/novox/mesh-controller/internal/inventory"
)
// novox/hq issue 485: the mesh records every send — sender, lease epoch, generation, modules — and only `plan`
// said it. `status` says per machine when it was last sent a declaration and by whom, in one line; `node show`
// says the last send in full and what the machine answered; and `status --json` carries it as fields, since
// the store's reader may not read the controller's tables and these verbs are the only window onto them.
func aLastSend(answer inventory.SendAnswer) inventory.Send {
sent := time.Date(2026, 10, 11, 18, 40, 5, 0, time.Local)
return inventory.Send{NodeName: "laptop", Sequence: 12, Epoch: 57,
Sender: "a one-shot push by jochen at a shell on anchor", Generation: 40, Digest: "d12",
Modules: []string{"docker", "sudo"}, SentAt: sent, Recorded: true, Answer: answer}
}
func TestStatusSaysEachMachinesLastSendInOneLine(t *testing.T) {
at := time.Date(2026, 10, 11, 18, 41, 0, 0, time.Local)
asked := answers{
nodes: []inventory.Node{{Name: "anchor", LastSeen: time.Now()}, {Name: "laptop", LastSeen: time.Now()}},
lastSent: map[string]inventory.Send{"laptop": aLastSend(inventory.SendAnswer{
Outcome: inventory.OutcomeApplied, At: &at})},
}
shown := printed(t, func() error { return printStatus(asked) })
var laptop, anchor []string
for _, line := range strings.Split(shown, "\n") {
fields := strings.Fields(line)
if len(fields) > 0 && fields[0] == "laptop" {
laptop = append(laptop, line)
}
if len(fields) > 0 && fields[0] == "anchor" {
anchor = append(anchor, line)
}
}
if len(laptop) != 1 || !strings.Contains(laptop[0], "2026-10-11 18:40") ||
!strings.Contains(laptop[0], "by a one-shot push by jochen at a shell on anchor") ||
!strings.Contains(laptop[0], "generation 40") || !strings.Contains(laptop[0], "applied") {
t.Fatalf("status does not say laptop's last send in one line:\n%s", shown)
}
if len(anchor) != 1 || !strings.Contains(anchor[0], "no send recorded") {
t.Fatalf("status does not say anchor has no send recorded, in one line:\n%s", shown)
}
}
func TestStatusSaysTheLastSendsCouldNotBeRead(t *testing.T) {
asked := answers{nodes: []inventory.Node{{Name: "laptop", LastSeen: time.Now()}},
lastSentUnread: "the store went away"}
if shown := printed(t, func() error { return printStatus(asked) }); !strings.Contains(shown, "the store went away") {
t.Fatalf("a record of sends that could not be read is not said:\n%s", shown)
}
if asked.well() {
t.Error("a mesh whose sends could not be read is called well")
}
}
func TestNodeShowSaysTheLastSendInFull(t *testing.T) {
at := time.Date(2026, 10, 11, 18, 41, 0, 0, time.Local)
refused := strings.Join(lastSendLines(aLastSend(inventory.SendAnswer{Outcome: inventory.OutcomeRefused,
Refused: "unknown field \"generation\"\nsecond line", At: &at})), "\n")
for _, want := range []string{"sequence 12", "2026-10-11 18:40:05", "a one-shot push by jochen at a shell on anchor",
"lease epoch 57", "generation 40", "docker, sudo", "refused", "unknown field \"generation\""} {
if !strings.Contains(refused, want) {
t.Errorf("node show's last send does not say %q:\n%s", want, refused)
}
}
if applied := strings.Join(lastSendLines(aLastSend(inventory.SendAnswer{Outcome: inventory.OutcomeApplied,
At: &at})), "\n"); !strings.Contains(applied, "applied it at 2026-10-11 18:41:00") {
t.Errorf("an applied send is not said applied:\n%s", applied)
}
if waiting := strings.Join(lastSendLines(aLastSend(inventory.SendAnswer{})), "\n"); !strings.Contains(waiting,
"not reported on it yet") {
t.Errorf("a send not reported on is not said so:\n%s", waiting)
}
stale := aLastSend(inventory.SendAnswer{})
when := at
stale.RefusedAt, stale.RefusedApplied = &when, 44
if s := strings.Join(lastSendLines(stale), "\n"); !strings.Contains(s, "refused it at 2026-10-11 18:41:00") ||
!strings.Contains(s, "generation 44") {
t.Errorf("a send refused for its generation is not said so:\n%s", s)
}
}
func TestStatusJSONCarriesEachMachinesLastSend(t *testing.T) {
at := time.Date(2026, 10, 11, 18, 41, 0, 0, time.UTC)
asked := answers{nodes: []inventory.Node{{Name: "anchor"}, {Name: "laptop"}},
lastSent: map[string]inventory.Send{"laptop": aLastSend(inventory.SendAnswer{Outcome: inventory.OutcomeRefused,
Refused: "unknown field", At: &at})}}
body, err := statusAsJSON(asked)
if err != nil {
t.Fatal(err)
}
var got struct {
LastSent []map[string]any `json:"lastSent"`
}
if err := json.Unmarshal(body, &got); err != nil {
t.Fatal(err)
}
if len(got.LastSent) != 1 {
t.Fatalf("status --json carries %d last sends, want laptop's only:\n%s", len(got.LastSent), body)
}
s := got.LastSent[0]
if s["node"] != "laptop" || s["sequence"] != float64(12) || s["epoch"] != float64(57) ||
s["generation"] != float64(40) || s["sender"] != "a one-shot push by jochen at a shell on anchor" ||
s["digest"] != "d12" || s["sentAt"] == nil || s["answer"] != "refused" || s["refused"] != "unknown field" ||
s["answeredAt"] == nil {
t.Fatalf("laptop's last send reads %v", s)
}
if modules, _ := s["modules"].([]any); len(modules) != 2 || modules[0] != "docker" {
t.Fatalf("the modules it named read %v", s["modules"])
}
}
// And through the store: a send recorded and refused by the machine is what `node show` and `status --json`
// read back.
func TestNodeShowAndStatusReadTheLastSendFromTheStore(t *testing.T) {
open := aMesh(t)
ctx := t.Context()
record, err := open.inventory.NodeByName(ctx, "laptop")
if err != nil {
t.Fatal(err)
}
if err := open.inventory.RecordSentWith(ctx, record.ID, "d12", nil, 57, 40, inventory.Send{Sequence: 12,
Epoch: 57, Sender: "a one-shot push by jochen at a shell on anchor", Generation: 40, Digest: "d12",
Modules: []string{"docker", "sudo"}}); err != nil {
t.Fatal(err)
}
if _, err := open.inventory.RecordDoing(ctx, record.ID, inventory.Doing{Outcome: inventory.OutcomeRefused,
Declared: "d12", Refused: "unknown field \"generation\""}); err != nil {
t.Fatal(err)
}
shown := printed(t, func() error { return showNode(ctx, open, "laptop") })
for _, want := range []string{"a one-shot push by jochen at a shell on anchor", "lease epoch 57", "generation 40",
"docker, sudo", "unknown field \"generation\""} {
if !strings.Contains(shown, want) {
t.Errorf("node show laptop does not say %q:\n%s", want, shown)
}
}
body := printed(t, func() error { return statusFor(ctx, open, true) })
var got struct {
LastSent []struct {
Node string `json:"node"`
Sender string `json:"sender"`
Answer string `json:"answer"`
} `json:"lastSent"`
}
if err := json.Unmarshal([]byte(body), &got); err != nil {
t.Fatalf("%v:\n%s", err, body)
}
if len(got.LastSent) != 1 || got.LastSent[0].Node != "laptop" || got.LastSent[0].Answer != "refused" ||
got.LastSent[0].Sender != "a one-shot push by jochen at a shell on anchor" {
t.Fatalf("status --json's last sends read %+v", got.LastSent)
}
}
+13
View File
@@ -804,6 +804,19 @@ func showNode(ctx context.Context, stored *stores, name string) error {
if err := showMode(ctx, inv, node); err != nil {
return err
}
// What it was last sent, by whom, under which epoch, from which generation, naming which modules, and what
// it answered (novox/hq issue 485): the record of sends is read through this verb, never the store's reader.
// Unreadable is said, not taken for none.
switch last, found, err := inv.LastSend(ctx, name); {
case err != nil:
fmt.Printf(" what it was last sent could NOT be read: %v\n", err)
case !found:
fmt.Printf(" last sent no send recorded since the mesh began keeping them\n")
default:
for _, line := range lastSendLines(last) {
fmt.Println(line)
}
}
// Whom agents run as here, and whether that account can become root without a person (ADR 0266).
for _, line := range agentAccountLines(ctx, inv, node) {
fmt.Println(line)
+45
View File
@@ -109,6 +109,33 @@ type meshStatus struct {
// why they could not be read.
Pending []pendingStatus `json:"pending,omitempty"`
PendingUnread string `json:"pendingUnread,omitempty"`
// LastSent is every machine's last recorded send — who sent it, under which lease epoch, from which
// assignment generation, naming which modules — and what the machine answered of it (novox/hq issue 485).
// A machine the mesh has recorded no send to is absent. LastSentUnread says why it could not be read.
LastSent []machineLastSent `json:"lastSent"`
LastSentUnread string `json:"lastSentUnread,omitempty"`
}
// machineLastSent is one machine's last send as status says it.
type machineLastSent struct {
Node string `json:"node"`
Sequence int64 `json:"sequence"`
SentAt time.Time `json:"sentAt"`
Sender string `json:"sender"`
Epoch int64 `json:"epoch"`
Generation int64 `json:"generation"`
Digest string `json:"digest"`
Modules []string `json:"modules"`
// Answer is what the machine said of it: applied, failed or refused, as its report has it; empty when it
// has not reported on it. Refused is its words when it refused it whole, Failed how many resources failed.
Answer string `json:"answer"`
Refused string `json:"refused,omitempty"`
Failed int `json:"failed,omitempty"`
AnsweredAt *time.Time `json:"answeredAt,omitempty"`
// RefusedForGeneration is when the machine refused it for coming from an older assignment generation than
// it applied, and AppliedGeneration that generation; absent when it did not.
RefusedForGeneration *time.Time `json:"refusedForGeneration,omitempty"`
AppliedGeneration int64 `json:"appliedGeneration,omitempty"`
}
// pendingStatus is one pending assignment as status says it.
@@ -267,6 +294,7 @@ func statusAsJSON(asked answers) ([]byte, error) {
out.Pending = append(out.Pending, pendingStatus{Node: p.Node, Module: p.Module, State: p.State,
Build: p.Build, Repository: p.Repository, Path: p.Path, Since: p.Since, Settled: p.Settled, Note: p.Note})
}
out.LastSent, out.LastSentUnread = lastSentStatus(asked.lastSent), asked.lastSentUnread
for name := range asked.refused {
out.Unresolved = append(out.Unresolved, machineUnresolved{
Node: name, Problem: asked.refused[name]})
@@ -323,3 +351,20 @@ func say(value any) error {
fmt.Println(string(body))
return nil
}
// lastSentStatus is every machine's last send as status --json says it, by machine name (novox/hq issue 485).
func lastSentStatus(sends map[string]inventory.Send) []machineLastSent {
out := []machineLastSent{}
for _, s := range sends {
modules := s.Modules
if modules == nil {
modules = []string{}
}
out = append(out, machineLastSent{Node: s.NodeName, Sequence: s.Sequence, SentAt: s.SentAt, Sender: s.Sender,
Epoch: s.Epoch, Generation: s.Generation, Digest: s.Digest, Modules: modules, Answer: s.Answer.Outcome,
Refused: s.Answer.Refused, Failed: s.Answer.Failed, AnsweredAt: s.Answer.At,
RefusedForGeneration: s.RefusedAt, AppliedGeneration: s.RefusedApplied})
}
sort.Slice(out, func(i, j int) bool { return out[i].Node < out[j].Node })
return out
}
+12 -1
View File
@@ -341,6 +341,10 @@ func printStatus(asked answers) error {
"lists them, and each is in its condition's tried\n\n", asked.heals.Acts, asked.heals.Escalated)
}
// What each machine was last sent and by whom, one line each (novox/hq issue 485). Not a fault, so it does not
// break "all well"; a record that could not be read does.
printLastSends(nodes, asked.lastSent, asked.lastSentUnread)
if adopted := adoptedNodes(nodes); len(adopted) > 0 {
// Said, because nothing forces the flip: a node left adopted is visible here rather than
// read as converged (novox/hq ADR 0100). Not a fault, so it does not break "all well".
@@ -485,6 +489,12 @@ func theThreeQuestions(ctx context.Context, open *stores) (answers, error) {
out.pendingUnread = err.Error()
err = nil
}
// And what each machine was last sent, by whom (novox/hq issue 485): the record of sends is read through
// the controller's verbs alone, the store's reader being shut out of its tables. Unreadable is said.
if out.lastSent, err = inv.LastSends(ctx); err != nil {
out.lastSentUnread = err.Error()
err = nil
}
out.plans, err = inv.RecentPlans(ctx, 5)
if err != nil {
return answers{}, err
@@ -617,7 +627,8 @@ func (a answers) well() bool {
return len(a.wrong) == 0 && len(a.quiet) == 0 && len(a.behind) == 0 &&
len(a.waiting) == 0 && len(a.refused) == 0 && a.network == "" && len(a.untaken) == 0 && len(a.leftOut) == 0 &&
len(a.filtered) == 0 && len(a.unheld) == 0 && len(a.overflowing) == 0 &&
len(a.conditions) == 0 && a.conditionsUnread == "" && a.pendingUnread == "" && !pendingOpen(a.pending)
len(a.conditions) == 0 && a.conditionsUnread == "" && a.pendingUnread == "" && !pendingOpen(a.pending) &&
a.lastSentUnread == ""
}
// pendingOpen is whether any pending assignment is still to be made. One that ended without being made is
+62 -9
View File
@@ -100,6 +100,23 @@ type Send struct {
CounterRaisedTo int64
// Recorded is false for a refusal of a sequence the mesh has no send for: its sender is unknown.
Recorded bool
// Answer is what the machine last reported of this send; filled only where a last send is read back
// (LastSend, LastSends), the zero answer elsewhere.
Answer SendAnswer
}
// SendAnswer is what a machine reported of one send, read from its last report when that report is about the
// send's declaration (novox/hq issue 485): whether it applied it, failed part of it or refused it, and why.
// The zero answer is a machine that has not reported on this send — yet, or since a later report replaced it.
type SendAnswer struct {
// Outcome is applied, failed or refused, as the machine reported it; empty when it has not reported on it.
Outcome string
// Refused is the machine's own words when it refused the declaration whole.
Refused string
// Failed is how many of its resources failed, when some did.
Failed int
// At is when the machine reported it; nil when it has not.
At *time.Time
}
// unknownSender is the sender of a refused sequence the mesh has no record of sending.
@@ -209,20 +226,56 @@ func scanSends(rows pgx.Rows) ([]Send, error) {
return out, rows.Err()
}
// LastSend is the last declaration a machine was sent, by its name; false when none was recorded. A refusal
// of a sequence the mesh has no send for is not a send, and is not it.
// LastSend is the last declaration a machine was sent, by its name, with what the machine answered of it; false
// when none was recorded. A refusal of a sequence the mesh has no send for is not a send, and is not it.
func (i *Inventory) LastSend(ctx context.Context, name string) (Send, bool, error) {
rows, err := i.store.Pool().Query(ctx, `select `+sendColumns+`
from declaration_send s join node n on n.id = s.node
where n.name = $1 and s.recorded order by s.id desc limit 1`, name)
sends, err := i.lastSends(ctx, `n.name = $1`, name)
if err != nil {
return Send{}, false, err
}
sends, err := scanSends(rows)
if err != nil || len(sends) == 0 {
return Send{}, false, err
s, found := sends[name]
return s, found, nil
}
// LastSends is every machine's last send, by machine name, each with what the machine answered of it: what
// `status` says per machine (novox/hq issue 485). A machine never sent anything has no entry.
func (i *Inventory) LastSends(ctx context.Context) (map[string]Send, error) {
return i.lastSends(ctx, `true`)
}
// lastSends reads the newest recorded send of each machine matching where, beside the machine's last report
// when that report is about the send (novox/hq issue 485). It names the send's declaration (its digest), and:
// - from an ordered node-engine, one that reports the epoch and sequence it acted on, those are the send's
// own. A periodic report about an identical earlier send names the same digest and may arrive after a later
// send the machine never received; only the sequence tells them apart. It also holds for a fast machine
// whose report lands before the send's row is stamped.
// - from an engine that reports no order, the report came after the send — both times the store's own — so
// a machine sent again a declaration it once applied does not read as having applied the new send.
func (i *Inventory) lastSends(ctx context.Context, where string, args ...any) (map[string]Send, error) {
rows, err := i.store.Pool().Query(ctx, `select distinct on (n.name) `+sendColumns+`,
coalesce(r.outcome, ''), coalesce(r.refused, ''), case when jsonb_typeof(r.failed) = 'array' then jsonb_array_length(r.failed) else 0 end, r.at
from declaration_send s join node n on n.id = s.node
left join node_report r on r.node = s.node and r.declared <> '' and r.declared = s.digest
and case when r.reported_sequence is not null
then r.reported_sequence = s.sequence and coalesce(r.reported_epoch, 0) = coalesce(s.epoch, 0)
else r.at >= s.sent_at end
where s.recorded and `+where+`
order by n.name, s.id desc`, args...)
if err != nil {
return nil, err
}
return sends[0], true, nil
defer rows.Close()
out := map[string]Send{}
for rows.Next() {
var s Send
if err := rows.Scan(&s.Node, &s.NodeName, &s.Sequence, &s.Epoch, &s.Sender, &s.Generation, &s.Digest,
&s.Modules, &s.SentAt, &s.RefusedAt, &s.RefusedApplied, &s.CounterRaisedTo, &s.Recorded,
&s.Answer.Outcome, &s.Answer.Refused, &s.Answer.Failed, &s.Answer.At); err != nil {
return nil, err
}
out[s.NodeName] = s
}
return out, rows.Err()
}
// RefusedSend keeps a machine's refusal of the send of a sequence for the generation it came from, and
+141
View File
@@ -205,3 +205,144 @@ func TestWhetherAMachineReadsAGenerationIsItsLatestWord(t *testing.T) {
}
}
}
// **The last send says what the machine answered of it** (novox/hq issue 485): applied, refused with its
// words, or not answered yet — read from the machine's report only when that report is about the send's
// declaration and came after it, so a report about an earlier send is never this one's answer. And every
// machine's last send is read at once, for status, by name; a machine never sent anything has none.
func TestTheLastSendSaysWhatTheMachineAnswered(t *testing.T) {
inv, node := aNodeWithModules(t)
ctx := t.Context()
record, err := inv.NodeByName(ctx, node)
if err != nil {
t.Fatal(err)
}
if _, err := inv.AddNode(ctx, "laptop"); err != nil {
t.Fatal(err)
}
send := func(sequence int64, digest string) {
t.Helper()
if err := inv.RecordSentWith(ctx, record.ID, digest, nil, 57, 40, Send{Sequence: sequence, Epoch: 57,
Sender: "the controller's daemon (pid 7 on anchor)", Generation: 40, Digest: digest,
Modules: []string{"docker", "sudo"}}); err != nil {
t.Fatal(err)
}
}
report := func(d Doing) {
t.Helper()
if _, err := inv.RecordDoing(ctx, record.ID, d); err != nil {
t.Fatal(err)
}
}
answer := func() SendAnswer {
t.Helper()
last, found, err := inv.LastSend(ctx, node)
if err != nil || !found {
t.Fatalf("the last send is not read back: %v, %v", found, err)
}
return last.Answer
}
send(11, "d11")
if a := answer(); a.Outcome != "" || a.At != nil {
t.Fatalf("a send not reported on reads as answered: %+v", a)
}
report(Doing{Outcome: OutcomeApplied, Declared: "d11", Applied: 3})
if a := answer(); a.Outcome != OutcomeApplied || a.At == nil {
t.Fatalf("a send the machine reported applying reads %+v", a)
}
send(12, "d12")
if a := answer(); a.Outcome != "" {
t.Fatalf("the report about the send before is taken for this one's answer: %+v", a)
}
report(Doing{Outcome: OutcomeRefused, Declared: "d12", Refused: "unknown field \"generation\"\nand more"})
if a := answer(); a.Outcome != OutcomeRefused || a.Refused != "unknown field \"generation\"\nand more" {
t.Fatalf("a send the machine refused reads %+v", a)
}
// Sent again a declaration it reported applying before the send: not answered until it reports again.
report(Doing{Outcome: OutcomeApplied, Declared: "d12", Applied: 3})
send(13, "d12")
if a := answer(); a.Outcome != "" {
t.Fatalf("a report from before the send is taken for its answer: %+v", a)
}
all, err := inv.LastSends(ctx)
if err != nil {
t.Fatal(err)
}
if len(all) != 1 || all[node].Sequence != 13 || all[node].Epoch != 57 || all[node].Generation != 40 ||
!slices.Equal(all[node].Modules, []string{"docker", "sudo"}) {
t.Fatalf("every machine's last send reads %+v", all)
}
if _, found := all["laptop"]; found {
t.Error("a machine never sent anything has a last send")
}
}
// **A report about another declaration, or another send of the same one, is not the last send's answer**
// (novox/hq issue 485, review of mesh-controller #234). A report naming an earlier send's digest that arrives
// after a later send is not the later send's answer. And from an ordered node-engine, a periodic report about
// an identical earlier send — the same digest, arriving after the later send — is not its answer either: only
// the report's epoch and sequence say which send it is about.
func TestAReportIsTheLastSendsAnswerOnlyWhenItIsAboutThatSend(t *testing.T) {
inv, node := aNodeWithModules(t)
ctx := t.Context()
record, err := inv.NodeByName(ctx, node)
if err != nil {
t.Fatal(err)
}
send := func(sequence int64, digest string) {
t.Helper()
if err := inv.RecordSentWith(ctx, record.ID, digest, nil, 57, 40, Send{Sequence: sequence, Epoch: 57,
Sender: "the controller's daemon (pid 7 on anchor)", Generation: 40, Digest: digest}); err != nil {
t.Fatal(err)
}
}
ordered := func(epoch, sequence int64, digest string) {
t.Helper()
if _, err := inv.RecordOrderedDoing(ctx, record.ID, Doing{Outcome: OutcomeApplied, Declared: digest, Applied: 1},
ReportOrder{Epoch: epoch, Sequence: sequence}, func(ReportOrder) bool { return false }); err != nil {
t.Fatal(err)
}
}
answer := func() string {
t.Helper()
last, found, err := inv.LastSend(ctx, node)
if err != nil || !found {
t.Fatalf("the last send is not read back: %v, %v", found, err)
}
return last.Answer.Outcome
}
// An engine that reports no order: the digest decides.
send(11, "d11")
send(12, "d12")
if _, err := inv.RecordDoing(ctx, record.ID, Doing{Outcome: OutcomeApplied, Declared: "d11"}); err != nil {
t.Fatal(err)
}
if a := answer(); a != "" {
t.Fatalf("a report about d11, after d12 was sent, reads as d12's answer: %q", a)
}
// An ordered engine: the epoch and sequence decide, whatever the digest and the time.
send(20, "d20")
ordered(57, 20, "d20")
if a := answer(); a != OutcomeApplied {
t.Fatalf("an ordered report about sequence 20 is not its answer: %q", a)
}
send(21, "d20")
ordered(57, 20, "d20")
if a := answer(); a != "" {
t.Fatalf("a periodic report about sequence 20, after 21 was sent with the same digest, reads as 21's answer: %q", a)
}
ordered(56, 21, "d20")
if a := answer(); a != "" {
t.Fatalf("a report about sequence 21 of another epoch reads as this send's answer: %q", a)
}
ordered(57, 21, "d20")
if a := answer(); a != OutcomeApplied {
t.Fatalf("an ordered report about sequence 21 is not its answer: %q", a)
}
}
+3
View File
@@ -7,3 +7,6 @@
./cmd/mesh-controller TestAStopNamesWhatStaysOnTheMachinesAndWhatStillCarriesIt
./cmd/mesh-controller TestAStopWithoutAMergeLeavesItOutOfABatch
./cmd/mesh-controller TestSavingPlansAndMovingMergesIsOneAct
./internal/inventory TestTheLastSendSaysWhatTheMachineAnswered
./internal/inventory TestAReportIsTheLastSendsAnswerOnlyWhenItIsAboutThatSend
./cmd/mesh-controller TestNodeShowAndStatusReadTheLastSendFromTheStore